From 5df3b41bd6b5432010d6d85e5aa7217bd8f6b0cb Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Tue, 9 Jan 2024 11:04:49 +0100 Subject: [PATCH 0001/1038] regulator: qcom_smd: Keep one rpm handle for all vregs For no apparent reason (as there's just one RPM per SoC), all vregs currently store a copy of a pointer to smd_rpm. Introduce a single, global one to save up on space in each definition. bloat-o-meter reports a slight uptick: Total: Before=44008, After=44080, chg +0.16% However the saved n * sizeof(ptr) for every dynamically allocated regulator quickly makes up for it. Signed-off-by: Konrad Dybcio Link: https://msgid.link/r/20240109-rpm_vreg_cleanup-v3-1-fa0201029f78@linaro.org Signed-off-by: Mark Brown --- drivers/regulator/qcom_smd-regulator.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c index d1be9568025e..3b7e06b9f5ce 100644 --- a/drivers/regulator/qcom_smd-regulator.c +++ b/drivers/regulator/qcom_smd-regulator.c @@ -11,11 +11,10 @@ #include #include +struct qcom_smd_rpm *smd_vreg_rpm; + struct qcom_rpm_reg { struct device *dev; - - struct qcom_smd_rpm *rpm; - u32 type; u32 id; @@ -70,7 +69,7 @@ static int rpm_reg_write_active(struct qcom_rpm_reg *vreg) if (!reqlen) return 0; - ret = qcom_rpm_smd_write(vreg->rpm, QCOM_SMD_RPM_ACTIVE_STATE, + ret = qcom_rpm_smd_write(smd_vreg_rpm, QCOM_SMD_RPM_ACTIVE_STATE, vreg->type, vreg->id, req, sizeof(req[0]) * reqlen); if (!ret) { @@ -1384,14 +1383,13 @@ MODULE_DEVICE_TABLE(of, rpm_of_match); * @dev: Pointer to the top level qcom_smd-regulator PMIC device * @node: Pointer to the individual qcom_smd-regulator resource * device node - * @rpm: Pointer to the rpm bus node * @pmic_rpm_data: Pointer to a null-terminated array of qcom_smd-regulator * resources defined for the top level PMIC device * * Return: 0 on success, errno on failure */ static int rpm_regulator_init_vreg(struct qcom_rpm_reg *vreg, struct device *dev, - struct device_node *node, struct qcom_smd_rpm *rpm, + struct device_node *node, const struct rpm_regulator_data *pmic_rpm_data) { struct regulator_config config = {}; @@ -1409,7 +1407,6 @@ static int rpm_regulator_init_vreg(struct qcom_rpm_reg *vreg, struct device *dev } vreg->dev = dev; - vreg->rpm = rpm; vreg->type = rpm_data->type; vreg->id = rpm_data->id; @@ -1449,6 +1446,11 @@ static int rpm_reg_probe(struct platform_device *pdev) return -ENODEV; } + if (smd_vreg_rpm && rpm != smd_vreg_rpm) + return dev_err_probe(dev, -EINVAL, "RPM mismatch\n"); + + smd_vreg_rpm = rpm; + vreg_data = of_device_get_match_data(dev); if (!vreg_data) return -ENODEV; @@ -1460,8 +1462,7 @@ static int rpm_reg_probe(struct platform_device *pdev) return -ENOMEM; } - ret = rpm_regulator_init_vreg(vreg, dev, node, rpm, vreg_data); - + ret = rpm_regulator_init_vreg(vreg, dev, node, vreg_data); if (ret < 0) { of_node_put(node); return ret; From a6ffa02ba7a28fe1bba06eacb224554d89a1cd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Apitzsch?= Date: Thu, 4 Jan 2024 23:40:40 +0100 Subject: [PATCH 0002/1038] dt-bindings: regulator: Convert ti,tps65132 to YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert TI TPS65132 bindings from .txt to .yaml format. Signed-off-by: André Apitzsch Reviewed-by: Krzysztof Kozlowski Link: https://msgid.link/r/20240104-tps65132-v2-1-6c77fe2f4787@apitzsch.eu Signed-off-by: Mark Brown --- .../bindings/regulator/ti,tps65132.yaml | 84 +++++++++++++++++++ .../bindings/regulator/tps65132-regulator.txt | 46 ---------- 2 files changed, 84 insertions(+), 46 deletions(-) create mode 100644 Documentation/devicetree/bindings/regulator/ti,tps65132.yaml delete mode 100644 Documentation/devicetree/bindings/regulator/tps65132-regulator.txt diff --git a/Documentation/devicetree/bindings/regulator/ti,tps65132.yaml b/Documentation/devicetree/bindings/regulator/ti,tps65132.yaml new file mode 100644 index 000000000000..6a6d1a3d6fa7 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/ti,tps65132.yaml @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/ti,tps65132.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI TPS65132 Dual Output Power Regulators + +maintainers: + - devicetree@vger.kernel.org + +description: | + The TPS65132 is designed to supply positive/negative driven applications. + + Datasheet is available at: + https://www.ti.com/lit/gpn/tps65132 + +properties: + compatible: + enum: + - ti,tps65132 + + reg: + maxItems: 1 + +patternProperties: + "^out[pn]$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + description: + Properties for single regulator. + + properties: + enable-gpios: + maxItems: 1 + description: + GPIO specifier to enable the GPIO control (on/off) for regulator. + + active-discharge-gpios: + maxItems: 1 + description: + GPIO specifier to actively discharge the delay mechanism. + + ti,active-discharge-time-us: + description: Regulator active discharge time in microseconds. + + dependencies: + active-discharge-gpios: [ 'ti,active-discharge-time-us' ] + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + regulator@3e { + compatible = "ti,tps65132"; + reg = <0x3e>; + + outp { + regulator-name = "outp"; + regulator-boot-on; + regulator-always-on; + enable-gpios = <&gpio 23 GPIO_ACTIVE_HIGH>; + }; + + outn { + regulator-name = "outn"; + regulator-boot-on; + regulator-always-on; + regulator-active-discharge = <0>; + enable-gpios = <&gpio 40 GPIO_ACTIVE_HIGH>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/tps65132-regulator.txt b/Documentation/devicetree/bindings/regulator/tps65132-regulator.txt deleted file mode 100644 index 3a3505520c69..000000000000 --- a/Documentation/devicetree/bindings/regulator/tps65132-regulator.txt +++ /dev/null @@ -1,46 +0,0 @@ -TPS65132 regulators - -Required properties: -- compatible: "ti,tps65132" -- reg: I2C slave address - -Optional Subnode: -Device supports two regulators OUTP and OUTN. A sub node within the - device node describe the properties of these regulators. The sub-node - names must be as follows: - -For regulator outp, the sub node name should be "outp". - -For regulator outn, the sub node name should be "outn". - --enable-gpios:(active high, output) Regulators are controlled by the input pins. - If it is connected to GPIO through host system then provide the - gpio number as per gpio.txt. --active-discharge-gpios: (active high, output) Some configurations use delay mechanisms - on the enable pin, to keep the regulator enabled for some time after - the enable signal goes low. This GPIO is used to actively discharge - the delay mechanism. Requires specification of ti,active-discharge-time-us --ti,active-discharge-time-us: how long the active discharge gpio should be - asserted for during active discharge, in microseconds. - -Each regulator is defined using the standard binding for regulators. - -Example: - - tps65132@3e { - compatible = "ti,tps65132"; - reg = <0x3e>; - - outp { - regulator-name = "outp"; - regulator-boot-on; - regulator-always-on; - enable-gpios = <&gpio 23 0>; - }; - - outn { - regulator-name = "outn"; - regulator-boot-on; - regulator-always-on; - regulator-active-discharge = <0>; - enable-gpios = <&gpio 40 0>; - }; - }; From 4247d7f2ca564bbede54054da1f35a359bb061bd Mon Sep 17 00:00:00 2001 From: Ruihai Zhou Date: Mon, 8 Jan 2024 20:08:02 +0800 Subject: [PATCH 0003/1038] spi: spi-mt65xx: Support sleep pin control Supports configuring sleep pin control during system suspend to prevent potential power leakage and additional power consumption. Signed-off-by: Ruihai Zhou Link: https://msgid.link/r/20240108120802.7601-1-zhouruihai@huaqin.corp-partner.google.com Signed-off-by: Mark Brown --- drivers/spi/spi-mt65xx.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 8d5d170d49cc..8d4633b353ee 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1316,6 +1317,8 @@ static int mtk_spi_suspend(struct device *dev) clk_disable_unprepare(mdata->spi_hclk); } + pinctrl_pm_select_sleep_state(dev); + return 0; } @@ -1325,6 +1328,8 @@ static int mtk_spi_resume(struct device *dev) struct spi_controller *host = dev_get_drvdata(dev); struct mtk_spi *mdata = spi_controller_get_devdata(host); + pinctrl_pm_select_default_state(dev); + if (!pm_runtime_suspended(dev)) { ret = clk_prepare_enable(mdata->spi_clk); if (ret < 0) { From 6df534cc7136fc9e023cbd4e0011a04e3659e74d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 5 Jan 2024 11:32:50 +0100 Subject: [PATCH 0004/1038] spi: make spi_bus_type const Now that the driver core can properly handle constant struct bus_type, move the spi_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Mark Brown Cc: Signed-off-by: Greg Kroah-Hartman Link: https://msgid.link/r/2024010549-erasure-swoop-1cc6@gregkh Signed-off-by: Mark Brown --- drivers/spi/spi.c | 2 +- include/linux/spi/spi.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 7477a11e12be..bc3d7c0ce6b4 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -459,7 +459,7 @@ static void spi_shutdown(struct device *dev) } } -struct bus_type spi_bus_type = { +const struct bus_type spi_bus_type = { .name = "spi", .dev_groups = spi_dev_groups, .match = spi_match_device, diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 471fe2ff9066..f306aececeaf 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -36,7 +36,7 @@ struct spi_message; * INTERFACES between SPI master-side drivers and SPI slave protocol handlers, * and SPI infrastructure. */ -extern struct bus_type spi_bus_type; +extern const struct bus_type spi_bus_type; /** * struct spi_statistics - statistics for spi transfers From 50c4f8172dbf9fdf152eae5179e233879a72dd47 Mon Sep 17 00:00:00 2001 From: Alexander Stein Date: Thu, 18 Jan 2024 13:10:16 +0100 Subject: [PATCH 0005/1038] spi: nxp-fspi: Adjust LUT debug output alignment Ensure the command value and LUT entry values have a fixed width. This way consecutive output lines can be read much easier. Signed-off-by: Alexander Stein Link: https://msgid.link/r/20240118121016.3734770-1-alexander.stein@ew.tq-group.com Signed-off-by: Mark Brown --- drivers/spi/spi-nxp-fspi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index e13f678f2395..88397f712a3b 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -591,7 +591,7 @@ static void nxp_fspi_prepare_lut(struct nxp_fspi *f, for (i = 0; i < ARRAY_SIZE(lutval); i++) fspi_writel(f, lutval[i], base + FSPI_LUT_REG(i)); - dev_dbg(f->dev, "CMD[%x] lutval[0:%x \t 1:%x \t 2:%x \t 3:%x], size: 0x%08x\n", + dev_dbg(f->dev, "CMD[%02x] lutval[0:%08x 1:%08x 2:%08x 3:%08x], size: 0x%08x\n", op->cmd.opcode, lutval[0], lutval[1], lutval[2], lutval[3], op->data.nbytes); /* lock LUT */ From f4028860a998f8a64c4e8c162a8091f7214502cf Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 4 Jan 2024 19:20:34 +0100 Subject: [PATCH 0006/1038] gpio: legacy: mark old interfaces as deprecated in kernel docs We've recently had someone try to use of_get_named_gpio() in new code. Mark legacy interfaces as deprecated in kernel docs to avoid any confusion. Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-legacy.c | 12 ++++++++++++ drivers/gpio/gpiolib-of.c | 2 ++ 2 files changed, 14 insertions(+) diff --git a/drivers/gpio/gpiolib-legacy.c b/drivers/gpio/gpiolib-legacy.c index 97f4b498e343..b138682fec3d 100644 --- a/drivers/gpio/gpiolib-legacy.c +++ b/drivers/gpio/gpiolib-legacy.c @@ -6,6 +6,9 @@ #include "gpiolib.h" +/* + * **DEPRECATED** This function is deprecated and must not be used in new code. + */ void gpio_free(unsigned gpio) { gpiod_free(gpio_to_desc(gpio)); @@ -17,6 +20,8 @@ EXPORT_SYMBOL_GPL(gpio_free); * @gpio: the GPIO number * @flags: GPIO configuration as specified by GPIOF_* * @label: a literal description string of this GPIO + * + * **DEPRECATED** This function is deprecated and must not be used in new code. */ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) { @@ -53,6 +58,9 @@ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) } EXPORT_SYMBOL_GPL(gpio_request_one); +/* + * **DEPRECATED** This function is deprecated and must not be used in new code. + */ int gpio_request(unsigned gpio, const char *label) { struct gpio_desc *desc = gpio_to_desc(gpio); @@ -69,6 +77,8 @@ EXPORT_SYMBOL_GPL(gpio_request); * gpio_request_array - request multiple GPIOs in a single call * @array: array of the 'struct gpio' * @num: how many GPIOs in the array + * + * **DEPRECATED** This function is deprecated and must not be used in new code. */ int gpio_request_array(const struct gpio *array, size_t num) { @@ -92,6 +102,8 @@ EXPORT_SYMBOL_GPL(gpio_request_array); * gpio_free_array - release multiple GPIOs in a single call * @array: array of the 'struct gpio' * @num: how many GPIOs in the array + * + * **DEPRECATED** This function is deprecated and must not be used in new code. */ void gpio_free_array(const struct gpio *array, size_t num) { diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index e7770eedd146..77509aa19900 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -414,6 +414,8 @@ out: * @propname: Name of property containing gpio specifier(s) * @index: index of the GPIO * + * **DEPRECATED** This function is deprecated and must not be used in new code. + * * Returns GPIO number to use with Linux generic GPIO API, or one of the errno * value on the error condition. */ From 7889968e64754149f9c84a6ebcafee7db856d8dc Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 9 Jan 2024 22:02:17 +0800 Subject: [PATCH 0007/1038] gpio: uapi: improve description of fd fields Only a successful operation modifies fd fields, but the current documentation wording could be taken to imply that a positive fd value after an ioctl() returns indicates a success. Reword documentation to clarify that the fd is only valid after a successful operation. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- include/uapi/linux/gpio.h | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index cb9966d49a16..650c1099d1a6 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -189,9 +189,8 @@ struct gpio_v2_line_config { * buffer. If this field is zero then the buffer size defaults to a minimum * of @num_lines * 16. * @padding: reserved for future use and must be zero filled - * @fd: if successful this field will contain a valid anonymous file handle - * after a %GPIO_GET_LINE_IOCTL operation, zero or negative value means - * error + * @fd: after a successful %GPIO_V2_GET_LINE_IOCTL operation, contains + * a valid anonymous file descriptor representing the request */ struct gpio_v2_line_request { __u32 offsets[GPIO_V2_LINES_MAX]; @@ -402,9 +401,8 @@ struct gpioline_info_changed { * such as "my-bitbanged-relay" * @lines: number of lines requested in this request, i.e. the number of * valid fields in the above arrays, set to 1 to request a single line - * @fd: if successful this field will contain a valid anonymous file handle - * after a %GPIO_GET_LINEHANDLE_IOCTL operation, zero or negative value - * means error + * @fd: after a successful %GPIO_GET_LINEHANDLE_IOCTL operation, contains + * a valid anonymous file descriptor representing the request * * Note: This struct is part of ABI v1 and is deprecated. * Use &struct gpio_v2_line_request instead. @@ -465,9 +463,8 @@ struct gpiohandle_data { * %GPIOEVENT_REQUEST_RISING_EDGE or %GPIOEVENT_REQUEST_FALLING_EDGE * @consumer_label: a desired consumer label for the selected GPIO line(s) * such as "my-listener" - * @fd: if successful this field will contain a valid anonymous file handle - * after a %GPIO_GET_LINEEVENT_IOCTL operation, zero or negative value - * means error + * @fd: after a successful %GPIO_GET_LINEEVENT_IOCTL operation, contains a + * valid anonymous file descriptor representing the request * * Note: This struct is part of ABI v1 and is deprecated. * Use &struct gpio_v2_line_request instead. From f75d508eeb24de5462c77835229f61d700914db0 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 9 Jan 2024 22:02:18 +0800 Subject: [PATCH 0008/1038] gpio: uapi: clarify hte references The full name of the HTE subsystem is "hardware timestamping engine", so correct references and highlight that this refers to the HTE subsystem. Extend the description of struct gpio_v2_line_event to clarify that the timestamp_ns is sourced from the HTE if the GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE is set. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- include/uapi/linux/gpio.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index 650c1099d1a6..88c2a326940e 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -67,7 +67,7 @@ struct gpiochip_info { * @GPIO_V2_LINE_FLAG_BIAS_DISABLED: line has bias disabled * @GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME: line events contain REALTIME timestamps * @GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE: line events contain timestamps from - * hardware timestamp engine + * the hardware timestamping engine (HTE) subsystem */ enum gpio_v2_line_flag { GPIO_V2_LINE_FLAG_USED = _BITULL(0), @@ -288,6 +288,10 @@ enum gpio_v2_line_event_id { * * If the %GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME flag is set then the * @timestamp_ns is read from %CLOCK_REALTIME. + * + * If the %GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE flag is set then the + * @timestamp_ns is provided by the hardware timestamping engine (HTE) + * subsystem. */ struct gpio_v2_line_event { __aligned_u64 timestamp_ns; From a6beb0b46121d9667208f367d99ca17b9daf7a0c Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 9 Jan 2024 22:02:19 +0800 Subject: [PATCH 0009/1038] gpio: uapi: drop trailing period from one sentence descriptions Make documentation more consistent by using trailing periods only for multi-sentence field descriptions, not single sentence descriptions. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- include/uapi/linux/gpio.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index 88c2a326940e..3ba8996e496c 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -88,10 +88,10 @@ enum gpio_v2_line_flag { /** * struct gpio_v2_line_values - Values of GPIO lines * @bits: a bitmap containing the value of the lines, set to 1 for active - * and 0 for inactive. + * and 0 for inactive * @mask: a bitmap identifying the lines to get or set, with each bit * number corresponding to the index into &struct - * gpio_v2_line_request.offsets. + * gpio_v2_line_request.offsets */ struct gpio_v2_line_values { __aligned_u64 bits; @@ -123,7 +123,7 @@ enum gpio_v2_line_attr_id { * @values: if id is %GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES, a bitmap * containing the values to which the lines will be set, with each bit * number corresponding to the index into &struct - * gpio_v2_line_request.offsets. + * gpio_v2_line_request.offsets * @debounce_period_us: if id is %GPIO_V2_LINE_ATTR_ID_DEBOUNCE, the * desired debounce period, in microseconds */ @@ -143,7 +143,7 @@ struct gpio_v2_line_attribute { * @attr: the configurable attribute * @mask: a bitmap identifying the lines to which the attribute applies, * with each bit number corresponding to the index into &struct - * gpio_v2_line_request.offsets. + * gpio_v2_line_request.offsets */ struct gpio_v2_line_config_attribute { struct gpio_v2_line_attribute attr; @@ -178,7 +178,7 @@ struct gpio_v2_line_config { * associated GPIO chip * @consumer: a desired consumer label for the selected GPIO lines such as * "my-bitbanged-relay" - * @config: requested configuration for the lines. + * @config: requested configuration for the lines * @num_lines: number of lines requested in this request, i.e. the number * of valid fields in the %GPIO_V2_LINES_MAX sized arrays, set to 1 to * request a single line @@ -216,7 +216,7 @@ struct gpio_v2_line_request { * @num_attrs: the number of attributes in @attrs * @flags: flags for this GPIO line, with values from &enum * gpio_v2_line_flag, such as %GPIO_V2_LINE_FLAG_ACTIVE_LOW, - * %GPIO_V2_LINE_FLAG_OUTPUT etc, added together. + * %GPIO_V2_LINE_FLAG_OUTPUT etc, added together * @attrs: the configuration attributes associated with the line * @padding: reserved for future use */ @@ -273,7 +273,7 @@ enum gpio_v2_line_event_id { /** * struct gpio_v2_line_event - The actual event being pushed to userspace - * @timestamp_ns: best estimate of time of event occurrence, in nanoseconds. + * @timestamp_ns: best estimate of time of event occurrence, in nanoseconds * @id: event identifier with value from &enum gpio_v2_line_event_id * @offset: the offset of the line that triggered the event * @seqno: the sequence number for this event in the sequence of events for From ead7c5817cff33aff2762b43caec6bc5b8dfff9a Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 9 Jan 2024 22:02:20 +0800 Subject: [PATCH 0010/1038] gpio: uapi: document possible values of gpioevent_data.id Clarify the possible values of event id, rather than requiring the reader to infer. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- include/uapi/linux/gpio.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index 3ba8996e496c..e68a56969f36 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -490,7 +490,8 @@ struct gpioevent_request { /** * struct gpioevent_data - The actual event being pushed to userspace * @timestamp: best estimate of time of event occurrence, in nanoseconds - * @id: event identifier + * @id: event identifier, one of %GPIOEVENT_EVENT_RISING_EDGE or + * %GPIOEVENT_EVENT_FALLING_EDGE * * Note: This struct is part of ABI v1 and is deprecated. * Use &struct gpio_v2_line_event instead. From b6747ef69fa44717f7ee6bbf012324bfdbdb5aa4 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 9 Jan 2024 22:02:21 +0800 Subject: [PATCH 0011/1038] gpio: uapi: clarify using v2 rather than v1 The documentation contains notes like This struct is part of ABI v1 and is deprecated. Use struct gpio_v2_line_info instead. This could be interpreted to mean the structs can be directly substituted in v1 calls. Clarify that the user should use the corresponding v2 ioctl() and structs. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- include/uapi/linux/gpio.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index e68a56969f36..103cd3c6c81e 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -333,7 +333,7 @@ struct gpio_v2_line_event { * also be empty if the consumer doesn't set this up * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_info instead. + * Use ABI v2 and &struct gpio_v2_line_info instead. */ struct gpioline_info { __u32 line_offset; @@ -368,7 +368,7 @@ enum { * at the end of the structure on 64-bit architectures. * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_info_changed instead. + * Use ABI v2 and &struct gpio_v2_line_info_changed instead. */ struct gpioline_info_changed { struct gpioline_info info; @@ -409,7 +409,7 @@ struct gpioline_info_changed { * a valid anonymous file descriptor representing the request * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_request instead. + * Use ABI v2 and &struct gpio_v2_line_request instead. */ struct gpiohandle_request { __u32 lineoffsets[GPIOHANDLES_MAX]; @@ -431,7 +431,7 @@ struct gpiohandle_request { * @padding: reserved for future use and should be zero filled * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_config instead. + * Use ABI v2 and &struct gpio_v2_line_config instead. */ struct gpiohandle_config { __u32 flags; @@ -446,7 +446,7 @@ struct gpiohandle_config { * the desired target state * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_values instead. + * Use ABI v2 and &struct gpio_v2_line_values instead. */ struct gpiohandle_data { __u8 values[GPIOHANDLES_MAX]; @@ -471,7 +471,7 @@ struct gpiohandle_data { * valid anonymous file descriptor representing the request * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_request instead. + * Use ABI v2 and &struct gpio_v2_line_request instead. */ struct gpioevent_request { __u32 lineoffset; @@ -494,7 +494,7 @@ struct gpioevent_request { * %GPIOEVENT_EVENT_FALLING_EDGE * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_event instead. + * Use ABI v2 and &struct gpio_v2_line_event instead. */ struct gpioevent_data { __u64 timestamp; From f1868165d2d737a85dfce18e1bfada86d0ed4f92 Mon Sep 17 00:00:00 2001 From: Xiongwei Song Date: Fri, 15 Dec 2023 11:41:47 +0800 Subject: [PATCH 0012/1038] Documentation: kernel-parameters: remove noaliencache Since slab allocator has already been removed. There is no users of the noaliencache parameter, so let's remove it. Suggested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Reviewed-by: Kees Cook Reviewed-by: Vlastimil Babka Signed-off-by: Xiongwei Song Signed-off-by: Vlastimil Babka --- Documentation/admin-guide/kernel-parameters.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 31b3a25680d0..89c67c2403fc 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3753,10 +3753,6 @@ no5lvl [X86-64,RISCV] Disable 5-level paging mode. Forces kernel to use 4-level paging instead. - noaliencache [MM, NUMA, SLAB] Disables the allocation of alien - caches in the slab allocator. Saves per-node memory, - but will impact performance. - noalign [KNL,ARM] noaltinstr [S390] Disables alternative instructions patching From 671776b32b26d0cb625bf834170e982fda712cab Mon Sep 17 00:00:00 2001 From: Xiongwei Song Date: Fri, 15 Dec 2023 11:41:48 +0800 Subject: [PATCH 0013/1038] mm/slub: unify all sl[au]b parameters with "slab_$param" Since the SLAB allocator has been removed, so we can clean up the sl[au]b_$params. With only one slab allocator left, it's better to use the generic "slab" term instead of "slub" which is an implementation detail, which is pointed out by Vlastimil Babka. For more information please see [1]. Hence, we are going to use "slab_$param" as the primary prefix. This patch is changing the following slab parameters - slub_max_order - slub_min_order - slub_min_objects - slub_debug to - slab_max_order - slab_min_order - slab_min_objects - slab_debug as the primary slab parameters for all references of them in docs and comments. But this patch won't change variables and functions inside slub as we will have wider slub/slab change. Meanwhile, "slub_$params" can also be passed by command line, which is to keep backward compatibility. Also mark all "slub_$params" as legacy. Remove the separate descriptions for slub_[no]merge, append legacy tip for them at the end of descriptions of slab_[no]merge. [1] https://lore.kernel.org/linux-mm/7512b350-4317-21a0-fab3-4101bc4d8f7a@suse.cz/ Signed-off-by: Xiongwei Song Reviewed-by: Vlastimil Babka Signed-off-by: Vlastimil Babka --- .../admin-guide/kernel-parameters.txt | 75 +++++++++---------- drivers/misc/lkdtm/heap.c | 2 +- mm/Kconfig.debug | 6 +- mm/slab.h | 2 +- mm/slab_common.c | 4 +- mm/slub.c | 41 +++++----- 6 files changed, 64 insertions(+), 66 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 89c67c2403fc..aefd68cec2e9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5891,11 +5891,42 @@ simeth= [IA-64] simscsi= - slram= [HW,MTD] + slab_debug[=options[,slabs][;[options[,slabs]]...] [MM] + Enabling slab_debug allows one to determine the + culprit if slab objects become corrupted. Enabling + slab_debug can create guard zones around objects and + may poison objects when not in use. Also tracks the + last alloc / free. For more information see + Documentation/mm/slub.rst. + (slub_debug legacy name also accepted for now) + + slab_max_order= [MM] + Determines the maximum allowed order for slabs. + A high setting may cause OOMs due to memory + fragmentation. For more information see + Documentation/mm/slub.rst. + (slub_max_order legacy name also accepted for now) slab_merge [MM] Enable merging of slabs with similar size when the kernel is built without CONFIG_SLAB_MERGE_DEFAULT. + (slub_merge legacy name also accepted for now) + + slab_min_objects= [MM] + The minimum number of objects per slab. SLUB will + increase the slab order up to slab_max_order to + generate a sufficiently large slab able to contain + the number of objects indicated. The higher the number + of objects the smaller the overhead of tracking slabs + and the less frequently locks need to be acquired. + For more information see Documentation/mm/slub.rst. + (slub_min_objects legacy name also accepted for now) + + slab_min_order= [MM] + Determines the minimum page order for slabs. Must be + lower or equal to slab_max_order. For more information see + Documentation/mm/slub.rst. + (slub_min_order legacy name also accepted for now) slab_nomerge [MM] Disable merging of slabs with similar size. May be @@ -5909,47 +5940,9 @@ unchanged). Debug options disable merging on their own. For more information see Documentation/mm/slub.rst. + (slub_nomerge legacy name also accepted for now) - slab_max_order= [MM, SLAB] - Determines the maximum allowed order for slabs. - A high setting may cause OOMs due to memory - fragmentation. Defaults to 1 for systems with - more than 32MB of RAM, 0 otherwise. - - slub_debug[=options[,slabs][;[options[,slabs]]...] [MM, SLUB] - Enabling slub_debug allows one to determine the - culprit if slab objects become corrupted. Enabling - slub_debug can create guard zones around objects and - may poison objects when not in use. Also tracks the - last alloc / free. For more information see - Documentation/mm/slub.rst. - - slub_max_order= [MM, SLUB] - Determines the maximum allowed order for slabs. - A high setting may cause OOMs due to memory - fragmentation. For more information see - Documentation/mm/slub.rst. - - slub_min_objects= [MM, SLUB] - The minimum number of objects per slab. SLUB will - increase the slab order up to slub_max_order to - generate a sufficiently large slab able to contain - the number of objects indicated. The higher the number - of objects the smaller the overhead of tracking slabs - and the less frequently locks need to be acquired. - For more information see Documentation/mm/slub.rst. - - slub_min_order= [MM, SLUB] - Determines the minimum page order for slabs. Must be - lower than slub_max_order. - For more information see Documentation/mm/slub.rst. - - slub_merge [MM, SLUB] - Same with slab_merge. - - slub_nomerge [MM, SLUB] - Same with slab_nomerge. This is supported for legacy. - See slab_nomerge for more information. + slram= [HW,MTD] smart2= [HW] Format: [,[,...,]] diff --git a/drivers/misc/lkdtm/heap.c b/drivers/misc/lkdtm/heap.c index 4f467d3972a6..b1b316f99703 100644 --- a/drivers/misc/lkdtm/heap.c +++ b/drivers/misc/lkdtm/heap.c @@ -48,7 +48,7 @@ static void lkdtm_VMALLOC_LINEAR_OVERFLOW(void) * correctly. * * This should get caught by either memory tagging, KASan, or by using - * CONFIG_SLUB_DEBUG=y and slub_debug=ZF (or CONFIG_SLUB_DEBUG_ON=y). + * CONFIG_SLUB_DEBUG=y and slab_debug=ZF (or CONFIG_SLUB_DEBUG_ON=y). */ static void lkdtm_SLAB_LINEAR_OVERFLOW(void) { diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 321ab379994f..afc72fde0f03 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -64,11 +64,11 @@ config SLUB_DEBUG_ON help Boot with debugging on by default. SLUB boots by default with the runtime debug capabilities switched off. Enabling this is - equivalent to specifying the "slub_debug" parameter on boot. + equivalent to specifying the "slab_debug" parameter on boot. There is no support for more fine grained debug control like - possible with slub_debug=xxx. SLUB debugging may be switched + possible with slab_debug=xxx. SLUB debugging may be switched off in a kernel built with CONFIG_SLUB_DEBUG_ON by specifying - "slub_debug=-". + "slab_debug=-". config PAGE_OWNER bool "Track page owner" diff --git a/mm/slab.h b/mm/slab.h index 54deeb0428c6..f7df6d701c5b 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -528,7 +528,7 @@ static inline bool __slub_debug_enabled(void) #endif /* - * Returns true if any of the specified slub_debug flags is enabled for the + * Returns true if any of the specified slab_debug flags is enabled for the * cache. Use only for flags parsed by setup_slub_debug() as it also enables * the static key. */ diff --git a/mm/slab_common.c b/mm/slab_common.c index 238293b1dbe1..230ef7cc3467 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -282,7 +282,7 @@ kmem_cache_create_usercopy(const char *name, #ifdef CONFIG_SLUB_DEBUG /* - * If no slub_debug was enabled globally, the static key is not yet + * If no slab_debug was enabled globally, the static key is not yet * enabled by setup_slub_debug(). Enable it if the cache is being * created with any of the debugging flags passed explicitly. * It's also possible that this is the first cache created with @@ -766,7 +766,7 @@ EXPORT_SYMBOL(kmalloc_size_roundup); } /* - * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. + * kmalloc_info[] is to make slab_debug=,kmalloc-xx option work at boot time. * kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is * kmalloc-2M. */ diff --git a/mm/slub.c b/mm/slub.c index 2ef88bbf56a3..e66bc888d23b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -295,7 +295,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) /* * Debugging flags that require metadata to be stored in the slab. These get - * disabled when slub_debug=O is used and a cache's min order increases with + * disabled when slab_debug=O is used and a cache's min order increases with * metadata. */ #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) @@ -1616,7 +1616,7 @@ static inline int free_consistency_checks(struct kmem_cache *s, } /* - * Parse a block of slub_debug options. Blocks are delimited by ';' + * Parse a block of slab_debug options. Blocks are delimited by ';' * * @str: start of block * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified @@ -1677,7 +1677,7 @@ parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init) break; default: if (init) - pr_err("slub_debug option '%c' unknown. skipped\n", *str); + pr_err("slab_debug option '%c' unknown. skipped\n", *str); } } check_slabs: @@ -1736,7 +1736,7 @@ static int __init setup_slub_debug(char *str) /* * For backwards compatibility, a single list of flags with list of * slabs means debugging is only changed for those slabs, so the global - * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending + * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as * long as there is no option specifying flags without a slab list. */ @@ -1760,7 +1760,8 @@ out: return 1; } -__setup("slub_debug", setup_slub_debug); +__setup("slab_debug", setup_slub_debug); +__setup_param("slub_debug", slub_debug, setup_slub_debug, 0); /* * kmem_cache_flags - apply debugging options to the cache @@ -1770,7 +1771,7 @@ __setup("slub_debug", setup_slub_debug); * * Debug option(s) are applied to @flags. In addition to the debug * option(s), if a slab name (or multiple) is specified i.e. - * slub_debug=,, ... + * slab_debug=,, ... * then only the select slabs will receive the debug option(s). */ slab_flags_t kmem_cache_flags(unsigned int object_size, @@ -3263,7 +3264,7 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) oo_order(s->min)); if (oo_order(s->min) > get_order(s->object_size)) - pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", + pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n", s->name); for_each_kmem_cache_node(s, node, n) { @@ -3792,11 +3793,11 @@ void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, zero_size = orig_size; /* - * When slub_debug is enabled, avoid memory initialization integrated + * When slab_debug is enabled, avoid memory initialization integrated * into KASAN and instead zero out the memory via the memset below with * the proper size. Otherwise, KASAN might overwrite SLUB redzones and * cause false-positive reports. This does not lead to a performance - * penalty on production builds, as slub_debug is not intended to be + * penalty on production builds, as slab_debug is not intended to be * enabled there. */ if (__slub_debug_enabled()) @@ -4702,8 +4703,8 @@ static unsigned int slub_min_objects; * activity on the partial lists which requires taking the list_lock. This is * less a concern for large slabs though which are rarely used. * - * slub_max_order specifies the order where we begin to stop considering the - * number of objects in a slab as critical. If we reach slub_max_order then + * slab_max_order specifies the order where we begin to stop considering the + * number of objects in a slab as critical. If we reach slab_max_order then * we try to keep the page order as low as possible. So we accept more waste * of space in favor of a small page order. * @@ -4770,14 +4771,14 @@ static inline int calculate_order(unsigned int size) * and backing off gradually. * * We start with accepting at most 1/16 waste and try to find the - * smallest order from min_objects-derived/slub_min_order up to - * slub_max_order that will satisfy the constraint. Note that increasing + * smallest order from min_objects-derived/slab_min_order up to + * slab_max_order that will satisfy the constraint. Note that increasing * the order can only result in same or less fractional waste, not more. * * If that fails, we increase the acceptable fraction of waste and try * again. The last iteration with fraction of 1/2 would effectively * accept any waste and give us the order determined by min_objects, as - * long as at least single object fits within slub_max_order. + * long as at least single object fits within slab_max_order. */ for (unsigned int fraction = 16; fraction > 1; fraction /= 2) { order = calc_slab_order(size, min_order, slub_max_order, @@ -4787,7 +4788,7 @@ static inline int calculate_order(unsigned int size) } /* - * Doh this slab cannot be placed using slub_max_order. + * Doh this slab cannot be placed using slab_max_order. */ order = get_order(size); if (order <= MAX_PAGE_ORDER) @@ -5313,7 +5314,9 @@ static int __init setup_slub_min_order(char *str) return 1; } -__setup("slub_min_order=", setup_slub_min_order); +__setup("slab_min_order=", setup_slub_min_order); +__setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0); + static int __init setup_slub_max_order(char *str) { @@ -5326,7 +5329,8 @@ static int __init setup_slub_max_order(char *str) return 1; } -__setup("slub_max_order=", setup_slub_max_order); +__setup("slab_max_order=", setup_slub_max_order); +__setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0); static int __init setup_slub_min_objects(char *str) { @@ -5335,7 +5339,8 @@ static int __init setup_slub_min_objects(char *str) return 1; } -__setup("slub_min_objects=", setup_slub_min_objects); +__setup("slab_min_objects=", setup_slub_min_objects); +__setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0); #ifdef CONFIG_HARDENED_USERCOPY /* From cb109a9d607056915d6a0dc8e1ebfc5854c72d92 Mon Sep 17 00:00:00 2001 From: Xiongwei Song Date: Fri, 15 Dec 2023 11:41:49 +0800 Subject: [PATCH 0014/1038] mm/slub: replace slub_$params with slab_$params in slub.rst We've unified slab parameters with "slab_$params", then we can use slab_$params in Documentation/mm/slub.rst. Signed-off-by: Xiongwei Song Reviewed-by: Vlastimil Babka Signed-off-by: Vlastimil Babka --- Documentation/mm/slub.rst | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/Documentation/mm/slub.rst b/Documentation/mm/slub.rst index be75971532f5..6579a55b7852 100644 --- a/Documentation/mm/slub.rst +++ b/Documentation/mm/slub.rst @@ -9,7 +9,7 @@ SLUB can enable debugging only for selected slabs in order to avoid an impact on overall system performance which may make a bug more difficult to find. -In order to switch debugging on one can add an option ``slub_debug`` +In order to switch debugging on one can add an option ``slab_debug`` to the kernel command line. That will enable full debugging for all slabs. @@ -26,16 +26,16 @@ be enabled on the command line. F.e. no tracking information will be available without debugging on and validation can only partially be performed if debugging was not switched on. -Some more sophisticated uses of slub_debug: +Some more sophisticated uses of slab_debug: ------------------------------------------- -Parameters may be given to ``slub_debug``. If none is specified then full +Parameters may be given to ``slab_debug``. If none is specified then full debugging is enabled. Format: -slub_debug= +slab_debug= Enable options for all slabs -slub_debug=,,,... +slab_debug=,,,... Enable options only for select slabs (no spaces after a comma) @@ -60,23 +60,23 @@ Possible debug options are:: F.e. in order to boot just with sanity checks and red zoning one would specify:: - slub_debug=FZ + slab_debug=FZ Trying to find an issue in the dentry cache? Try:: - slub_debug=,dentry + slab_debug=,dentry to only enable debugging on the dentry cache. You may use an asterisk at the end of the slab name, in order to cover all slabs with the same prefix. For example, here's how you can poison the dentry cache as well as all kmalloc slabs:: - slub_debug=P,kmalloc-*,dentry + slab_debug=P,kmalloc-*,dentry Red zoning and tracking may realign the slab. We can just apply sanity checks to the dentry cache with:: - slub_debug=F,dentry + slab_debug=F,dentry Debugging options may require the minimum possible slab order to increase as a result of storing the metadata (for example, caches with PAGE_SIZE object @@ -84,20 +84,20 @@ sizes). This has a higher liklihood of resulting in slab allocation errors in low memory situations or if there's high fragmentation of memory. To switch off debugging for such caches by default, use:: - slub_debug=O + slab_debug=O You can apply different options to different list of slab names, using blocks of options. This will enable red zoning for dentry and user tracking for kmalloc. All other slabs will not get any debugging enabled:: - slub_debug=Z,dentry;U,kmalloc-* + slab_debug=Z,dentry;U,kmalloc-* You can also enable options (e.g. sanity checks and poisoning) for all caches except some that are deemed too performance critical and don't need to be debugged by specifying global debug options followed by a list of slab names with "-" as options:: - slub_debug=FZ;-,zs_handle,zspage + slab_debug=FZ;-,zs_handle,zspage The state of each debug option for a slab can be found in the respective files under:: @@ -105,7 +105,7 @@ under:: /sys/kernel/slab// If the file contains 1, the option is enabled, 0 means disabled. The debug -options from the ``slub_debug`` parameter translate to the following files:: +options from the ``slab_debug`` parameter translate to the following files:: F sanity_checks Z red_zone @@ -129,7 +129,7 @@ in order to reduce overhead and increase cache hotness of objects. Slab validation =============== -SLUB can validate all object if the kernel was booted with slub_debug. In +SLUB can validate all object if the kernel was booted with slab_debug. In order to do so you must have the ``slabinfo`` tool. Then you can do :: @@ -150,29 +150,29 @@ list_lock once in a while to deal with partial slabs. That overhead is governed by the order of the allocation for each slab. The allocations can be influenced by kernel parameters: -.. slub_min_objects=x (default 4) -.. slub_min_order=x (default 0) -.. slub_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) +.. slab_min_objects=x (default 4) +.. slab_min_order=x (default 0) +.. slab_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) -``slub_min_objects`` +``slab_min_objects`` allows to specify how many objects must at least fit into one slab in order for the allocation order to be acceptable. In general slub will be able to perform this number of allocations on a slab without consulting centralized resources (list_lock) where contention may occur. -``slub_min_order`` +``slab_min_order`` specifies a minimum order of slabs. A similar effect like - ``slub_min_objects``. + ``slab_min_objects``. -``slub_max_order`` - specified the order at which ``slub_min_objects`` should no +``slab_max_order`` + specified the order at which ``slab_min_objects`` should no longer be checked. This is useful to avoid SLUB trying to - generate super large order pages to fit ``slub_min_objects`` + generate super large order pages to fit ``slab_min_objects`` of a slab cache with large object sizes into one high order page. Setting command line parameter ``debug_guardpage_minorder=N`` (N > 0), forces setting - ``slub_max_order`` to 0, what cause minimum possible order of + ``slab_max_order`` to 0, what cause minimum possible order of slabs allocation. SLUB Debug output @@ -219,7 +219,7 @@ Here is a sample of slub debug output:: FIX kmalloc-8: Restoring Redzone 0xc90f6d28-0xc90f6d2b=0xcc If SLUB encounters a corrupted object (full detection requires the kernel -to be booted with slub_debug) then the following output will be dumped +to be booted with slab_debug) then the following output will be dumped into the syslog: 1. Description of the problem encountered @@ -239,7 +239,7 @@ into the syslog: pid= (Object allocation / free information is only available if SLAB_STORE_USER is - set for the slab. slub_debug sets that option) + set for the slab. slab_debug sets that option) 2. The object contents if an object was involved. @@ -262,7 +262,7 @@ into the syslog: the object boundary. (Redzone information is only available if SLAB_RED_ZONE is set. - slub_debug sets that option) + slab_debug sets that option) Padding
: Unused data to fill up the space in order to get the next object @@ -296,7 +296,7 @@ Emergency operations Minimal debugging (sanity checks alone) can be enabled by booting with:: - slub_debug=F + slab_debug=F This will be generally be enough to enable the resiliency features of slub which will keep the system running even if a bad kernel component will @@ -311,13 +311,13 @@ and enabling debugging only for that cache I.e.:: - slub_debug=F,dentry + slab_debug=F,dentry If the corruption occurs by writing after the end of the object then it may be advisable to enable a Redzone to avoid corrupting the beginning of other objects:: - slub_debug=FZ,dentry + slab_debug=FZ,dentry Extended slabinfo mode and plotting =================================== From 98d3b6d98f8013d4e96f40d8d4b22d4da0d3f699 Mon Sep 17 00:00:00 2001 From: Xiongwei Song Date: Fri, 15 Dec 2023 11:41:50 +0800 Subject: [PATCH 0015/1038] mm/slub: make the description of slab_min_objects helpful in doc There is no a value assigned to slab_min_objects by default, it always is 0 that is initialized by compiler if no assigned value by command line. min_objects is calculated based on processor numbers in calculate_order(). For more details, see commit 9b2cd506e5f2 ("slub: Calculate min_objects based on number of processors.") Signed-off-by: Xiongwei Song Reviewed-by: Vlastimil Babka Signed-off-by: Vlastimil Babka --- Documentation/mm/slub.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/mm/slub.rst b/Documentation/mm/slub.rst index 6579a55b7852..b517ee28a955 100644 --- a/Documentation/mm/slub.rst +++ b/Documentation/mm/slub.rst @@ -150,7 +150,7 @@ list_lock once in a while to deal with partial slabs. That overhead is governed by the order of the allocation for each slab. The allocations can be influenced by kernel parameters: -.. slab_min_objects=x (default 4) +.. slab_min_objects=x (default: automatically scaled by number of cpus) .. slab_min_order=x (default 0) .. slab_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) From 32a0a0da530eab78d755474270bf53274b93a1e6 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:39 +0800 Subject: [PATCH 0016/1038] Documentation: gpio: add chardev userspace API documentation Add documentation for the GPIO character device userspace API. Added to the userspace-api book, but also provide a link from the admin-guide book, as historically the GPIO documentation has been there. Signed-off-by: Kent Gibson Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- Documentation/admin-guide/gpio/index.rst | 1 + Documentation/userspace-api/gpio/chardev.rst | 116 ++++++++++++++++++ .../userspace-api/gpio/error-codes.rst | 78 ++++++++++++ .../gpio/gpio-get-chipinfo-ioctl.rst | 41 +++++++ .../gpio/gpio-get-lineinfo-unwatch-ioctl.rst | 49 ++++++++ .../gpio/gpio-v2-get-line-ioctl.rst | 101 +++++++++++++++ .../gpio/gpio-v2-get-lineinfo-ioctl.rst | 50 ++++++++ .../gpio/gpio-v2-get-lineinfo-watch-ioctl.rst | 67 ++++++++++ .../gpio/gpio-v2-line-event-read.rst | 83 +++++++++++++ .../gpio/gpio-v2-line-get-values-ioctl.rst | 51 ++++++++ .../gpio/gpio-v2-line-set-config-ioctl.rst | 57 +++++++++ .../gpio/gpio-v2-line-set-values-ioctl.rst | 47 +++++++ .../gpio/gpio-v2-lineinfo-changed-read.rst | 81 ++++++++++++ Documentation/userspace-api/gpio/index.rst | 17 +++ Documentation/userspace-api/index.rst | 1 + 15 files changed, 840 insertions(+) create mode 100644 Documentation/userspace-api/gpio/chardev.rst create mode 100644 Documentation/userspace-api/gpio/error-codes.rst create mode 100644 Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst create mode 100644 Documentation/userspace-api/gpio/index.rst diff --git a/Documentation/admin-guide/gpio/index.rst b/Documentation/admin-guide/gpio/index.rst index f6861ca16ffe..b40f0a2a6822 100644 --- a/Documentation/admin-guide/gpio/index.rst +++ b/Documentation/admin-guide/gpio/index.rst @@ -7,6 +7,7 @@ gpio .. toctree:: :maxdepth: 1 + Character Device Userspace API <../../userspace-api/gpio/chardev> gpio-aggregator sysfs gpio-mockup diff --git a/Documentation/userspace-api/gpio/chardev.rst b/Documentation/userspace-api/gpio/chardev.rst new file mode 100644 index 000000000000..c58dd9771ac9 --- /dev/null +++ b/Documentation/userspace-api/gpio/chardev.rst @@ -0,0 +1,116 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================== +GPIO Character Device Userspace API +=================================== + +This is latest version (v2) of the character device API, as defined in +``include/uapi/linux/gpio.h.`` + +First added in 5.10. + +.. note:: + Do NOT abuse userspace APIs to control hardware that has proper kernel + drivers. There may already be a driver for your use case, and an existing + kernel driver is sure to provide a superior solution to bitbashing + from userspace. + + Read Documentation/driver-api/gpio/drivers-on-gpio.rst to avoid reinventing + kernel wheels in userspace. + + Similarly, for multi-function lines there may be other subsystems, such as + Documentation/spi/index.rst, Documentation/i2c/index.rst, + Documentation/driver-api/pwm.rst, Documentation/w1/index.rst etc, that + provide suitable drivers and APIs for your hardware. + +Basic examples using the character device API can be found in ``tools/gpio/*``. + +The API is based around two major objects, the :ref:`gpio-v2-chip` and the +:ref:`gpio-v2-line-request`. + +.. _gpio-v2-chip: + +Chip +==== + +The Chip represents a single GPIO chip and is exposed to userspace using device +files of the form ``/dev/gpiochipX``. + +Each chip supports a number of GPIO lines, +:c:type:`chip.lines`. Lines on the chip are identified by an +``offset`` in the range from 0 to ``chip.lines - 1``, i.e. `[0,chip.lines)`. + +Lines are requested from the chip using gpio-v2-get-line-ioctl.rst +and the resulting line request is used to access the GPIO chip's lines or +monitor the lines for edge events. + +Within this documentation, the file descriptor returned by calling `open()` +on the GPIO device file is referred to as ``chip_fd``. + +Operations +---------- + +The following operations may be performed on the chip: + +.. toctree:: + :titlesonly: + + Get Line + Get Chip Info + Get Line Info + Watch Line Info + Unwatch Line Info + Read Line Info Changed Events + +.. _gpio-v2-line-request: + +Line Request +============ + +Line requests are created by gpio-v2-get-line-ioctl.rst and provide +access to a set of requested lines. The line request is exposed to userspace +via the anonymous file descriptor returned in +:c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +Within this documentation, the line request file descriptor is referred to +as ``req_fd``. + +Operations +---------- + +The following operations may be performed on the line request: + +.. toctree:: + :titlesonly: + + Get Line Values + Set Line Values + Read Line Edge Events + Reconfigure Lines + +Types +===== + +This section contains the structs and enums that are referenced by the API v2, +as defined in ``include/uapi/linux/gpio.h``. + +.. kernel-doc:: include/uapi/linux/gpio.h + :identifiers: + gpio_v2_line_attr_id + gpio_v2_line_attribute + gpio_v2_line_changed_type + gpio_v2_line_config + gpio_v2_line_config_attribute + gpio_v2_line_event + gpio_v2_line_event_id + gpio_v2_line_flag + gpio_v2_line_info + gpio_v2_line_info_changed + gpio_v2_line_request + gpio_v2_line_values + gpiochip_info + +.. toctree:: + :hidden: + + error-codes diff --git a/Documentation/userspace-api/gpio/error-codes.rst b/Documentation/userspace-api/gpio/error-codes.rst new file mode 100644 index 000000000000..edf01f2cf9d2 --- /dev/null +++ b/Documentation/userspace-api/gpio/error-codes.rst @@ -0,0 +1,78 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _gpio_errors: + +******************* +GPIO Error Codes +******************* + +.. _gpio-errors: + +.. tabularcolumns:: |p{2.5cm}|p{15.0cm}| + +.. flat-table:: Common GPIO error codes + :header-rows: 0 + :stub-columns: 0 + :widths: 1 16 + + - - ``EAGAIN`` (aka ``EWOULDBLOCK``) + + - The device was opened in non-blocking mode and a read can't + be performed as there is no data available. + + - - ``EBADF`` + + - The file descriptor is not valid. + + - - ``EBUSY`` + + - The ioctl can't be handled because the device is busy. Typically + returned when an ioctl attempts something that would require the + usage of a resource that was already allocated. The ioctl must not + be retried without performing another action to fix the problem + first. + + - - ``EFAULT`` + + - There was a failure while copying data from/to userspace, probably + caused by an invalid pointer reference. + + - - ``EINVAL`` + + - One or more of the ioctl parameters are invalid or out of the + allowed range. This is a widely used error code. + + - - ``ENODEV`` + + - Device not found or was removed. + + - - ``ENOMEM`` + + - There's not enough memory to handle the desired operation. + + - - ``EPERM`` + + - Permission denied. Typically returned in response to an attempt + to perform an action incompatible with the current line + configuration. + + - - ``EIO`` + + - I/O error. Typically returned when there are problems communicating + with a hardware device or requesting features that hardware does not + support. This could indicate broken or flaky hardware. + It's a 'Something is wrong, I give up!' type of error. + + - - ``ENXIO`` + + - No device corresponding to this device special file exists. + +.. note:: + + #. This list is not exhaustive; ioctls may return other error codes. + Since errors may have side effects such as a driver reset, + applications should abort on unexpected errors, or otherwise + assume that the device is in a bad state. + + #. Request-specific error codes are listed in the individual + requests descriptions. diff --git a/Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst new file mode 100644 index 000000000000..05f07fdefe2f --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst @@ -0,0 +1,41 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_CHIPINFO_IOCTL: + +*********************** +GPIO_GET_CHIPINFO_IOCTL +*********************** + +Name +==== + +GPIO_GET_CHIPINFO_IOCTL - Get the publicly available information for a chip. + +Synopsis +======== + +.. c:macro:: GPIO_GET_CHIPINFO_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_CHIPINFO_IOCTL, struct gpiochip_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`chip_info` to be populated. + +Description +=========== + +Gets the publicly available information for a particular GPIO chip. + +Return Value +============ + +On success 0 and ``info`` is populated with the chip info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst new file mode 100644 index 000000000000..2fec6f12e3bf --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst @@ -0,0 +1,49 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEINFO_UNWATCH_IOCTL: + +******************************* +GPIO_GET_LINEINFO_UNWATCH_IOCTL +******************************* + +Name +==== + +GPIO_GET_LINEINFO_UNWATCH_IOCTL - Disable watching a line for changes to its +requested state and configuration information. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEINFO_UNWATCH_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEINFO_UNWATCH_IOCTL, u32 *offset)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``offset`` + The offset of the line to no longer watch. + +Description +=========== + +Remove the line from the list of lines being watched on this ``chip_fd``. + +This is the opposite of gpio-v2-get-lineinfo-watch-ioctl.rst (v2) and +gpio-get-lineinfo-watch-ioctl.rst (v1). + +Unwatching a line that is not watched is an error (**EBUSY**). + +First added in 5.7. + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst new file mode 100644 index 000000000000..d76e614c8343 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst @@ -0,0 +1,101 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_GET_LINE_IOCTL: + +********************** +GPIO_V2_GET_LINE_IOCTL +********************** + +Name +==== + +GPIO_V2_GET_LINE_IOCTL - Request a line or lines from the kernel. + +Synopsis +======== + +.. c:macro:: GPIO_V2_GET_LINE_IOCTL + +``int ioctl(int chip_fd, GPIO_V2_GET_LINE_IOCTL, struct gpio_v2_line_request *request)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``request`` + The :c:type:`line_request` specifying the lines + to request and their configuration. + +Description +=========== + +On success, the requesting process is granted exclusive access to the line +value, write access to the line configuration, and may receive events when +edges are detected on the line, all of which are described in more detail in +:ref:`gpio-v2-line-request`. + +A number of lines may be requested in the one line request, and request +operations are performed on the requested lines by the kernel as atomically +as possible. e.g. gpio-v2-line-get-values-ioctl.rst will read all the +requested lines at once. + +The state of a line, including the value of output lines, is guaranteed to +remain as requested until the returned file descriptor is closed. Once the +file descriptor is closed, the state of the line becomes uncontrolled from +the userspace perspective, and may revert to its default state. + +Requesting a line already in use is an error (**EBUSY**). + +Closing the ``chip_fd`` has no effect on existing line requests. + +.. _gpio-v2-get-line-config-rules: + +Configuration Rules +------------------- + +For any given requested line, the following configuration rules apply: + +The direction flags, ``GPIO_V2_LINE_FLAG_INPUT`` and +``GPIO_V2_LINE_FLAG_OUTPUT``, cannot be combined. If neither are set then +the only other flag that may be set is ``GPIO_V2_LINE_FLAG_ACTIVE_LOW`` +and the line is requested "as-is" to allow reading of the line value +without altering the electrical configuration. + +The drive flags, ``GPIO_V2_LINE_FLAG_OPEN_xxx``, require the +``GPIO_V2_LINE_FLAG_OUTPUT`` to be set. +Only one drive flag may be set. +If none are set then the line is assumed push-pull. + +Only one bias flag, ``GPIO_V2_LINE_FLAG_BIAS_xxx``, may be set, and it +requires a direction flag to also be set. +If no bias flags are set then the bias configuration is not changed. + +The edge flags, ``GPIO_V2_LINE_FLAG_EDGE_xxx``, require +``GPIO_V2_LINE_FLAG_INPUT`` to be set and may be combined to detect both rising +and falling edges. + +Only one event clock flag, ``GPIO_V2_LINE_FLAG_EVENT_CLOCK_xxx``, may be set. +If none are set then the event clock defaults to ``CLOCK_MONOTONIC``. +The ``GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE`` flag requires supporting hardware +and a kernel with ``CONFIG_HTE`` set. Requesting HTE from a device that +doesn't support it is an error (**EOPNOTSUP**). + +The :c:type:`debounce_period_us` attribute may only +be applied to lines with ``GPIO_V2_LINE_FLAG_INPUT`` set. When set, debounce +applies to both the values returned by gpio-v2-line-get-values-ioctl.rst and +the edges returned by gpio-v2-line-event-read.rst. If not +supported directly by hardware, the debouncing is performed in software by the +kernel. + +Requesting an invalid configuration is an error (**EINVAL**). + +Return Value +============ + +On success 0 and the :c:type:`request.fd` contains the +file descriptor for the request. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst new file mode 100644 index 000000000000..bc4d8df887d4 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst @@ -0,0 +1,50 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_GET_LINEINFO_IOCTL: + +************************** +GPIO_V2_GET_LINEINFO_IOCTL +************************** + +Name +==== + +GPIO_V2_GET_LINEINFO_IOCTL - Get the publicly available information for a line. + +Synopsis +======== + +.. c:macro:: GPIO_V2_GET_LINEINFO_IOCTL + +``int ioctl(int chip_fd, GPIO_V2_GET_LINEINFO_IOCTL, struct gpio_v2_line_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` to be populated, with the + ``offset`` field set to indicate the line to be collected. + +Description +=========== + +Get the publicly available information for a line. + +This information is available independent of whether the line is in use. + +.. note:: + The line info does not include the line value. + + The line must be requested using gpio-v2-get-line-ioctl.rst to access its + value. + +Return Value +============ + +On success 0 and ``info`` is populated with the chip info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst new file mode 100644 index 000000000000..938ff85a9322 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_GET_LINEINFO_WATCH_IOCTL: + +******************************** +GPIO_V2_GET_LINEINFO_WATCH_IOCTL +******************************** + +Name +==== + +GPIO_V2_GET_LINEINFO_WATCH_IOCTL - Enable watching a line for changes to its +request state and configuration information. + +Synopsis +======== + +.. c:macro:: GPIO_V2_GET_LINEINFO_WATCH_IOCTL + +``int ioctl(int chip_fd, GPIO_V2_GET_LINEINFO_WATCH_IOCTL, struct gpio_v2_line_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` struct to be populated, with + the ``offset`` set to indicate the line to watch + +Description +=========== + +Enable watching a line for changes to its request state and configuration +information. Changes to line info include a line being requested, released +or reconfigured. + +.. note:: + Watching line info is not generally required, and would typically only be + used by a system monitoring component. + + The line info does NOT include the line value. + The line must be requested using gpio-v2-get-line-ioctl.rst to access + its value, and the line request can monitor a line for events using + gpio-v2-line-event-read.rst. + +By default all lines are unwatched when the GPIO chip is opened. + +Multiple lines may be watched simultaneously by adding a watch for each. + +Once a watch is set, any changes to line info will generate events which can be +read from the ``chip_fd`` as described in +gpio-v2-lineinfo-changed-read.rst. + +Adding a watch to a line that is already watched is an error (**EBUSY**). + +Watches are specific to the ``chip_fd`` and are independent of watches +on the same GPIO chip opened with a separate call to `open()`. + +Return Value +============ + +On success 0 and ``info`` is populated with the current line info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst b/Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst new file mode 100644 index 000000000000..6513c23fb7ca --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst @@ -0,0 +1,83 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_EVENT_READ: + +*********************** +GPIO_V2_LINE_EVENT_READ +*********************** + +Name +==== + +GPIO_V2_LINE_EVENT_READ - Read edge detection events for lines from a request. + +Synopsis +======== + +``int read(int req_fd, void *buf, size_t count)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at + least the size of a :c:type:`gpio_v2_line_event`. + +Description +=========== + +Read edge detection events for lines from a request. + +Edge detection must be enabled for the input line using either +``GPIO_V2_LINE_FLAG_EDGE_RISING`` or ``GPIO_V2_LINE_FLAG_EDGE_FALLING``, or +both. Edge events are then generated whenever edge interrupts are detected on +the input line. + +The kernel captures and timestamps edge events as close as possible to their +occurrence and stores them in a buffer from where they can be read by +userspace at its convenience using `read()`. + +Events read from the buffer are always in the same order that they were +detected by the kernel, including when multiple lines are being monitored by +the one request. + +The size of the kernel event buffer is fixed at the time of line request +creation, and can be influenced by the +:c:type:`request.event_buffer_size`. +The default size is 16 times the number of lines requested. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the oldest buffered event is +discarded. Overflow can be detected from userspace by monitoring the event +sequence numbers. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +Changing the edge detection flags using gpio-v2-line-set-config-ioctl.rst +does not remove or modify the events already contained in the kernel event +buffer. + +The `read()` will block if no event is available and the ``req_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``req_fd`` is +readable using `poll()` or an equivalent. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size of a +:c:type:`gpio_v2_line_event` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst new file mode 100644 index 000000000000..e4e74a1926d8 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst @@ -0,0 +1,51 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_GET_VALUES_IOCTL: + +***************************** +GPIO_V2_LINE_GET_VALUES_IOCTL +***************************** + +Name +==== + +GPIO_V2_LINE_GET_VALUES_IOCTL - Get the values of requested lines. + +Synopsis +======== + +.. c:macro:: GPIO_V2_LINE_GET_VALUES_IOCTL + +``int ioctl(int req_fd, GPIO_V2_LINE_GET_VALUES_IOCTL, struct gpio_v2_line_values *values)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``values`` + The :c:type:`line_values` to get with the ``mask`` set + to indicate the subset of requested lines to get. + +Description +=========== + +Get the values of requested lines. + +The values of both input and output lines may be read. + +For output lines, the value returned is driver and configuration dependent and +may be either the output buffer (the last requested value set) or the input +buffer (the actual level of the line), and depending on the hardware and +configuration these may differ. + +Return Value +============ + +On success 0 and the corresponding :c:type:`values.bits` +contain the value read. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst new file mode 100644 index 000000000000..126c2626ba6b --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst @@ -0,0 +1,57 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_SET_CONFIG_IOCTL: + +***************************** +GPIO_V2_LINE_SET_CONFIG_IOCTL +***************************** + +Name +==== + +GPIO_V2_LINE_SET_CONFIG_IOCTL - Update the configuration of previously requested lines. + +Synopsis +======== + +.. c:macro:: GPIO_V2_LINE_SET_CONFIG_IOCTL + +``int ioctl(int req_fd, GPIO_V2_LINE_SET_CONFIG_IOCTL, struct gpio_v2_line_config *config)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``config`` + The new :c:type:`configuration` to apply to the + requested lines. + +Description +=========== + +Update the configuration of previously requested lines, without releasing the +line or introducing potential glitches. + +The new configuration must specify the configuration of all requested lines. + +The same :ref:`gpio-v2-get-line-config-rules` that apply when requesting the lines +also apply when updating the line configuration. + +The motivating use case for this command is changing direction of +bi-directional lines between input and output, but it may also be used to +dynamically control edge detection, or more generally move lines seamlessly +from one configuration state to another. + +To only change the value of output lines, use +gpio-v2-line-set-values-ioctl.rst. + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst new file mode 100644 index 000000000000..6d2d1886950b --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst @@ -0,0 +1,47 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_SET_VALUES_IOCTL: + +***************************** +GPIO_V2_LINE_SET_VALUES_IOCTL +***************************** + +Name +==== + +GPIO_V2_LINE_SET_VALUES_IOCTL - Set the values of requested output lines. + +Synopsis +======== + +.. c:macro:: GPIO_V2_LINE_SET_VALUES_IOCTL + +``int ioctl(int req_fd, GPIO_V2_LINE_SET_VALUES_IOCTL, struct gpio_v2_line_values *values)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``values`` + The :c:type:`line_values` to set with the ``mask`` set + to indicate the subset of requested lines to set and ``bits`` set to + indicate the new value. + +Description +=========== + +Set the values of requested output lines. + +Only the values of output lines may be set. +Attempting to set the value of an input line is an error (**EPERM**). + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst b/Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst new file mode 100644 index 000000000000..24ad325e7253 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst @@ -0,0 +1,81 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINEINFO_CHANGED_READ: + +***************************** +GPIO_V2_LINEINFO_CHANGED_READ +***************************** + +Name +==== + +GPIO_V2_LINEINFO_CHANGED_READ - Read line info changed events for watched +lines from the chip. + +Synopsis +======== + +``int read(int chip_fd, void *buf, size_t count)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at least the size + of a :c:type:`gpio_v2_line_info_changed` event. + +Description +=========== + +Read line info changed events for watched lines from the chip. + +.. note:: + Monitoring line info changes is not generally required, and would typically + only be performed by a system monitoring component. + + These events relate to changes in a line's request state or configuration, + not its value. Use gpio-v2-line-event-read.rst to receive events when a + line changes value. + +A line must be watched using gpio-v2-get-lineinfo-watch-ioctl.rst to generate +info changed events. Subsequently, a request, release, or reconfiguration +of the line will generate an info changed event. + +The kernel timestamps events when they occur and stores them in a buffer +from where they can be read by userspace at its convenience using `read()`. + +The size of the kernel event buffer is fixed at 32 events per ``chip_fd``. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the most recent event is discarded. +Overflow cannot be detected from userspace. + +Events read from the buffer are always in the same order that they were +detected by the kernel, including when multiple lines are being monitored by +the one ``chip_fd``. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +A `read()` will block if no event is available and the ``chip_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``chip_fd`` is +readable using `poll()` or an equivalent. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size +of a :c:type:`gpio_v2_line_info_changed` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/index.rst b/Documentation/userspace-api/gpio/index.rst new file mode 100644 index 000000000000..072b9fa18aea --- /dev/null +++ b/Documentation/userspace-api/gpio/index.rst @@ -0,0 +1,17 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==== +GPIO +==== + +.. toctree:: + :maxdepth: 1 + + Character Device Userspace API + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index 09f61bd2ac2e..ee58d92c53c7 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -22,6 +22,7 @@ place where this information is gathered. dma-buf-alloc-exchange ebpf/index ELF + gpio/index ioctl/index iommu iommufd From c27cdd7a3010b4e0b9248c86c120335585afd085 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:40 +0800 Subject: [PATCH 0017/1038] Documentation: ABI: update gpio-cdev to reference chardev.rst Update the gpio-cdev interface document to refer to the new chardev.rst. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/ABI/testing/gpio-cdev | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/gpio-cdev b/Documentation/ABI/testing/gpio-cdev index 66bdcd188b6c..c9689b2a6fed 100644 --- a/Documentation/ABI/testing/gpio-cdev +++ b/Documentation/ABI/testing/gpio-cdev @@ -6,8 +6,9 @@ Description: The character device files /dev/gpiochip* are the interface between GPIO chips and userspace. - The ioctl(2)-based ABI is defined and documented in - [include/uapi]. + The ioctl(2)-based ABI is defined in + [include/uapi] and documented in + Documentation/userspace-api/gpio/chardev.rst. The following file operations are supported: @@ -17,8 +18,8 @@ Description: ioctl(2) Initiate various actions. - See the inline documentation in [include/uapi] - for descriptions of all ioctls. + See Documentation/userspace-api/gpio/chardev.rst + for a description of all ioctls. close(2) Stops and free up the I/O contexts that was associated From f1fc93d9e551ccf9e75d19ec53bfbdbc0b10da52 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:41 +0800 Subject: [PATCH 0018/1038] Documentation: ABI: update sysfs-gpio to reference gpio-cdev Update the sysfs-gpio interface document to refer to the gpio-cdev interface that obsoletes it. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/ABI/obsolete/sysfs-gpio | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/obsolete/sysfs-gpio b/Documentation/ABI/obsolete/sysfs-gpio index b8b0fd341c17..da1345d854b4 100644 --- a/Documentation/ABI/obsolete/sysfs-gpio +++ b/Documentation/ABI/obsolete/sysfs-gpio @@ -28,5 +28,5 @@ Description: /label ... (r/o) descriptive, not necessarily unique /ngpio ... (r/o) number of GPIOs; numbered N to N + (ngpio - 1) - This ABI is deprecated and will be removed after 2020. It is - replaced with the GPIO character device. + This ABI is obsoleted by Documentation/ABI/testing/gpio-cdev and will be + removed after 2020. From 5054626a9b801ff17aafa298d2d8f6a23ca84298 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:42 +0800 Subject: [PATCH 0019/1038] Documentation: gpio: move sysfs into an obsolete section The GPIO sysfs API is long obsolete, so highlight this even further by moving it into an obsolete APIs section in both the admin-guide and userspace-api books. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/admin-guide/gpio/index.rst | 2 +- Documentation/admin-guide/gpio/obsolete.rst | 11 +++++++++++ Documentation/userspace-api/gpio/index.rst | 1 + Documentation/userspace-api/gpio/obsolete.rst | 10 ++++++++++ .../{admin-guide => userspace-api}/gpio/sysfs.rst | 0 5 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Documentation/admin-guide/gpio/obsolete.rst create mode 100644 Documentation/userspace-api/gpio/obsolete.rst rename Documentation/{admin-guide => userspace-api}/gpio/sysfs.rst (100%) diff --git a/Documentation/admin-guide/gpio/index.rst b/Documentation/admin-guide/gpio/index.rst index b40f0a2a6822..3ac3aa15fa8b 100644 --- a/Documentation/admin-guide/gpio/index.rst +++ b/Documentation/admin-guide/gpio/index.rst @@ -9,9 +9,9 @@ gpio Character Device Userspace API <../../userspace-api/gpio/chardev> gpio-aggregator - sysfs gpio-mockup gpio-sim + Obsolete APIs .. only:: subproject and html diff --git a/Documentation/admin-guide/gpio/obsolete.rst b/Documentation/admin-guide/gpio/obsolete.rst new file mode 100644 index 000000000000..dbf88f13c680 --- /dev/null +++ b/Documentation/admin-guide/gpio/obsolete.rst @@ -0,0 +1,11 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================== +Obsolete GPIO APIs +================== + +.. toctree:: + :maxdepth: 1 + + Sysfs Interface <../../userspace-api/gpio/sysfs> + diff --git a/Documentation/userspace-api/gpio/index.rst b/Documentation/userspace-api/gpio/index.rst index 072b9fa18aea..f258de4ef370 100644 --- a/Documentation/userspace-api/gpio/index.rst +++ b/Documentation/userspace-api/gpio/index.rst @@ -8,6 +8,7 @@ GPIO :maxdepth: 1 Character Device Userspace API + Obsolete Userspace APIs .. only:: subproject and html diff --git a/Documentation/userspace-api/gpio/obsolete.rst b/Documentation/userspace-api/gpio/obsolete.rst new file mode 100644 index 000000000000..c27bd6014a3d --- /dev/null +++ b/Documentation/userspace-api/gpio/obsolete.rst @@ -0,0 +1,10 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================ +Obsolete GPIO Userspace APIs +============================ + +.. toctree:: + :maxdepth: 1 + + Sysfs Interface diff --git a/Documentation/admin-guide/gpio/sysfs.rst b/Documentation/userspace-api/gpio/sysfs.rst similarity index 100% rename from Documentation/admin-guide/gpio/sysfs.rst rename to Documentation/userspace-api/gpio/sysfs.rst From e10b641753837e267462b64f8cb395ab1e625027 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:43 +0800 Subject: [PATCH 0020/1038] Documentation: gpio: update sysfs documentation to reference new chardev doc Update GPIO sysfs interface documentation to reference the new chardev document rather than gpio.h. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/userspace-api/gpio/sysfs.rst | 27 ++++++++++++---------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Documentation/userspace-api/gpio/sysfs.rst b/Documentation/userspace-api/gpio/sysfs.rst index 35171d15f78d..e12037a0f2b4 100644 --- a/Documentation/userspace-api/gpio/sysfs.rst +++ b/Documentation/userspace-api/gpio/sysfs.rst @@ -2,18 +2,18 @@ GPIO Sysfs Interface for Userspace ================================== .. warning:: + This API is obsoleted by the chardev.rst and the ABI documentation has + been moved to Documentation/ABI/obsolete/sysfs-gpio. - THIS ABI IS DEPRECATED, THE ABI DOCUMENTATION HAS BEEN MOVED TO - Documentation/ABI/obsolete/sysfs-gpio AND NEW USERSPACE CONSUMERS - ARE SUPPOSED TO USE THE CHARACTER DEVICE ABI. THIS OLD SYSFS ABI WILL - NOT BE DEVELOPED (NO NEW FEATURES), IT WILL JUST BE MAINTAINED. + New developments should use the chardev.rst, and existing developments are + encouraged to migrate as soon as possible, as this API will be removed + in the future. -Refer to the examples in tools/gpio/* for an introduction to the new -character device ABI. Also see the userspace header in -include/uapi/linux/gpio.h + This interface will continue to be maintained for the migration period, + but new features will only be added to the new API. -The deprecated sysfs ABI ------------------------- +The obsolete sysfs ABI +---------------------- Platforms which use the "gpiolib" implementors framework may choose to configure a sysfs user interface to GPIOs. This is different from the debugfs interface, since it provides control over GPIO direction and @@ -33,9 +33,12 @@ userspace GPIO can be used to determine system configuration data that standard kernels won't know about. And for some tasks, simple userspace GPIO drivers could be all that the system really needs. -DO NOT ABUSE SYSFS TO CONTROL HARDWARE THAT HAS PROPER KERNEL DRIVERS. -PLEASE READ THE DOCUMENT AT Documentation/driver-api/gpio/drivers-on-gpio.rst -TO AVOID REINVENTING KERNEL WHEELS IN USERSPACE. I MEAN IT. REALLY. +.. note:: + Do NOT abuse sysfs to control hardware that has proper kernel drivers. + Please read Documentation/driver-api/gpio/drivers-on-gpio.rst + to avoid reinventing kernel wheels in userspace. + + I MEAN IT. REALLY. Paths in Sysfs -------------- From 16b2bb7fe5e4a0757608e844902fdf05617f51f8 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:44 +0800 Subject: [PATCH 0021/1038] Documentation: gpio: add chardev v1 userspace API documentation Add documentation for v1 of the GPIO character device userspace API to the obsolete section of both the admin-guide and userspace-api books. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/admin-guide/gpio/obsolete.rst | 1 + .../userspace-api/gpio/chardev_v1.rst | 131 ++++++++++++++++++ .../gpio/gpio-get-lineevent-ioctl.rst | 78 +++++++++++ .../gpio/gpio-get-linehandle-ioctl.rst | 86 ++++++++++++ .../gpio/gpio-get-lineinfo-ioctl.rst | 54 ++++++++ .../gpio/gpio-get-lineinfo-unwatch-ioctl.rst | 2 +- .../gpio/gpio-get-lineinfo-watch-ioctl.rst | 74 ++++++++++ .../gpio-handle-get-line-values-ioctl.rst | 56 ++++++++ .../gpio/gpio-handle-set-config-ioctl.rst | 62 +++++++++ .../gpio-handle-set-line-values-ioctl.rst | 48 +++++++ .../gpio/gpio-lineevent-data-read.rst | 84 +++++++++++ .../gpio/gpio-lineinfo-changed-read.rst | 87 ++++++++++++ Documentation/userspace-api/gpio/obsolete.rst | 1 + 13 files changed, 763 insertions(+), 1 deletion(-) create mode 100644 Documentation/userspace-api/gpio/chardev_v1.rst create mode 100644 Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst create mode 100644 Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst create mode 100644 Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst diff --git a/Documentation/admin-guide/gpio/obsolete.rst b/Documentation/admin-guide/gpio/obsolete.rst index dbf88f13c680..5926e5440207 100644 --- a/Documentation/admin-guide/gpio/obsolete.rst +++ b/Documentation/admin-guide/gpio/obsolete.rst @@ -7,5 +7,6 @@ Obsolete GPIO APIs .. toctree:: :maxdepth: 1 + Character Device Userspace API (v1) <../../userspace-api/gpio/chardev_v1> Sysfs Interface <../../userspace-api/gpio/sysfs> diff --git a/Documentation/userspace-api/gpio/chardev_v1.rst b/Documentation/userspace-api/gpio/chardev_v1.rst new file mode 100644 index 000000000000..67124b1d0487 --- /dev/null +++ b/Documentation/userspace-api/gpio/chardev_v1.rst @@ -0,0 +1,131 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================================== +GPIO Character Device Userspace API (v1) +======================================== + +.. warning:: + This API is obsoleted by chardev.rst (v2). + + New developments should use the v2 API, and existing developments are + encouraged to migrate as soon as possible, as this API will be removed + in the future. The v2 API is a functional superset of the v1 API so any + v1 call can be directly translated to a v2 equivalent. + + This interface will continue to be maintained for the migration period, + but new features will only be added to the new API. + +First added in 4.8. + +The API is based around three major objects, the :ref:`gpio-v1-chip`, the +:ref:`gpio-v1-line-handle`, and the :ref:`gpio-v1-line-event`. + +Where "line event" is used in this document it refers to the request that can +monitor a line for edge events, not the edge events themselves. + +.. _gpio-v1-chip: + +Chip +==== + +The Chip represents a single GPIO chip and is exposed to userspace using device +files of the form ``/dev/gpiochipX``. + +Each chip supports a number of GPIO lines, +:c:type:`chip.lines`. Lines on the chip are identified by an +``offset`` in the range from 0 to ``chip.lines - 1``, i.e. `[0,chip.lines)`. + +Lines are requested from the chip using either gpio-get-linehandle-ioctl.rst +and the resulting line handle is used to access the GPIO chip's lines, or +gpio-get-lineevent-ioctl.rst and the resulting line event is used to monitor +a GPIO line for edge events. + +Within this documentation, the file descriptor returned by calling `open()` +on the GPIO device file is referred to as ``chip_fd``. + +Operations +---------- + +The following operations may be performed on the chip: + +.. toctree:: + :titlesonly: + + Get Line Handle + Get Line Event + Get Chip Info + Get Line Info + Watch Line Info + Unwatch Line Info + Read Line Info Changed Events + +.. _gpio-v1-line-handle: + +Line Handle +=========== + +Line handles are created by gpio-get-linehandle-ioctl.rst and provide +access to a set of requested lines. The line handle is exposed to userspace +via the anonymous file descriptor returned in +:c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +Within this documentation, the line handle file descriptor is referred to +as ``handle_fd``. + +Operations +---------- + +The following operations may be performed on the line handle: + +.. toctree:: + :titlesonly: + + Get Line Values + Set Line Values + Reconfigure Lines + +.. _gpio-v1-line-event: + +Line Event +========== + +Line events are created by gpio-get-lineevent-ioctl.rst and provide +access to a requested line. The line event is exposed to userspace +via the anonymous file descriptor returned in +:c:type:`request.fd` by gpio-get-lineevent-ioctl.rst. + +Within this documentation, the line event file descriptor is referred to +as ``event_fd``. + +Operations +---------- + +The following operations may be performed on the line event: + +.. toctree:: + :titlesonly: + + Get Line Value + Read Line Edge Events + +Types +===== + +This section contains the structs that are referenced by the ABI v1. + +The :c:type:`struct gpiochip_info` is common to ABI v1 and v2. + +.. kernel-doc:: include/uapi/linux/gpio.h + :identifiers: + gpioevent_data + gpioevent_request + gpiohandle_config + gpiohandle_data + gpiohandle_request + gpioline_info + gpioline_info_changed + +.. toctree:: + :hidden: + + error-codes diff --git a/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst new file mode 100644 index 000000000000..7d0b932925c6 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst @@ -0,0 +1,78 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEEVENT_IOCTL: + +************************ +GPIO_GET_LINEEVENT_IOCTL +************************ + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-line-ioctl.rst. + +Name +==== + +GPIO_GET_LINEEVENT_IOCTL - Request a line with edge detection from the kernel. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEEVENT_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEEVENT_IOCTL, struct gpioevent_request *request)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``request`` + The :c:type:`event_request` specifying the line + to request and its configuration. + +Description +=========== + +Request a line with edge detection from the kernel. + +On success, the requesting process is granted exclusive access to the line +value and may receive events when edges are detected on the line, as +described in gpio-lineevent-data-read.rst. + +The state of a line is guaranteed to remain as requested until the returned +file descriptor is closed. Once the file descriptor is closed, the state of +the line becomes uncontrolled from the userspace perspective, and may revert +to its default state. + +Requesting a line already in use is an error (**EBUSY**). + +Closing the ``chip_fd`` has no effect on existing line events. + +Configuration Rules +------------------- + +The following configuration rules apply: + +The line event is requested as an input, so no flags specific to output lines, +``GPIOHANDLE_REQUEST_OUTPUT``, ``GPIOHANDLE_REQUEST_OPEN_DRAIN``, or +``GPIOHANDLE_REQUEST_OPEN_SOURCE``, may be set. + +Only one bias flag, ``GPIOHANDLE_REQUEST_BIAS_xxx``, may be set. +If no bias flags are set then the bias configuration is not changed. + +The edge flags, ``GPIOEVENT_REQUEST_RISING_EDGE`` and +``GPIOEVENT_REQUEST_FALLING_EDGE``, may be combined to detect both rising +and falling edges. + +Requesting an invalid configuration is an error (**EINVAL**). + +Return Value +============ + +On success 0 and the :c:type:`request.fd` contains the file +descriptor for the request. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst new file mode 100644 index 000000000000..c8256afe306e --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst @@ -0,0 +1,86 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEHANDLE_IOCTL: + +************************* +GPIO_GET_LINEHANDLE_IOCTL +************************* + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-line-ioctl.rst. + +Name +==== + +GPIO_GET_LINEHANDLE_IOCTL - Request a line or lines from the kernel. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEHANDLE_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEHANDLE_IOCTL, struct gpiohandle_request *request)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``request`` + The :c:type:`handle_request` specifying the lines to + request and their configuration. + +Description +=========== + +Request a line or lines from the kernel. + +While multiple lines may be requested, the same configuration applies to all +lines in the request. + +On success, the requesting process is granted exclusive access to the line +value and write access to the line configuration. + +The state of a line, including the value of output lines, is guaranteed to +remain as requested until the returned file descriptor is closed. Once the +file descriptor is closed, the state of the line becomes uncontrolled from +the userspace perspective, and may revert to its default state. + +Requesting a line already in use is an error (**EBUSY**). + +Closing the ``chip_fd`` has no effect on existing line handles. + +.. _gpio-get-linehandle-config-rules: + +Configuration Rules +------------------- + +The following configuration rules apply: + +The direction flags, ``GPIOHANDLE_REQUEST_INPUT`` and +``GPIOHANDLE_REQUEST_OUTPUT``, cannot be combined. If neither are set then the +only other flag that may be set is ``GPIOHANDLE_REQUEST_ACTIVE_LOW`` and the +line is requested "as-is" to allow reading of the line value without altering +the electrical configuration. + +The drive flags, ``GPIOHANDLE_REQUEST_OPEN_xxx``, require the +``GPIOHANDLE_REQUEST_OUTPUT`` to be set. +Only one drive flag may be set. +If none are set then the line is assumed push-pull. + +Only one bias flag, ``GPIOHANDLE_REQUEST_BIAS_xxx``, may be set, and +it requires a direction flag to also be set. +If no bias flags are set then the bias configuration is not changed. + +Requesting an invalid configuration is an error (**EINVAL**). + +Return Value +============ + +On success 0 and the :c:type:`request.fd` contains the +file descriptor for the request. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst new file mode 100644 index 000000000000..c895b8910b25 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst @@ -0,0 +1,54 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEINFO_IOCTL: + +*********************** +GPIO_GET_LINEINFO_IOCTL +*********************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-lineinfo-ioctl.rst. + +Name +==== + +GPIO_GET_LINEINFO_IOCTL - Get the publicly available information for a line. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEINFO_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEINFO_IOCTL, struct gpioline_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` to be populated, with the + ``offset`` field set to indicate the line to be collected. + +Description +=========== + +Get the publicly available information for a line. + +This information is available independent of whether the line is in use. + +.. note:: + The line info does not include the line value. + + The line must be requested using gpio-get-linehandle-ioctl.rst or + gpio-get-lineevent-ioctl.rst to access its value. + +Return Value +============ + +On success 0 and ``info`` is populated with the chip info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst index 2fec6f12e3bf..a82d0161daf8 100644 --- a/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst @@ -33,7 +33,7 @@ Description Remove the line from the list of lines being watched on this ``chip_fd``. -This is the opposite of gpio-v2-get-lineinfo-watch-ioctl.rst (v2) and +This is the reverse of gpio-v2-get-lineinfo-watch-ioctl.rst (v2) and gpio-get-lineinfo-watch-ioctl.rst (v1). Unwatching a line that is not watched is an error (**EBUSY**). diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst new file mode 100644 index 000000000000..f5c92b69a496 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEINFO_WATCH_IOCTL: + +***************************** +GPIO_GET_LINEINFO_WATCH_IOCTL +***************************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-lineinfo-watch-ioctl.rst. + +Name +==== + +GPIO_GET_LINEINFO_WATCH_IOCTL - Enable watching a line for changes to its +request state and configuration information. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEINFO_WATCH_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEINFO_WATCH_IOCTL, struct gpioline_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` struct to be populated, with + the ``offset`` set to indicate the line to watch + +Description +=========== + +Enable watching a line for changes to its request state and configuration +information. Changes to line info include a line being requested, released +or reconfigured. + +.. note:: + Watching line info is not generally required, and would typically only be + used by a system monitoring component. + + The line info does NOT include the line value. + + The line must be requested using gpio-get-linehandle-ioctl.rst or + gpio-get-lineevent-ioctl.rst to access its value, and the line event can + monitor a line for events using gpio-lineevent-data-read.rst. + +By default all lines are unwatched when the GPIO chip is opened. + +Multiple lines may be watched simultaneously by adding a watch for each. + +Once a watch is set, any changes to line info will generate events which can be +read from the ``chip_fd`` as described in +gpio-lineinfo-changed-read.rst. + +Adding a watch to a line that is already watched is an error (**EBUSY**). + +Watches are specific to the ``chip_fd`` and are independent of watches +on the same GPIO chip opened with a separate call to `open()`. + +First added in 5.7. + +Return Value +============ + +On success 0 and ``info`` is populated with the current line info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst new file mode 100644 index 000000000000..25263b8f0588 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIOHANDLE_GET_LINE_VALUES_IOCTL: + +******************************** +GPIOHANDLE_GET_LINE_VALUES_IOCTL +******************************** +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-get-values-ioctl.rst. + +Name +==== + +GPIOHANDLE_GET_LINE_VALUES_IOCTL - Get the values of all requested lines. + +Synopsis +======== + +.. c:macro:: GPIOHANDLE_GET_LINE_VALUES_IOCTL + +``int ioctl(int handle_fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, struct gpiohandle_data *values)`` + +Arguments +========= + +``handle_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +``values`` + The :c:type:`line_values` to be populated. + +Description +=========== + +Get the values of all requested lines. + +The values of both input and output lines may be read. + +For output lines, the value returned is driver and configuration dependent and +may be either the output buffer (the last requested value set) or the input +buffer (the actual level of the line), and depending on the hardware and +configuration these may differ. + +This ioctl can also be used to read the line value for line events, +substituting the ``event_fd`` for the ``handle_fd``. As there is only +one line requested in that case, only the one value is returned in ``values``. + +Return Value +============ + +On success 0 and ``values`` populated with the values read. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst new file mode 100644 index 000000000000..8f1e748dccc8 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst @@ -0,0 +1,62 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIOHANDLE_SET_CONFIG_IOCTL: + +*************************** +GPIOHANDLE_SET_CONFIG_IOCTL +*************************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-set-config-ioctl.rst. + +Name +==== + +GPIOHANDLE_SET_CONFIG_IOCTL - Update the configuration of previously requested lines. + +Synopsis +======== + +.. c:macro:: GPIOHANDLE_SET_CONFIG_IOCTL + +``int ioctl(int handle_fd, GPIOHANDLE_SET_CONFIG_IOCTL, struct gpiohandle_config *config)`` + +Arguments +========= + +``handle_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +``config`` + The new :c:type:`configuration` to apply to the + requested lines. + +Description +=========== + +Update the configuration of previously requested lines, without releasing the +line or introducing potential glitches. + +The configuration applies to all requested lines. + +The same :ref:`gpio-get-linehandle-config-rules` that apply when requesting the +lines also apply when updating the line configuration. + +The motivating use case for this command is changing direction of +bi-directional lines between input and output, but it may be used more +generally move lines seamlessly from one configuration state to another. + +To only change the value of output lines, use +gpio-handle-set-line-values-ioctl.rst. + +First added in 5.5. + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst new file mode 100644 index 000000000000..0aa05e623a6c --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst @@ -0,0 +1,48 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_HANDLE_SET_LINE_VALUES_IOCTL: + +********************************* +GPIO_HANDLE_SET_LINE_VALUES_IOCTL +********************************* +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-set-values-ioctl.rst. + +Name +==== + +GPIO_HANDLE_SET_LINE_VALUES_IOCTL - Set the values of all requested output lines. + +Synopsis +======== + +.. c:macro:: GPIO_HANDLE_SET_LINE_VALUES_IOCTL + +``int ioctl(int handle_fd, GPIO_HANDLE_SET_LINE_VALUES_IOCTL, struct gpiohandle_data *values)`` + +Arguments +========= + +``handle_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +``values`` + The :c:type:`line_values` to set. + +Description +=========== + +Set the values of all requested output lines. + +Only the values of output lines may be set. +Attempting to set the value of input lines is an error (**EPERM**). + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst b/Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst new file mode 100644 index 000000000000..68b8d4f9f604 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_LINEEVENT_DATA_READ: + +************************ +GPIO_LINEEVENT_DATA_READ +************************ + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-event-read.rst. + +Name +==== + +GPIO_LINEEVENT_DATA_READ - Read edge detection events from a line event. + +Synopsis +======== + +``int read(int event_fd, void *buf, size_t count)`` + +Arguments +========= + +``event_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-lineevent-ioctl.rst. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at + least the size of a :c:type:`gpioevent_data`. + +Description +=========== + +Read edge detection events for a line from a line event. + +Edge detection must be enabled for the input line using either +``GPIOEVENT_REQUEST_RISING_EDGE`` or ``GPIOEVENT_REQUEST_FALLING_EDGE``, or +both. Edge events are then generated whenever edge interrupts are detected on +the input line. + +The kernel captures and timestamps edge events as close as possible to their +occurrence and stores them in a buffer from where they can be read by +userspace at its convenience using `read()`. + +The source of the clock for :c:type:`event.timestamp` is +``CLOCK_MONOTONIC``, except for kernels earlier than Linux 5.7 when it was +``CLOCK_REALTIME``. There is no indication in the :c:type:`gpioevent_data` +as to which clock source is used, it must be determined from either the kernel +version or sanity checks on the timestamp itself. + +Events read from the buffer are always in the same order that they were +detected by the kernel. + +The size of the kernel event buffer is fixed at 16 events. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the most recent event is discarded. +Overflow cannot be detected from userspace. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +The `read()` will block if no event is available and the ``event_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``event_fd`` is +readable using `poll()` or an equivalent. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size of +a :c:type:`gpio_lineevent_data` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst b/Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst new file mode 100644 index 000000000000..c4f5e1787a9d --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst @@ -0,0 +1,87 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_LINEINFO_CHANGED_READ: + +************************** +GPIO_LINEINFO_CHANGED_READ +************************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-lineinfo-changed-read.rst. + +Name +==== + +GPIO_LINEINFO_CHANGED_READ - Read line info change events for watched lines +from the chip. + +Synopsis +======== + +``int read(int chip_fd, void *buf, size_t count)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at least the size + of a :c:type:`gpioline_info_changed` event. + +Description +=========== + +Read line info change events for watched lines from the chip. + +.. note:: + Monitoring line info changes is not generally required, and would typically + only be performed by a system monitoring component. + + These events relate to changes in a line's request state or configuration, + not its value. Use gpio-lineevent-data-read.rst to receive events when a + line changes value. + +A line must be watched using gpio-get-lineinfo-watch-ioctl.rst to generate +info changed events. Subsequently, a request, release, or reconfiguration +of the line will generate an info changed event. + +The kernel timestamps events when they occur and stores them in a buffer +from where they can be read by userspace at its convenience using `read()`. + +The size of the kernel event buffer is fixed at 32 events per ``chip_fd``. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the most recent event is discarded. +Overflow cannot be detected from userspace. + +Events read from the buffer are always in the same order that they were +detected by the kernel, including when multiple lines are being monitored by +the one ``chip_fd``. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +A `read()` will block if no event is available and the ``chip_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``chip_fd`` is +readable using `poll()` or an equivalent. + +First added in 5.7. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size of +a :c:type:`gpioline_info_changed` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/obsolete.rst b/Documentation/userspace-api/gpio/obsolete.rst index c27bd6014a3d..c42538b44ec8 100644 --- a/Documentation/userspace-api/gpio/obsolete.rst +++ b/Documentation/userspace-api/gpio/obsolete.rst @@ -7,4 +7,5 @@ Obsolete GPIO Userspace APIs .. toctree:: :maxdepth: 1 + Character Device Userspace API (v1) Sysfs Interface From f1ccbe9aaea6488c4ef71f8f51ac2a620993acde Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:45 +0800 Subject: [PATCH 0022/1038] Documentation: gpio: capitalize GPIO in index title Capitalise the title of the GPIO documentation page to match other subsystems. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/admin-guide/gpio/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/gpio/index.rst b/Documentation/admin-guide/gpio/index.rst index 3ac3aa15fa8b..3f6d5a76702b 100644 --- a/Documentation/admin-guide/gpio/index.rst +++ b/Documentation/admin-guide/gpio/index.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0 ==== -gpio +GPIO ==== .. toctree:: From c3d336cae79728f8ed6711c5a6179cdc66802e89 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:46 +0800 Subject: [PATCH 0023/1038] Documentation: gpio: document gpio-mockup as obsoleted by gpio-sim Update the gpio-mockup documentation to note that is has been obsoleted by the gpio-sim. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/admin-guide/gpio/gpio-mockup.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/admin-guide/gpio/gpio-mockup.rst b/Documentation/admin-guide/gpio/gpio-mockup.rst index 493071da1738..d6e7438a7550 100644 --- a/Documentation/admin-guide/gpio/gpio-mockup.rst +++ b/Documentation/admin-guide/gpio/gpio-mockup.rst @@ -3,6 +3,14 @@ GPIO Testing Driver =================== +.. note:: + + This module has been obsoleted by the more flexible gpio-sim.rst. + New developments should use that API and existing developments are + encouraged to migrate as soon as possible. + This module will continue to be maintained but no new features will be + added. + The GPIO Testing Driver (gpio-mockup) provides a way to create simulated GPIO chips for testing purposes. The lines exposed by these chips can be accessed using the standard GPIO character device interface as well as manipulated From c055f7ed9744a5bae69a592eca84e22837736b92 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 15 Jan 2024 08:48:47 +0800 Subject: [PATCH 0024/1038] Documentation: gpio: move gpio-mockup into obsolete section The gpio-mockup has been obsoleted by the gpio-sim, so relocate its documentation into the obsolete section of the admin-guide book. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- Documentation/admin-guide/gpio/index.rst | 1 - Documentation/admin-guide/gpio/obsolete.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/gpio/index.rst b/Documentation/admin-guide/gpio/index.rst index 3f6d5a76702b..460afd29617e 100644 --- a/Documentation/admin-guide/gpio/index.rst +++ b/Documentation/admin-guide/gpio/index.rst @@ -9,7 +9,6 @@ GPIO Character Device Userspace API <../../userspace-api/gpio/chardev> gpio-aggregator - gpio-mockup gpio-sim Obsolete APIs diff --git a/Documentation/admin-guide/gpio/obsolete.rst b/Documentation/admin-guide/gpio/obsolete.rst index 5926e5440207..5adbff02d61f 100644 --- a/Documentation/admin-guide/gpio/obsolete.rst +++ b/Documentation/admin-guide/gpio/obsolete.rst @@ -9,4 +9,5 @@ Obsolete GPIO APIs Character Device Userspace API (v1) <../../userspace-api/gpio/chardev_v1> Sysfs Interface <../../userspace-api/gpio/sysfs> + Mockup Testing Module From 44a0d880b91dcaf3bb62a1ffa7253fe5373cd296 Mon Sep 17 00:00:00 2001 From: Wenhua Lin Date: Tue, 9 Jan 2024 15:38:49 +0800 Subject: [PATCH 0025/1038] gpio: eic-sprd: Optimize the calculation method of eic number The num_eics is a default value, but some SoCs support more than 8. In order to adapt to all projects, the total number of eics is automatically calculated through dts. Signed-off-by: Wenhua Lin Acked-by: Chunyan Zhang Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-eic-sprd.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c index be7f2fa5aa7b..6aed3e2732f7 100644 --- a/drivers/gpio/gpio-eic-sprd.c +++ b/drivers/gpio/gpio-eic-sprd.c @@ -108,7 +108,6 @@ static struct sprd_eic *to_sprd_eic(struct notifier_block *nb) struct sprd_eic_variant_data { enum sprd_eic_type type; - u32 num_eics; }; static const char *sprd_eic_label_name[SPRD_EIC_MAX] = { @@ -118,22 +117,18 @@ static const char *sprd_eic_label_name[SPRD_EIC_MAX] = { static const struct sprd_eic_variant_data sc9860_eic_dbnc_data = { .type = SPRD_EIC_DEBOUNCE, - .num_eics = 8, }; static const struct sprd_eic_variant_data sc9860_eic_latch_data = { .type = SPRD_EIC_LATCH, - .num_eics = 8, }; static const struct sprd_eic_variant_data sc9860_eic_async_data = { .type = SPRD_EIC_ASYNC, - .num_eics = 8, }; static const struct sprd_eic_variant_data sc9860_eic_sync_data = { .type = SPRD_EIC_SYNC, - .num_eics = 8, }; static inline void __iomem *sprd_eic_offset_base(struct sprd_eic *sprd_eic, @@ -595,6 +590,7 @@ static int sprd_eic_probe(struct platform_device *pdev) struct gpio_irq_chip *irq; struct sprd_eic *sprd_eic; struct resource *res; + u16 num_banks = 0; int ret, i; pdata = of_device_get_match_data(dev); @@ -628,10 +624,12 @@ static int sprd_eic_probe(struct platform_device *pdev) sprd_eic->base[i] = devm_ioremap_resource(dev, res); if (IS_ERR(sprd_eic->base[i])) return PTR_ERR(sprd_eic->base[i]); + + num_banks++; } sprd_eic->chip.label = sprd_eic_label_name[sprd_eic->type]; - sprd_eic->chip.ngpio = pdata->num_eics; + sprd_eic->chip.ngpio = num_banks * SPRD_EIC_PER_BANK_NR; sprd_eic->chip.base = -1; sprd_eic->chip.parent = dev; sprd_eic->chip.direction_input = sprd_eic_direction_input; From 3473765920eaa4e7d71a0e4869937d2117bb0398 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 22 Jan 2024 19:36:40 +0800 Subject: [PATCH 0026/1038] MAINTAINERS: add Documentation/userspace-api/gpio/ to GPIO UAPI section Add Documentation/userspace-api/gpio/ to the GPIO UAPI section. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 8d1052fa6a69..2d0bc0e128bb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9148,6 +9148,7 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git F: Documentation/ABI/obsolete/sysfs-gpio F: Documentation/ABI/testing/gpio-cdev +F: Documentation/userspace-api/gpio/ F: drivers/gpio/gpiolib-cdev.c F: include/uapi/linux/gpio.h F: tools/gpio/ From 460efee706c2b6a4daba62ec143fea29c2e7b358 Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Sat, 20 Jan 2024 11:00:01 -0600 Subject: [PATCH 0027/1038] spi: s3c64xx: Extract FIFO depth calculation to a dedicated macro Simplify the code by extracting all cases of FIFO depth calculation into a dedicated macro. No functional change. Signed-off-by: Sam Protsenko Reviewed-by: Andi Shyti Link: https://msgid.link/r/20240120170001.3356-1-semen.protsenko@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 0e48ffd499b9..432ec60d3568 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -109,6 +109,7 @@ #define TX_FIFO_LVL(v, i) (((v) >> 6) & FIFO_LVL_MASK(i)) #define RX_FIFO_LVL(v, i) (((v) >> (i)->port_conf->rx_lvl_offset) & \ FIFO_LVL_MASK(i)) +#define FIFO_DEPTH(i) ((FIFO_LVL_MASK(i) >> 1) + 1) #define S3C64XX_SPI_MAX_TRAILCNT 0x3ff #define S3C64XX_SPI_TRAILCNT_OFF 19 @@ -406,7 +407,7 @@ static bool s3c64xx_spi_can_dma(struct spi_controller *host, struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); if (sdd->rx_dma.ch && sdd->tx_dma.ch) { - return xfer->len > (FIFO_LVL_MASK(sdd) >> 1) + 1; + return xfer->len > FIFO_DEPTH(sdd); } else { return false; } @@ -495,9 +496,7 @@ static u32 s3c64xx_spi_wait_for_timeout(struct s3c64xx_spi_driver_data *sdd, void __iomem *regs = sdd->regs; unsigned long val = 1; u32 status; - - /* max fifo depth available */ - u32 max_fifo = (FIFO_LVL_MASK(sdd) >> 1) + 1; + u32 max_fifo = FIFO_DEPTH(sdd); if (timeout_ms) val = msecs_to_loops(timeout_ms); @@ -604,7 +603,7 @@ static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd, * For any size less than the fifo size the below code is * executed atleast once. */ - loops = xfer->len / ((FIFO_LVL_MASK(sdd) >> 1) + 1); + loops = xfer->len / FIFO_DEPTH(sdd); buf = xfer->rx_buf; do { /* wait for data to be received in the fifo */ @@ -741,7 +740,7 @@ static int s3c64xx_spi_transfer_one(struct spi_controller *host, struct spi_transfer *xfer) { struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); - const unsigned int fifo_len = (FIFO_LVL_MASK(sdd) >> 1) + 1; + const unsigned int fifo_len = FIFO_DEPTH(sdd); const void *tx_buf = NULL; void *rx_buf = NULL; int target_len = 0, origin_len = 0; @@ -1280,7 +1279,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) dev_dbg(&pdev->dev, "Samsung SoC SPI Driver loaded for Bus SPI-%d with %d Targets attached\n", sdd->port_id, host->num_chipselect); dev_dbg(&pdev->dev, "\tIOmem=[%pR]\tFIFO %dbytes\n", - mem_res, (FIFO_LVL_MASK(sdd) >> 1) + 1); + mem_res, FIFO_DEPTH(sdd)); pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); From 4c716711a5c5e89202facc9ca816bc89d4b8c745 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 20 Jan 2024 12:03:26 +0100 Subject: [PATCH 0028/1038] regulator: fixed-helper: Save a few bytes of memory when registering a 'nfc_llc' engine regulator_register_always_on() calls pass a string literal as the 'name' parameter. So kstrdup_const() can be used instead of kfree() to avoid a memory allocation in such cases. Signed-off-by: Christophe JAILLET Link: https://msgid.link/r/3f990a7b80d72c31371735b24ed08acbb8392c80.1705748586.git.christophe.jaillet@wanadoo.fr Signed-off-by: Mark Brown --- drivers/regulator/fixed-helper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/regulator/fixed-helper.c b/drivers/regulator/fixed-helper.c index 0eb2442456f0..2d5a42b2b3d8 100644 --- a/drivers/regulator/fixed-helper.c +++ b/drivers/regulator/fixed-helper.c @@ -15,7 +15,7 @@ static void regulator_fixed_release(struct device *dev) { struct fixed_regulator_data *data = container_of(dev, struct fixed_regulator_data, pdev.dev); - kfree(data->cfg.supply_name); + kfree_const(data->cfg.supply_name); kfree(data); } @@ -36,7 +36,7 @@ struct platform_device *regulator_register_always_on(int id, const char *name, if (!data) return NULL; - data->cfg.supply_name = kstrdup(name, GFP_KERNEL); + data->cfg.supply_name = kstrdup_const(name, GFP_KERNEL); if (!data->cfg.supply_name) { kfree(data); return NULL; From 6685d552a0cc3a86e10dbe6d98e1b51717a27a63 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 22 Jan 2024 17:15:09 +0800 Subject: [PATCH 0029/1038] dt-bindings: spi: fsl-lpspi: support i.MX95 LPSPI Add i.MX95 LPSPI compatible string, same as i.MX93 compatible with i.MX7ULP Signed-off-by: Peng Fan Acked-by: Conor Dooley Link: https://msgid.link/r/20240122091510.2077498-1-peng.fan@oss.nxp.com Signed-off-by: Mark Brown --- Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml index 727c5346b8ce..2ff174244795 100644 --- a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml +++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml @@ -22,6 +22,7 @@ properties: - enum: - fsl,imx8ulp-spi - fsl,imx93-spi + - fsl,imx95-spi - const: fsl,imx7ulp-spi reg: maxItems: 1 From 18ab9e9e8889ecba23a5e8b7f8924f09284e33d8 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 22 Jan 2024 17:15:10 +0800 Subject: [PATCH 0030/1038] dt-bindings: spi: nxp-fspi: support i.MX93 and i.MX95 Add i.MX93/95 flexspi compatible strings, which are compatible with i.MX8MM Signed-off-by: Peng Fan Acked-by: Han Xu Acked-by: Conor Dooley Link: https://msgid.link/r/20240122091510.2077498-2-peng.fan@oss.nxp.com Signed-off-by: Mark Brown --- .../devicetree/bindings/spi/spi-nxp-fspi.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml b/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml index 7fd591145480..4a5f41bde00f 100644 --- a/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml +++ b/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml @@ -15,12 +15,18 @@ allOf: properties: compatible: - enum: - - nxp,imx8dxl-fspi - - nxp,imx8mm-fspi - - nxp,imx8mp-fspi - - nxp,imx8qxp-fspi - - nxp,lx2160a-fspi + oneOf: + - enum: + - nxp,imx8dxl-fspi + - nxp,imx8mm-fspi + - nxp,imx8mp-fspi + - nxp,imx8qxp-fspi + - nxp,lx2160a-fspi + - items: + - enum: + - nxp,imx93-fspi + - nxp,imx95-fspi + - const: nxp,imx8mm-fspi reg: items: From f21401ca96e63cf5dbbc5e5a71def953c5cce8d9 Mon Sep 17 00:00:00 2001 From: tianyu2 Date: Tue, 9 Jan 2024 19:45:21 +0800 Subject: [PATCH 0031/1038] cpufreq: imx6: use regmap to read ocotp register Reading the ocotp register directly is unsafe and will cause the system to hang if its clock is not turned on in CCM. The regmap interface has clk enabled, which can solve this problem. Signed-off-by: tianyu2 Signed-off-by: Viresh Kumar --- drivers/cpufreq/imx6q-cpufreq.c | 43 +++++++++++---------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index 33728c242f66..c20d3ecc5a81 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #define PU_SOC_VOLTAGE_NORMAL 1250000 #define PU_SOC_VOLTAGE_HIGH 1275000 @@ -225,8 +227,6 @@ static void imx6x_disable_freq_in_opp(struct device *dev, unsigned long freq) static int imx6q_opp_check_speed_grading(struct device *dev) { - struct device_node *np; - void __iomem *base; u32 val; int ret; @@ -235,16 +235,11 @@ static int imx6q_opp_check_speed_grading(struct device *dev) if (ret) return ret; } else { - np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-ocotp"); - if (!np) - return -ENOENT; + struct regmap *ocotp; - base = of_iomap(np, 0); - of_node_put(np); - if (!base) { - dev_err(dev, "failed to map ocotp\n"); - return -EFAULT; - } + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6q-ocotp"); + if (IS_ERR(ocotp)) + return -ENOENT; /* * SPEED_GRADING[1:0] defines the max speed of ARM: @@ -254,8 +249,7 @@ static int imx6q_opp_check_speed_grading(struct device *dev) * 2b'00: 792000000Hz; * We need to set the max speed of ARM according to fuse map. */ - val = readl_relaxed(base + OCOTP_CFG3); - iounmap(base); + regmap_read(ocotp, OCOTP_CFG3, &val); } val >>= OCOTP_CFG3_SPEED_SHIFT; @@ -290,25 +284,16 @@ static int imx6ul_opp_check_speed_grading(struct device *dev) if (ret) return ret; } else { - struct device_node *np; - void __iomem *base; + struct regmap *ocotp; - np = of_find_compatible_node(NULL, NULL, "fsl,imx6ul-ocotp"); - if (!np) - np = of_find_compatible_node(NULL, NULL, - "fsl,imx6ull-ocotp"); - if (!np) + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6ul-ocotp"); + if (IS_ERR(ocotp)) + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6ull-ocotp"); + + if (IS_ERR(ocotp)) return -ENOENT; - base = of_iomap(np, 0); - of_node_put(np); - if (!base) { - dev_err(dev, "failed to map ocotp\n"); - return -EFAULT; - } - - val = readl_relaxed(base + OCOTP_CFG3); - iounmap(base); + regmap_read(ocotp, OCOTP_CFG3, &val); } /* From f661017e6d326ee187db24194cabb013d81bc2a6 Mon Sep 17 00:00:00 2001 From: Anastasia Belova Date: Wed, 17 Jan 2024 10:12:20 +0300 Subject: [PATCH 0032/1038] cpufreq: brcmstb-avs-cpufreq: add check for cpufreq_cpu_get's return value cpufreq_cpu_get may return NULL. To avoid NULL-dereference check it and return 0 in case of error. Found by Linux Verification Center (linuxtesting.org) with SVACE. Fixes: de322e085995 ("cpufreq: brcmstb-avs-cpufreq: AVS CPUfreq driver for Broadcom STB SoCs") Signed-off-by: Anastasia Belova Signed-off-by: Viresh Kumar --- drivers/cpufreq/brcmstb-avs-cpufreq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c index 35fb3a559ea9..1a1857b0a6f4 100644 --- a/drivers/cpufreq/brcmstb-avs-cpufreq.c +++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c @@ -481,6 +481,8 @@ static bool brcm_avs_is_firmware_loaded(struct private_data *priv) static unsigned int brcm_avs_cpufreq_get(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + if (!policy) + return 0; struct private_data *priv = policy->driver_data; cpufreq_cpu_put(policy); From 788715b5f21c6455264fe00a1779e61bec407fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Wed, 10 Jan 2024 11:23:02 -0300 Subject: [PATCH 0033/1038] cpufreq: mediatek-hw: Wait for CPU supplies before probing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before proceeding with the probe and enabling frequency scaling for the CPUs, make sure that all supplies feeding the CPUs have probed. This fixes an issue observed on MT8195-Tomato where if the mediatek-cpufreq-hw driver enabled the hardware (by writing to REG_FREQ_ENABLE) before the SPMI controller driver (spmi-mtk-pmif), behind which lies the big CPU supply, probed the platform would hang shortly after with "rcu: INFO: rcu_preempt detected stalls on CPUs/tasks" being printed in the log. Fixes: 4855e26bcf4d ("cpufreq: mediatek-hw: Add support for CPUFREQ HW") Signed-off-by: Nícolas F. R. A. Prado Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Signed-off-by: Viresh Kumar --- drivers/cpufreq/mediatek-cpufreq-hw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index d46afb3c0092..a1aa9385980a 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #define LUT_MAX_ENTRIES 32U @@ -300,7 +301,23 @@ static struct cpufreq_driver cpufreq_mtk_hw_driver = { static int mtk_cpufreq_hw_driver_probe(struct platform_device *pdev) { const void *data; - int ret; + int ret, cpu; + struct device *cpu_dev; + struct regulator *cpu_reg; + + /* Make sure that all CPU supplies are available before proceeding. */ + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "Failed to get cpu%d device\n", cpu); + + cpu_reg = devm_regulator_get_optional(cpu_dev, "cpu"); + if (IS_ERR(cpu_reg)) + return dev_err_probe(&pdev->dev, PTR_ERR(cpu_reg), + "CPU%d regulator get failed\n", cpu); + } + data = of_device_get_match_data(&pdev->dev); if (!data) From 9ac3ebaef3cc43ecd136911c44f1427286ee5a05 Mon Sep 17 00:00:00 2001 From: Erick Archer Date: Sun, 21 Jan 2024 11:43:44 +0100 Subject: [PATCH 0034/1038] Documentation: power: Use kcalloc() instead of kzalloc() As noted in the "Deprecated Interfaces, Language Features, Attributes, and Conventions" documentation [1], size calculations (especially multiplication) should not be performed in memory allocator (or similar) function arguments due to the risk of them overflowing. This could lead to values wrapping around and a smaller allocation being made than the caller was expecting. Using those allocations could lead to linear overflows of heap memory and other misbehaviors. So, in the example code use the purpose specific kcalloc() function instead of the argument size * count in the kzalloc() function. At the same time, modify the translations accordingly. Signed-off-by: Erick Archer Reviewed-by: Hu Haowen <2023002089@link.tyut.edu.cn> Reviewed-by: Yanteng Si Reviewed-by: Hu Haowen <2023002089@link.tyut.edu.cn> Signed-off-by: Viresh Kumar --- Documentation/power/opp.rst | 2 +- Documentation/translations/zh_CN/power/opp.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/power/opp.rst b/Documentation/power/opp.rst index a7c03c470980..1b7f1d854f14 100644 --- a/Documentation/power/opp.rst +++ b/Documentation/power/opp.rst @@ -305,7 +305,7 @@ dev_pm_opp_get_opp_count { /* Do things */ num_available = dev_pm_opp_get_opp_count(dev); - speeds = kzalloc(sizeof(u32) * num_available, GFP_KERNEL); + speeds = kcalloc(num_available, sizeof(u32), GFP_KERNEL); /* populate the table in increasing order */ freq = 0; while (!IS_ERR(opp = dev_pm_opp_find_freq_ceil(dev, &freq))) { diff --git a/Documentation/translations/zh_CN/power/opp.rst b/Documentation/translations/zh_CN/power/opp.rst index 8d6e3f6f6202..7470fa2d4c43 100644 --- a/Documentation/translations/zh_CN/power/opp.rst +++ b/Documentation/translations/zh_CN/power/opp.rst @@ -274,7 +274,7 @@ dev_pm_opp_get_opp_count { /* 做一些事情 */ num_available = dev_pm_opp_get_opp_count(dev); - speeds = kzalloc(sizeof(u32) * num_available, GFP_KERNEL); + speeds = kcalloc(num_available, sizeof(u32), GFP_KERNEL); /* 按升序填充表 */ freq = 0; while (!IS_ERR(opp = dev_pm_opp_find_freq_ceil(dev, &freq))) { From 90b1e56641bbab801e22141c56aa79dc095a3764 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Tue, 23 Jan 2024 09:33:29 +0000 Subject: [PATCH 0035/1038] mm/slub: directly load freelist from cpu partial slab in the likely case The likely case is that we get a usable slab from the cpu partial list, we can directly load freelist from it and return back, instead of going the other way that need more work, like reenable interrupt and recheck. But we need to remove the "VM_BUG_ON(!new.frozen)" in get_freelist() for reusing it, since cpu partial slab is not frozen. It seems acceptable since it's only for debug purpose. And get_freelist() also assumes it can return NULL if the freelist is empty, which is not possible for the cpu partial slab case, so we add "VM_BUG_ON(!freelist)" after get_freelist() to make it explicit. There is some small performance improvement too, which shows by: perf bench sched messaging -g 5 -t -l 100000 mm-stable slub-optimize Total time 7.473 7.209 Signed-off-by: Chengming Zhou Reviewed-by: Vlastimil Babka Signed-off-by: Vlastimil Babka --- mm/slub.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 2ef88bbf56a3..fda402b2d649 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3326,7 +3326,6 @@ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) counters = slab->counters; new.counters = counters; - VM_BUG_ON(!new.frozen); new.inuse = slab->objects; new.frozen = freelist != NULL; @@ -3498,18 +3497,20 @@ new_slab: slab = slub_percpu_partial(c); slub_set_percpu_partial(c, slab); - local_unlock_irqrestore(&s->cpu_slab->lock, flags); - stat(s, CPU_PARTIAL_ALLOC); - if (unlikely(!node_match(slab, node) || - !pfmemalloc_match(slab, gfpflags))) { - slab->next = NULL; - __put_partials(s, slab); - continue; + if (likely(node_match(slab, node) && + pfmemalloc_match(slab, gfpflags))) { + c->slab = slab; + freelist = get_freelist(s, slab); + VM_BUG_ON(!freelist); + stat(s, CPU_PARTIAL_ALLOC); + goto load_freelist; } - freelist = freeze_slab(s, slab); - goto retry_load_slab; + local_unlock_irqrestore(&s->cpu_slab->lock, flags); + + slab->next = NULL; + __put_partials(s, slab); } #endif From a6def11b6dcde5d8f1fcc9e2c0ae71399432b62e Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Tue, 23 Jan 2024 09:33:30 +0000 Subject: [PATCH 0036/1038] mm/slub: remove full list manipulation for non-debug slab Since debug slab is processed by free_to_partial_list(), and only debug slab which has SLAB_STORE_USER flag would care about the full list, we can remove these unrelated full list manipulations from __slab_free(). Acked-by: Christoph Lameter (Ampere) Reviewed-by: Vlastimil Babka Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slub.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index fda402b2d649..5c6fbeef05a8 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4188,7 +4188,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, * then add it. */ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { - remove_full(s, n, slab); add_partial(n, slab, DEACTIVATE_TO_TAIL); stat(s, FREE_ADD_PARTIAL); } @@ -4202,9 +4201,6 @@ slab_empty: */ remove_partial(n, slab); stat(s, FREE_REMOVE_PARTIAL); - } else { - /* Slab must be on the full list */ - remove_full(s, n, slab); } spin_unlock_irqrestore(&n->list_lock, flags); From c63349fc4a2d10f5d1b5ee805cb639ee88fd6e4a Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Tue, 23 Jan 2024 09:33:31 +0000 Subject: [PATCH 0037/1038] mm/slub: remove unused parameter in next_freelist_entry() The parameter "struct slab *slab" is unused in next_freelist_entry(), so just remove it. Acked-by: Christoph Lameter (Ampere) Reviewed-by: Vlastimil Babka Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slub.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 5c6fbeef05a8..7f235fa6592d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2243,7 +2243,7 @@ static void __init init_freelist_randomization(void) } /* Get the next entry on the pre-computed freelist randomized */ -static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab, +static void *next_freelist_entry(struct kmem_cache *s, unsigned long *pos, void *start, unsigned long page_limit, unsigned long freelist_count) @@ -2282,13 +2282,12 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) start = fixup_red_left(s, slab_address(slab)); /* First entry is used as the base of the freelist */ - cur = next_freelist_entry(s, slab, &pos, start, page_limit, - freelist_count); + cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count); cur = setup_object(s, cur); slab->freelist = cur; for (idx = 1; idx < slab->objects; idx++) { - next = next_freelist_entry(s, slab, &pos, start, page_limit, + next = next_freelist_entry(s, &pos, start, page_limit, freelist_count); next = setup_object(s, next); set_freepointer(s, cur, next); From 0229278bf33ea69cc1bba12c287f173e9b18c1f8 Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Fri, 19 Jan 2024 19:29:46 -0600 Subject: [PATCH 0038/1038] spi: s3c64xx: Add Exynos850 support Add SPI port configuration for Exynos850 SoC. It has 3 USI blocks which can be configured in SPI mode: * spi_0: BLK_PERI_SPI_0 (0x13940000) * spi_1: BLK_ALIVE_USI_CMGP00 (0x11d00000) * spi_2: BLK_ALIVE_USI_CMGP01 (0x11d20000) SPI FIFO depth is 64 bytes for all those SPI blocks, so the .fifo_lvl_mask value is set to 0x7f. All blocks have DIV_4 as the default internal clock divider, and an internal loopback mode to run a loopback test. Signed-off-by: Sam Protsenko Reviewed-by: Tudor Ambarus Link: https://msgid.link/r/20240120012948.8836-6-semen.protsenko@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 432ec60d3568..7f7eb8f742e4 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1460,6 +1460,17 @@ static const struct s3c64xx_spi_port_config exynos5433_spi_port_config = { .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, }; +static const struct s3c64xx_spi_port_config exynos850_spi_port_config = { + .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f }, + .rx_lvl_offset = 15, + .tx_st_done = 25, + .clk_div = 4, + .high_speed = true, + .clk_from_cmu = true, + .has_loopback = true, + .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, +}; + static const struct s3c64xx_spi_port_config exynosautov9_spi_port_config = { .fifo_lvl_mask = { 0x1ff, 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, @@ -1514,6 +1525,9 @@ static const struct of_device_id s3c64xx_spi_dt_match[] = { { .compatible = "samsung,exynos5433-spi", .data = (void *)&exynos5433_spi_port_config, }, + { .compatible = "samsung,exynos850-spi", + .data = (void *)&exynos850_spi_port_config, + }, { .compatible = "samsung,exynosautov9-spi", .data = (void *)&exynosautov9_spi_port_config, }, From 737cf74b38007fd5d5d2f15d4d4bc16e5f1cbfed Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Fri, 19 Jan 2024 19:29:43 -0600 Subject: [PATCH 0039/1038] spi: dt-bindings: samsung: Add Exynos850 SPI Document samsung,exynos850-spi compatible which will be used on Exynos850 SoC. Exynos850 doesn't have ioclk, so only two clocks are needed (bus clock and functional SPI clock). Signed-off-by: Sam Protsenko Reviewed-by: Tudor Ambarus Reviewed-by: Krzysztof Kozlowski Link: https://msgid.link/r/20240120012948.8836-3-semen.protsenko@linaro.org Signed-off-by: Mark Brown --- Documentation/devicetree/bindings/spi/samsung,spi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/spi/samsung,spi.yaml b/Documentation/devicetree/bindings/spi/samsung,spi.yaml index 79da99ca0e53..f71099852653 100644 --- a/Documentation/devicetree/bindings/spi/samsung,spi.yaml +++ b/Documentation/devicetree/bindings/spi/samsung,spi.yaml @@ -22,6 +22,7 @@ properties: - samsung,s5pv210-spi # for S5PV210 and S5PC110 - samsung,exynos4210-spi - samsung,exynos5433-spi + - samsung,exynos850-spi - samsung,exynosautov9-spi - tesla,fsd-spi - const: samsung,exynos7-spi From b017500ab53c06441ff7d3a681484e37039b4f57 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 22 Jan 2024 17:11:26 +0100 Subject: [PATCH 0040/1038] PM: sleep: Use bool for all 1-bit fields in struct dev_pm_info For some 1-bit fields in struct dev_pm_info the data type is bool, while for some other 1-bit fields in there it is unsigned int, and these differences are somewhat arbitrary. For consistency, change the data type of the latter to bool, so that all of the 1-bit fields in struct dev_pm_info fields are bool. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Greg Kroah-Hartman --- include/linux/pm.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/include/linux/pm.h b/include/linux/pm.h index a2f3e53a8196..97b0e23363c8 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -662,8 +662,8 @@ struct pm_subsys_data { struct dev_pm_info { pm_message_t power_state; - unsigned int can_wakeup:1; - unsigned int async_suspend:1; + bool can_wakeup:1; + bool async_suspend:1; bool in_dpm_list:1; /* Owned by the PM core */ bool is_prepared:1; /* Owned by the PM core */ bool is_suspended:1; /* Ditto */ @@ -682,10 +682,10 @@ struct dev_pm_info { bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ bool async_in_progress:1; /* Owned by the PM core */ - unsigned int must_resume:1; /* Owned by the PM core */ - unsigned int may_skip_resume:1; /* Set by subsystems */ + bool must_resume:1; /* Owned by the PM core */ + bool may_skip_resume:1; /* Set by subsystems */ #else - unsigned int should_wakeup:1; + bool should_wakeup:1; #endif #ifdef CONFIG_PM struct hrtimer suspend_timer; @@ -696,17 +696,17 @@ struct dev_pm_info { atomic_t usage_count; atomic_t child_count; unsigned int disable_depth:3; - unsigned int idle_notification:1; - unsigned int request_pending:1; - unsigned int deferred_resume:1; - unsigned int needs_force_resume:1; - unsigned int runtime_auto:1; + bool idle_notification:1; + bool request_pending:1; + bool deferred_resume:1; + bool needs_force_resume:1; + bool runtime_auto:1; bool ignore_children:1; - unsigned int no_callbacks:1; - unsigned int irq_safe:1; - unsigned int use_autosuspend:1; - unsigned int timer_autosuspends:1; - unsigned int memalloc_noio:1; + bool no_callbacks:1; + bool irq_safe:1; + bool use_autosuspend:1; + bool timer_autosuspends:1; + bool memalloc_noio:1; unsigned int links_count; enum rpm_request request; enum rpm_status runtime_status; From b204aa0f99cfe3c9d796ecfc0bc6f3f89585789e Mon Sep 17 00:00:00 2001 From: David Lechner Date: Tue, 23 Jan 2024 15:49:46 -0600 Subject: [PATCH 0041/1038] spi: consolidate setting message->spi Previously, __spi_sync() and __spi_async() set message->spi to the spi device independently after calling __spi_validate(). __spi_validate() also would conditionally set this if it needed to split the message since it wasn't set yet. Since both __spi_sync() and __spi_async() call __spi_validate(), we can consolidate this into only setting message->spi once (unconditionally) in __spi_validate(). This will also save any future callers of __spi_validate() from also needing to set message->spi. Signed-off-by: David Lechner Link: https://msgid.link/r/20240123214946.2616786-1-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index bc3d7c0ce6b4..7a70ef47cdf6 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -4059,6 +4059,8 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) if (list_empty(&message->transfers)) return -EINVAL; + message->spi = spi; + /* * If an SPI controller does not support toggling the CS line on each * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO @@ -4071,9 +4073,6 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) size_t maxsize = BITS_TO_BYTES(spi->bits_per_word); int ret; - /* spi_split_transfers_maxsize() requires message->spi */ - message->spi = spi; - ret = spi_split_transfers_maxsize(ctlr, message, maxsize, GFP_KERNEL); if (ret) @@ -4210,8 +4209,6 @@ static int __spi_async(struct spi_device *spi, struct spi_message *message) if (!ctlr->transfer) return -ENOTSUPP; - message->spi = spi; - SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_async); SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_async); @@ -4391,8 +4388,6 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) if (status != 0) return status; - message->spi = spi; - SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync); SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync); From 42969726a19f796f0e731ec74347fd8a0e4e91a2 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 23 Jan 2024 21:38:28 +0800 Subject: [PATCH 0042/1038] Documentation: gpio: describe uAPI behaviour for unsupported config The existing uAPI documentation does not adequately describe how the kernel handles the case where the underlying hardware or driver does not support the requested configuration. Add a Configuration Support section describing that behaviour to both the v1 and v2 documentation, and better document the errors returned where the requested configuration cannot be supported. Signed-off-by: Kent Gibson Signed-off-by: Bartosz Golaszewski --- .../userspace-api/gpio/error-codes.rst | 3 +- .../gpio/gpio-get-lineevent-ioctl.rst | 6 ++ .../gpio/gpio-get-linehandle-ioctl.rst | 39 +++++++++++++ .../gpio/gpio-handle-set-config-ioctl.rst | 5 +- .../gpio/gpio-v2-get-line-ioctl.rst | 57 ++++++++++++++++++- .../gpio/gpio-v2-line-set-config-ioctl.rst | 3 +- 6 files changed, 106 insertions(+), 7 deletions(-) diff --git a/Documentation/userspace-api/gpio/error-codes.rst b/Documentation/userspace-api/gpio/error-codes.rst index edf01f2cf9d2..6bf2948990cd 100644 --- a/Documentation/userspace-api/gpio/error-codes.rst +++ b/Documentation/userspace-api/gpio/error-codes.rst @@ -65,7 +65,8 @@ GPIO Error Codes - - ``ENXIO`` - - No device corresponding to this device special file exists. + - Typically returned when a feature requiring interrupt support was + requested, but the line does not support interrupts. .. note:: diff --git a/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst index 7d0b932925c6..09a9254f38cf 100644 --- a/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst +++ b/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst @@ -48,6 +48,12 @@ to its default state. Requesting a line already in use is an error (**EBUSY**). +Requesting edge detection on a line that does not support interrupts is an +error (**ENXIO**). + +As with the :ref:`line handle`, the +bias configuration is best effort. + Closing the ``chip_fd`` has no effect on existing line events. Configuration Rules diff --git a/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst index c8256afe306e..9112a9d31174 100644 --- a/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst +++ b/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst @@ -76,6 +76,45 @@ If no bias flags are set then the bias configuration is not changed. Requesting an invalid configuration is an error (**EINVAL**). + +.. _gpio-get-linehandle-config-support: + +Configuration Support +--------------------- + +Where the requested configuration is not directly supported by the underlying +hardware and driver, the kernel applies one of these approaches: + + - reject the request + - emulate the feature in software + - treat the feature as best effort + +The approach applied depends on whether the feature can reasonably be emulated +in software, and the impact on the hardware and userspace if the feature is not +supported. +The approach applied for each feature is as follows: + +============== =========== +Feature Approach +============== =========== +Bias best effort +Direction reject +Drive emulate +============== =========== + +Bias is treated as best effort to allow userspace to apply the same +configuration for platforms that support internal bias as those that require +external bias. +Worst case the line floats rather than being biased as expected. + +Drive is emulated by switching the line to an input when the line should not +be driven. + +In all cases, the configuration reported by gpio-get-lineinfo-ioctl.rst +is the requested configuration, not the resulting hardware configuration. +Userspace cannot determine if a feature is supported in hardware, is +emulated, or is best effort. + Return Value ============ diff --git a/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst index 8f1e748dccc8..d002a84681ac 100644 --- a/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst +++ b/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst @@ -41,12 +41,13 @@ line or introducing potential glitches. The configuration applies to all requested lines. -The same :ref:`gpio-get-linehandle-config-rules` that apply when requesting the +The same :ref:`gpio-get-linehandle-config-rules` and +:ref:`gpio-get-linehandle-config-support` that apply when requesting the lines also apply when updating the line configuration. The motivating use case for this command is changing direction of bi-directional lines between input and output, but it may be used more -generally move lines seamlessly from one configuration state to another. +generally to move lines seamlessly from one configuration state to another. To only change the value of output lines, use gpio-handle-set-line-values-ioctl.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst index d76e614c8343..56b975801b6a 100644 --- a/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst +++ b/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst @@ -74,7 +74,8 @@ If no bias flags are set then the bias configuration is not changed. The edge flags, ``GPIO_V2_LINE_FLAG_EDGE_xxx``, require ``GPIO_V2_LINE_FLAG_INPUT`` to be set and may be combined to detect both rising -and falling edges. +and falling edges. Requesting edge detection from a line that does not support +it is an error (**ENXIO**). Only one event clock flag, ``GPIO_V2_LINE_FLAG_EVENT_CLOCK_xxx``, may be set. If none are set then the event clock defaults to ``CLOCK_MONOTONIC``. @@ -86,11 +87,61 @@ The :c:type:`debounce_period_us` attribute may only be applied to lines with ``GPIO_V2_LINE_FLAG_INPUT`` set. When set, debounce applies to both the values returned by gpio-v2-line-get-values-ioctl.rst and the edges returned by gpio-v2-line-event-read.rst. If not -supported directly by hardware, the debouncing is performed in software by the -kernel. +supported directly by hardware, debouncing is emulated in software by the +kernel. Requesting debounce on a line that supports neither debounce in +hardware nor interrupts, as required for software emulation, is an error +(**ENXIO**). Requesting an invalid configuration is an error (**EINVAL**). +.. _gpio-v2-get-line-config-support: + +Configuration Support +--------------------- + +Where the requested configuration is not directly supported by the underlying +hardware and driver, the kernel applies one of these approaches: + + - reject the request + - emulate the feature in software + - treat the feature as best effort + +The approach applied depends on whether the feature can reasonably be emulated +in software, and the impact on the hardware and userspace if the feature is not +supported. +The approach applied for each feature is as follows: + +============== =========== +Feature Approach +============== =========== +Bias best effort +Debounce emulate +Direction reject +Drive emulate +Edge Detection reject +============== =========== + +Bias is treated as best effort to allow userspace to apply the same +configuration for platforms that support internal bias as those that require +external bias. +Worst case the line floats rather than being biased as expected. + +Debounce is emulated by applying a filter to hardware interrupts on the line. +An edge event is generated after an edge is detected and the line remains +stable for the debounce period. +The event timestamp corresponds to the end of the debounce period. + +Drive is emulated by switching the line to an input when the line should not +be actively driven. + +Edge detection requires interrupt support, and is rejected if that is not +supported. Emulation by polling can still be performed from userspace. + +In all cases, the configuration reported by gpio-v2-get-lineinfo-ioctl.rst +is the requested configuration, not the resulting hardware configuration. +Userspace cannot determine if a feature is supported in hardware, is +emulated, or is best effort. + Return Value ============ diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst index 126c2626ba6b..9b942a8a53ca 100644 --- a/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst +++ b/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst @@ -37,7 +37,8 @@ line or introducing potential glitches. The new configuration must specify the configuration of all requested lines. -The same :ref:`gpio-v2-get-line-config-rules` that apply when requesting the lines +The same :ref:`gpio-v2-get-line-config-rules` and +:ref:`gpio-v2-get-line-config-support` that apply when requesting the lines also apply when updating the line configuration. The motivating use case for this command is changing direction of From a6fc5c5b35f688480167a7a7947ddd91c07e72c8 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Thu, 25 Jan 2024 10:34:26 +0000 Subject: [PATCH 0043/1038] spi: cs42l43: Handle error from devm_pm_runtime_enable() As it devm_pm_runtime_enable() can fail due to memory allocations, it is best to handle the error. Suggested-by: Andy Shevchenko Reviewed-by: Andy Shevchenko Signed-off-by: Charles Keepax Link: https://msgid.link/r/20240125103426.2622549-1-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/spi/spi-cs42l43.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-cs42l43.c b/drivers/spi/spi-cs42l43.c index f13073e12593..b24190526ce9 100644 --- a/drivers/spi/spi-cs42l43.c +++ b/drivers/spi/spi-cs42l43.c @@ -244,7 +244,10 @@ static int cs42l43_spi_probe(struct platform_device *pdev) priv->ctlr->use_gpio_descriptors = true; priv->ctlr->auto_runtime_pm = true; - devm_pm_runtime_enable(priv->dev); + ret = devm_pm_runtime_enable(priv->dev); + if (ret) + return ret; + pm_runtime_idle(priv->dev); regmap_write(priv->regmap, CS42L43_TRAN_CONFIG6, CS42L43_FIFO_SIZE - 1); From 0da9a5794cfda615668eaefde811e8ef378134fe Mon Sep 17 00:00:00 2001 From: David Lechner Date: Thu, 25 Jan 2024 17:47:31 -0600 Subject: [PATCH 0044/1038] spi: avoid double validation in __spi_sync() The __spi_sync() function calls __spi_validate() early in the function. Later, it can call spi_async_locked() which calls __spi_validate() again. __spi_validate() is an expensive function, so we can improve performance measurably by avoiding calling it twice. Instead of calling spi_async_locked(), we can call __spi_async() with the spin lock held. spi_async_locked() is removed since there are no more callers. Signed-off-by: David Lechner Link: https://msgid.link/r/20240125234732.3530278-2-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 58 +++++------------------------------------------ 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 7a70ef47cdf6..6610aeced765 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -4278,57 +4278,6 @@ int spi_async(struct spi_device *spi, struct spi_message *message) } EXPORT_SYMBOL_GPL(spi_async); -/** - * spi_async_locked - version of spi_async with exclusive bus usage - * @spi: device with which data will be exchanged - * @message: describes the data transfers, including completion callback - * Context: any (IRQs may be blocked, etc) - * - * This call may be used in_irq and other contexts which can't sleep, - * as well as from task contexts which can sleep. - * - * The completion callback is invoked in a context which can't sleep. - * Before that invocation, the value of message->status is undefined. - * When the callback is issued, message->status holds either zero (to - * indicate complete success) or a negative error code. After that - * callback returns, the driver which issued the transfer request may - * deallocate the associated memory; it's no longer in use by any SPI - * core or controller driver code. - * - * Note that although all messages to a spi_device are handled in - * FIFO order, messages may go to different devices in other orders. - * Some device might be higher priority, or have various "hard" access - * time requirements, for example. - * - * On detection of any fault during the transfer, processing of - * the entire message is aborted, and the device is deselected. - * Until returning from the associated message completion callback, - * no other spi_message queued to that device will be processed. - * (This rule applies equally to all the synchronous transfer calls, - * which are wrappers around this core asynchronous primitive.) - * - * Return: zero on success, else a negative error code. - */ -static int spi_async_locked(struct spi_device *spi, struct spi_message *message) -{ - struct spi_controller *ctlr = spi->controller; - int ret; - unsigned long flags; - - ret = __spi_validate(spi, message); - if (ret != 0) - return ret; - - spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); - - ret = __spi_async(spi, message); - - spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); - - return ret; - -} - static void __spi_transfer_message_noqueue(struct spi_controller *ctlr, struct spi_message *msg) { bool was_busy; @@ -4376,6 +4325,7 @@ static void spi_complete(void *arg) static int __spi_sync(struct spi_device *spi, struct spi_message *message) { DECLARE_COMPLETION_ONSTACK(done); + unsigned long flags; int status; struct spi_controller *ctlr = spi->controller; @@ -4419,7 +4369,11 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) */ message->complete = spi_complete; message->context = &done; - status = spi_async_locked(spi, message); + + spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); + status = __spi_async(spi, message); + spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); + if (status == 0) { wait_for_completion(&done); status = message->status; From 86686b8f7ad3abe5aca17643efcee2bbce31a8f7 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 22 Jan 2024 12:22:38 +0100 Subject: [PATCH 0045/1038] PM: sleep: Simplify dpm_suspended_list walk in dpm_resume() Notice that devices can be moved to dpm_prepared_list before running their resume callbacks, in analogy with dpm_noirq_resume_devices() and dpm_resume_early(), because doing so will not affect the final ordering of that list. Namely, if a device is the first dpm_suspended_list entry while dpm_list_mtx is held, it has not been removed so far and it cannot be removed until dpm_list_mtx is released, so moving it to dpm_prepared_list at that point is valid. If it is removed later, while its resume callback is running, it will be deleted from dpm_prepared_list without changing the ordering of the other devices in that list. Accordingly, rearrange the while () loop in dpm_resume() to move devices to dpm_prepared_list before running their resume callbacks and implify the locking and device reference counting in it. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka --- drivers/base/power/main.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index fadcd0379dc2..10984aa5192b 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1017,25 +1017,19 @@ void dpm_resume(pm_message_t state) while (!list_empty(&dpm_suspended_list)) { dev = to_device(dpm_suspended_list.next); - - get_device(dev); + list_move_tail(&dev->power.entry, &dpm_prepared_list); if (!dev->power.async_in_progress) { + get_device(dev); + mutex_unlock(&dpm_list_mtx); device_resume(dev, state, false); + put_device(dev); + mutex_lock(&dpm_list_mtx); } - - if (!list_empty(&dev->power.entry)) - list_move_tail(&dev->power.entry, &dpm_prepared_list); - - mutex_unlock(&dpm_list_mtx); - - put_device(dev); - - mutex_lock(&dpm_list_mtx); } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); From 9cb1c9820f960a4b100e7760cb3773f344e7ae35 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 22 Jan 2024 12:24:21 +0100 Subject: [PATCH 0046/1038] PM: sleep: Relocate two device PM core functions Move is_async() and dpm_async_fn() in the PM core to a more suitable place. No functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka --- drivers/base/power/main.c | 58 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 10984aa5192b..a2cdef95d8c4 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -578,6 +578,35 @@ bool dev_pm_skip_resume(struct device *dev) return !dev->power.must_resume; } +static bool is_async(struct device *dev) +{ + return dev->power.async_suspend && pm_async_enabled + && !pm_trace_is_enabled(); +} + +static bool dpm_async_fn(struct device *dev, async_func_t func) +{ + reinit_completion(&dev->power.completion); + + if (is_async(dev)) { + dev->power.async_in_progress = true; + + get_device(dev); + + if (async_schedule_dev_nocall(func, dev)) + return true; + + put_device(dev); + } + /* + * Because async_schedule_dev_nocall() above has returned false or it + * has not been called at all, func() is not running and it is safe to + * update the async_in_progress flag without extra synchronization. + */ + dev->power.async_in_progress = false; + return false; +} + /** * device_resume_noirq - Execute a "noirq resume" callback for given device. * @dev: Device to handle. @@ -664,35 +693,6 @@ Out: } } -static bool is_async(struct device *dev) -{ - return dev->power.async_suspend && pm_async_enabled - && !pm_trace_is_enabled(); -} - -static bool dpm_async_fn(struct device *dev, async_func_t func) -{ - reinit_completion(&dev->power.completion); - - if (is_async(dev)) { - dev->power.async_in_progress = true; - - get_device(dev); - - if (async_schedule_dev_nocall(func, dev)) - return true; - - put_device(dev); - } - /* - * Because async_schedule_dev_nocall() above has returned false or it - * has not been called at all, func() is not running and it is safe to - * update the async_in_progress flag without extra synchronization. - */ - dev->power.async_in_progress = false; - return false; -} - static void async_resume_noirq(void *data, async_cookie_t cookie) { struct device *dev = data; From eaffb10b51bf74415c9252fd8fb4dd77122501ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Wed, 24 Jan 2024 17:31:43 -0300 Subject: [PATCH 0047/1038] cpufreq: mediatek-hw: Don't error out if supply is not found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit devm_regulator_get_optional() returns -ENODEV if no supply can be found. By introducing its usage, commit 788715b5f21c ("cpufreq: mediatek-hw: Wait for CPU supplies before probing") caused the driver to fail probe if no supply was present in any of the CPU DT nodes. Use devm_regulator_get() instead since the CPUs do require supplies even if not described in the DT. It will gracefully return a dummy regulator if none is found in the DT node, allowing probe to succeed. Fixes: 788715b5f21c ("cpufreq: mediatek-hw: Wait for CPU supplies before probing") Reported-by: kernelci.org bot Closes: https://linux.kernelci.org/test/case/id/65b0b169710edea22852a3fa/ Signed-off-by: Nícolas F. R. A. Prado Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Viresh Kumar --- drivers/cpufreq/mediatek-cpufreq-hw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index a1aa9385980a..8d097dcddda4 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -312,7 +312,7 @@ static int mtk_cpufreq_hw_driver_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, -EPROBE_DEFER, "Failed to get cpu%d device\n", cpu); - cpu_reg = devm_regulator_get_optional(cpu_dev, "cpu"); + cpu_reg = devm_regulator_get(cpu_dev, "cpu"); if (IS_ERR(cpu_reg)) return dev_err_probe(&pdev->dev, PTR_ERR(cpu_reg), "CPU%d regulator get failed\n", cpu); From 83a517c77715c4c268e893424ecfd4a407056af6 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 17:17:25 +0100 Subject: [PATCH 0048/1038] gpio: cdev: remove leftover function pointer typedefs The locking wrappers were replaces with lock guards. These typedefs are no longer needed. Signed-off-by: Bartosz Golaszewski Reviewed-by: Kent Gibson --- drivers/gpio/gpiolib-cdev.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 2a88736629ef..34d6712fa07c 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -61,11 +61,6 @@ static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_values), 8)); * interface to gpiolib GPIOs via ioctl()s. */ -typedef __poll_t (*poll_fn)(struct file *, struct poll_table_struct *); -typedef long (*ioctl_fn)(struct file *, unsigned int, unsigned long); -typedef ssize_t (*read_fn)(struct file *, char __user *, - size_t count, loff_t *); - /* * GPIO line handle management */ From 88b7049635dc5d0e2a7dfaaf89e70a9654ed6561 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 11:06:06 +0100 Subject: [PATCH 0049/1038] gpio: unexport GPIO irq domain functions only used internally There are no external users for the irq domain helpers so unexport them and remove the prototypes from the driver header. Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 93 +++++++++++++++++-------------------- include/linux/gpio/driver.h | 12 ----- 2 files changed, 42 insertions(+), 63 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 44c8f5743a24..d50a786f8176 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1254,8 +1254,8 @@ static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc) gpiochip_free_mask(&gc->irq.valid_mask); } -bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, - unsigned int offset) +static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, + unsigned int offset) { if (!gpiochip_line_is_valid(gc, offset)) return false; @@ -1264,7 +1264,6 @@ bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, return true; return test_bit(offset, gc->irq.valid_mask); } -EXPORT_SYMBOL_GPL(gpiochip_irqchip_irq_valid); #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY @@ -1439,6 +1438,43 @@ static unsigned int gpiochip_child_offset_to_irq_noop(struct gpio_chip *gc, return offset; } +/** + * gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ + * @domain: The IRQ domain used by this IRQ chip + * @data: Outermost irq_data associated with the IRQ + * @reserve: If set, only reserve an interrupt vector instead of assigning one + * + * This function is a wrapper that calls gpiochip_lock_as_irq() and is to be + * used as the activate function for the &struct irq_domain_ops. The host_data + * for the IRQ domain must be the &struct gpio_chip. + */ +static int gpiochip_irq_domain_activate(struct irq_domain *domain, + struct irq_data *data, bool reserve) +{ + struct gpio_chip *gc = domain->host_data; + unsigned int hwirq = irqd_to_hwirq(data); + + return gpiochip_lock_as_irq(gc, hwirq); +} + +/** + * gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ + * @domain: The IRQ domain used by this IRQ chip + * @data: Outermost irq_data associated with the IRQ + * + * This function is a wrapper that will call gpiochip_unlock_as_irq() and is to + * be used as the deactivate function for the &struct irq_domain_ops. The + * host_data for the IRQ domain must be the &struct gpio_chip. + */ +static void gpiochip_irq_domain_deactivate(struct irq_domain *domain, + struct irq_data *data) +{ + struct gpio_chip *gc = domain->host_data; + unsigned int hwirq = irqd_to_hwirq(data); + + return gpiochip_unlock_as_irq(gc, hwirq); +} + static void gpiochip_hierarchy_setup_domain_ops(struct irq_domain_ops *ops) { ops->activate = gpiochip_irq_domain_activate; @@ -1556,7 +1592,8 @@ static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc) * gpiochip by assigning the gpiochip as chip data, and using the irqchip * stored inside the gpiochip. */ -int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) +static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) { struct gpio_chip *gc = d->host_data; int ret = 0; @@ -1593,9 +1630,8 @@ int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwi return 0; } -EXPORT_SYMBOL_GPL(gpiochip_irq_map); -void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) +static void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) { struct gpio_chip *gc = d->host_data; @@ -1604,7 +1640,6 @@ void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) irq_set_chip_and_handler(irq, NULL, NULL); irq_set_chip_data(irq, NULL); } -EXPORT_SYMBOL_GPL(gpiochip_irq_unmap); static const struct irq_domain_ops gpiochip_domain_ops = { .map = gpiochip_irq_map, @@ -1626,50 +1661,6 @@ static struct irq_domain *gpiochip_simple_create_domain(struct gpio_chip *gc) return domain; } -/* - * TODO: move these activate/deactivate in under the hierarchicial - * irqchip implementation as static once SPMI and SSBI (all external - * users) are phased over. - */ -/** - * gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ - * @domain: The IRQ domain used by this IRQ chip - * @data: Outermost irq_data associated with the IRQ - * @reserve: If set, only reserve an interrupt vector instead of assigning one - * - * This function is a wrapper that calls gpiochip_lock_as_irq() and is to be - * used as the activate function for the &struct irq_domain_ops. The host_data - * for the IRQ domain must be the &struct gpio_chip. - */ -int gpiochip_irq_domain_activate(struct irq_domain *domain, - struct irq_data *data, bool reserve) -{ - struct gpio_chip *gc = domain->host_data; - unsigned int hwirq = irqd_to_hwirq(data); - - return gpiochip_lock_as_irq(gc, hwirq); -} -EXPORT_SYMBOL_GPL(gpiochip_irq_domain_activate); - -/** - * gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ - * @domain: The IRQ domain used by this IRQ chip - * @data: Outermost irq_data associated with the IRQ - * - * This function is a wrapper that will call gpiochip_unlock_as_irq() and is to - * be used as the deactivate function for the &struct irq_domain_ops. The - * host_data for the IRQ domain must be the &struct gpio_chip. - */ -void gpiochip_irq_domain_deactivate(struct irq_domain *domain, - struct irq_data *data) -{ - struct gpio_chip *gc = domain->host_data; - unsigned int hwirq = irqd_to_hwirq(data); - - return gpiochip_unlock_as_irq(gc, hwirq); -} -EXPORT_SYMBOL_GPL(gpiochip_irq_domain_deactivate); - static int gpiochip_to_irq(struct gpio_chip *gc, unsigned int offset) { struct irq_domain *domain = gc->irq.domain; diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 9a5c6c76e653..363d06c7b637 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -704,18 +704,6 @@ int bgpio_init(struct gpio_chip *gc, struct device *dev, #define BGPIOF_NO_OUTPUT BIT(5) /* only input */ #define BGPIOF_NO_SET_ON_INPUT BIT(6) -int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, - irq_hw_number_t hwirq); -void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq); - -int gpiochip_irq_domain_activate(struct irq_domain *domain, - struct irq_data *data, bool reserve); -void gpiochip_irq_domain_deactivate(struct irq_domain *domain, - struct irq_data *data); - -bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, - unsigned int offset); - #ifdef CONFIG_GPIOLIB_IRQCHIP int gpiochip_irqchip_add_domain(struct gpio_chip *gc, struct irq_domain *domain); From 6933ba529d06afdd3faf5501855e410b46b77160 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 25 Jan 2024 09:35:07 +0100 Subject: [PATCH 0050/1038] gpio: improve the API contract for setting direction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a GPIO driver returns a positive integer from one of the direction setter callbacks, we'll end up propagating it to user-space. Whether we should sanitize the values returned by callbacks is a different question but let's first improve the documentation and fortify the contract with GPIO providers. Reported-by: José Guilherme de Castro Rodrigues Signed-off-by: Bartosz Golaszewski Reviewed-by: Kent Gibson --- include/linux/gpio/driver.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 363d06c7b637..3a37d058cfcf 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -335,10 +335,12 @@ struct gpio_irq_chip { * (same as GPIO_LINE_DIRECTION_OUT / GPIO_LINE_DIRECTION_IN), * or negative error. It is recommended to always implement this * function, even on input-only or output-only gpio chips. - * @direction_input: configures signal "offset" as input, or returns error - * This can be omitted on input-only or output-only gpio chips. - * @direction_output: configures signal "offset" as output, or returns error - * This can be omitted on input-only or output-only gpio chips. + * @direction_input: configures signal "offset" as input, returns 0 on success + * or a negative error number. This can be omitted on input-only or + * output-only gpio chips. + * @direction_output: configures signal "offset" as output, returns 0 on + * success or a negative error number. This can be omitted on input-only + * or output-only gpio chips. * @get: returns value for signal "offset", 0=low, 1=high, or negative error * @get_multiple: reads values for multiple signals defined by "mask" and * stores them in "bits", returns 0 on success or negative error From f2675e588f928f7f3051765563a18f90332bd57b Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 15 Jan 2024 18:55:13 +0100 Subject: [PATCH 0051/1038] thermal: gov_fair_share: Fix dependency on trip points ordering The computation in the fair share governor's get_trip_level() function currently works under the assumption that the temperature ordering of trips[] in a thermal zone is ascending, which need not be the case. However, get_trip_level() can be made work regardless of whether or not the trips table is ordered by temperature in any way, so change it accordingly. Signed-off-by: Rafael J. Wysocki --- drivers/thermal/gov_fair_share.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/thermal/gov_fair_share.c b/drivers/thermal/gov_fair_share.c index 538abb7de4e2..4da25a0009d7 100644 --- a/drivers/thermal/gov_fair_share.c +++ b/drivers/thermal/gov_fair_share.c @@ -18,22 +18,24 @@ static int get_trip_level(struct thermal_zone_device *tz) { const struct thermal_trip *trip, *level_trip = NULL; - int trip_level; + int trip_level = -1; for_each_trip(tz, trip) { if (trip->temperature >= tz->temperature) - break; + continue; - level_trip = trip; + trip_level++; + + if (!level_trip || trip->temperature > level_trip->temperature) + level_trip = trip; } /* Bail out if the temperature is not greater than any trips. */ - if (!level_trip) + if (trip_level < 0) return 0; - trip_level = thermal_zone_trip_id(tz, level_trip); - - trace_thermal_zone_trip(tz, trip_level, level_trip->type); + trace_thermal_zone_trip(tz, thermal_zone_trip_id(tz, level_trip), + level_trip->type); return trip_level; } From 54d94009cb6f51d3ecd56bfc63c83e789c0b18b4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 15 Jan 2024 18:57:06 +0100 Subject: [PATCH 0052/1038] thermal: gov_bang_bang: Fix possible cooling device state ping-pong The current behavior of thermal_zone_trip_update() in the bang-bang thermal governor may be problematic for trip points with 0 hysteresis, because when the zone temperature reaches the trip temperature and stays there, it will then cause the cooling device go "on" and "off" alternately, which is not desirable. Address this by requiring the zone temperature to actually fall below trip->temperature - trip->hysteresis for the cooling device to go off. Signed-off-by: Rafael J. Wysocki --- drivers/thermal/gov_bang_bang.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/gov_bang_bang.c b/drivers/thermal/gov_bang_bang.c index 6ddf0accdc98..c3b2943a2db8 100644 --- a/drivers/thermal/gov_bang_bang.c +++ b/drivers/thermal/gov_bang_bang.c @@ -49,7 +49,7 @@ static int thermal_zone_trip_update(struct thermal_zone_device *tz, if (instance->target == 0 && tz->temperature >= trip->temperature) instance->target = 1; else if (instance->target == 1 && - tz->temperature <= trip->temperature - trip->hysteresis) + tz->temperature < trip->temperature - trip->hysteresis) instance->target = 0; dev_dbg(&instance->cdev->device, "target=%d\n", From 8613dda6af8ee1e8675a37c86bf8d4661d24bf14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahelenia=20Ziemia=C5=84ska?= Date: Sun, 28 Jan 2024 06:53:19 +0100 Subject: [PATCH 0053/1038] spi: Kconfig: cap[c]ability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ahelenia Ziemiańska Link: https://msgid.link/r/lq6gstev3sd7i4iw2btiq3gg7lhsraj5w74fkbp6lpbl6nkyff@tarta.nabijaczleweli.xyz Signed-off-by: Mark Brown --- drivers/spi/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index ddae0fde798e..bc7021da2fe9 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -694,7 +694,7 @@ config SPI_MTK_SNFI This enables support for SPI-NAND mode on the MediaTek NAND Flash Interface found on MediaTek ARM SoCs. This controller is implemented as a SPI-MEM controller with pipelined ECC - capcability. + capability. config SPI_WPCM_FIU tristate "Nuvoton WPCM450 Flash Interface Unit" From 66b3dc1f04135f89384af39ec4e38546b82e1510 Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Tue, 30 Jan 2024 09:41:07 +0800 Subject: [PATCH 0054/1038] mm/slub: remove parameter 'flags' in create_kmalloc_caches() After commit 16a1d968358a ("mm/slab: remove mm/slab.c and slab_def.h"), parameter 'flags' is only passed as 0 in create_kmalloc_caches(), and then it is only passed to new_kmalloc_cache(). So we can change parameter 'flags' to be a local variable with initial value 0 in new_kmalloc_cache() and remove parameter 'flags' in create_kmalloc_caches(). Also make new_kmalloc_cache() static due to it is only used in mm/slab_common.c. Signed-off-by: Zheng Yejian Acked-by: David Rientjes Reviewed-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slab.h | 4 +--- mm/slab_common.c | 13 +++++++------ mm/slub.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index f7df6d701c5b..9abec38be1d0 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -387,7 +387,7 @@ extern const struct kmalloc_info_struct { /* Kmalloc array related functions */ void setup_kmalloc_cache_index_table(void); -void create_kmalloc_caches(slab_flags_t); +void create_kmalloc_caches(void); extern u8 kmalloc_size_index[24]; @@ -422,8 +422,6 @@ gfp_t kmalloc_fix_flags(gfp_t flags); int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags); void __init kmem_cache_init(void); -void __init new_kmalloc_cache(int idx, enum kmalloc_cache_type type, - slab_flags_t flags); extern void create_boot_cache(struct kmem_cache *, const char *name, unsigned int size, slab_flags_t flags, unsigned int useroffset, unsigned int usersize); diff --git a/mm/slab_common.c b/mm/slab_common.c index 230ef7cc3467..1910252d7e89 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -853,9 +853,10 @@ static unsigned int __kmalloc_minalign(void) return max(minalign, arch_slab_minalign()); } -void __init -new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) +static void __init +new_kmalloc_cache(int idx, enum kmalloc_cache_type type) { + slab_flags_t flags = 0; unsigned int minalign = __kmalloc_minalign(); unsigned int aligned_size = kmalloc_info[idx].size; int aligned_idx = idx; @@ -902,7 +903,7 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) * may already have been created because they were needed to * enable allocations for slab creation. */ -void __init create_kmalloc_caches(slab_flags_t flags) +void __init create_kmalloc_caches(void) { int i; enum kmalloc_cache_type type; @@ -913,7 +914,7 @@ void __init create_kmalloc_caches(slab_flags_t flags) for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) { for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { if (!kmalloc_caches[type][i]) - new_kmalloc_cache(i, type, flags); + new_kmalloc_cache(i, type); /* * Caches that are not of the two-to-the-power-of size. @@ -922,10 +923,10 @@ void __init create_kmalloc_caches(slab_flags_t flags) */ if (KMALLOC_MIN_SIZE <= 32 && i == 6 && !kmalloc_caches[type][1]) - new_kmalloc_cache(1, type, flags); + new_kmalloc_cache(1, type); if (KMALLOC_MIN_SIZE <= 64 && i == 7 && !kmalloc_caches[type][2]) - new_kmalloc_cache(2, type, flags); + new_kmalloc_cache(2, type); } } #ifdef CONFIG_RANDOM_KMALLOC_CACHES diff --git a/mm/slub.c b/mm/slub.c index e66bc888d23b..4ebc0df410ff 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5668,7 +5668,7 @@ void __init kmem_cache_init(void) /* Now we can use the kmem_cache to allocate kmalloc slabs */ setup_kmalloc_cache_index_table(); - create_kmalloc_caches(0); + create_kmalloc_caches(); /* Setup random freelists for each cache */ init_freelist_randomization(); From 3598e577d1290008dcc753a015675fe617cdde45 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:04:56 +0800 Subject: [PATCH 0055/1038] x86: Drop CPU_SUP_INTEL from SCHED_MC_PRIO for the expansion amd-pstate driver also uses SCHED_MC_PRIO, so decouple the requirement of CPU_SUP_INTEL from the dependencies to allow compilation in kernels without Intel CPU support. Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Acked-by: Borislav Petkov (AMD) Signed-off-by: Rafael J. Wysocki --- arch/x86/Kconfig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5edec175b9bf..29d110285438 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1054,8 +1054,9 @@ config SCHED_MC config SCHED_MC_PRIO bool "CPU core priorities scheduler support" - depends on SCHED_MC && CPU_SUP_INTEL - select X86_INTEL_PSTATE + depends on SCHED_MC + select X86_INTEL_PSTATE if CPU_SUP_INTEL + select X86_AMD_PSTATE if CPU_SUP_AMD && ACPI select CPU_FREQ default y help From 12753d71e8c5c3e716cedba23ddeed508da0bdc4 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:04:57 +0800 Subject: [PATCH 0056/1038] ACPI: CPPC: Add helper to get the highest performance value Add support for getting the highest performance to the generic CPPC driver. This enables downstream drivers such as amd-pstate to discover and use these values. Refer to Chapter 8.4.6.1.1.1. Highest Performance of ACPI Specification 6.5 for details on continuous performance control of CPPC (linked below). Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Reviewed-by: Perry Yuan Acked-by: Huang Rui Signed-off-by: Meng Li Link: https://uefi.org/specs/ACPI/6.5/08_Processor_Configuration_and_Control.html?highlight=cppc#highest-performance [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/cppc_acpi.c | 13 +++++++++++++ include/acpi/cppc_acpi.h | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index d155a86a8614..a50e70abdf19 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1157,6 +1157,19 @@ int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf) return cppc_get_perf(cpunum, NOMINAL_PERF, nominal_perf); } +/** + * cppc_get_highest_perf - Get the highest performance register value. + * @cpunum: CPU from which to get highest performance. + * @highest_perf: Return address. + * + * Return: 0 for success, -EIO otherwise. + */ +int cppc_get_highest_perf(int cpunum, u64 *highest_perf) +{ + return cppc_get_perf(cpunum, HIGHEST_PERF, highest_perf); +} +EXPORT_SYMBOL_GPL(cppc_get_highest_perf); + /** * cppc_get_epp_perf - Get the epp register value. * @cpunum: CPU from which to get epp preference value. diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 3a0995f8bce8..930b6afba6f4 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -139,6 +139,7 @@ struct cppc_cpudata { #ifdef CONFIG_ACPI_CPPC_LIB extern int cppc_get_desired_perf(int cpunum, u64 *desired_perf); extern int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf); +extern int cppc_get_highest_perf(int cpunum, u64 *highest_perf); extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs); extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_enable(int cpu, bool enable); @@ -167,6 +168,10 @@ static inline int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf) { return -ENOTSUPP; } +static inline int cppc_get_highest_perf(int cpunum, u64 *highest_perf) +{ + return -ENOTSUPP; +} static inline int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs) { return -ENOTSUPP; From f3a052391822b772b4e27f2594526cf1eb103cab Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:04:58 +0800 Subject: [PATCH 0057/1038] cpufreq: amd-pstate: Enable amd-pstate preferred core support amd-pstate driver utilizes the functions and data structures provided by the ITMT architecture to enable the scheduler to favor scheduling on cores which can be get a higher frequency with lower voltage. We call it amd-pstate preferrred core. Here sched_set_itmt_core_prio() is called to set priorities and sched_set_itmt_support() is called to enable ITMT feature. amd-pstate driver uses the highest performance value to indicate the priority of CPU. The higher value has a higher priority. The initial core rankings are set up by amd-pstate when the system boots. Add a variable hw_prefcore in cpudata structure. It will check if the processor and power firmware support preferred core feature. Add one new early parameter `disable` to allow user to disable the preferred core. Only when hardware supports preferred core and user set `enabled` in early parameter, amd pstate driver supports preferred core featue. Tested-by: Oleksandr Natalenko Reviewed-by: Huang Rui Reviewed-by: Wyes Karny Reviewed-by: Mario Limonciello Co-developed-by: Perry Yuan Signed-off-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 131 ++++++++++++++++++++++++++++++++--- include/linux/amd-pstate.h | 4 ++ 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 1791d37fbc53..7c533ad91f4f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ #define AMD_PSTATE_TRANSITION_LATENCY 20000 #define AMD_PSTATE_TRANSITION_DELAY 1000 +#define AMD_PSTATE_PREFCORE_THRESHOLD 166 /* * TODO: We need more time to fine tune processors with shared memory solution @@ -64,6 +66,7 @@ static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; static bool cppc_enabled; +static bool amd_pstate_prefcore = true; /* * AMD Energy Preference Performance (EPP) @@ -297,13 +300,14 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - /* - * TODO: Introduce AMD specific power feature. - * - * CPPC entry doesn't indicate the highest performance in some ASICs. + /* For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as + * the default max perf. */ - highest_perf = amd_get_highest_perf(); - if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) + if (cpudata->hw_prefcore) + highest_perf = AMD_PSTATE_PREFCORE_THRESHOLD; + else highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); WRITE_ONCE(cpudata->highest_perf, highest_perf); @@ -324,8 +328,9 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - highest_perf = amd_get_highest_perf(); - if (highest_perf > cppc_perf.highest_perf) + if (cpudata->hw_prefcore) + highest_perf = AMD_PSTATE_PREFCORE_THRESHOLD; + else highest_perf = cppc_perf.highest_perf; WRITE_ONCE(cpudata->highest_perf, highest_perf); @@ -706,6 +711,80 @@ static void amd_perf_ctl_reset(unsigned int cpu) wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); } +/* + * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks + * due to locking, so queue the work for later. + */ +static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) +{ + sched_set_itmt_support(); +} +static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); + +/* + * Get the highest performance register value. + * @cpu: CPU from which to get highest performance. + * @highest_perf: Return address. + * + * Return: 0 for success, -EIO otherwise. + */ +static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) +{ + int ret; + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + u64 cap1; + + ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); + if (ret) + return ret; + WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); + } else { + u64 cppc_highest_perf; + + ret = cppc_get_highest_perf(cpu, &cppc_highest_perf); + if (ret) + return ret; + WRITE_ONCE(*highest_perf, cppc_highest_perf); + } + + return (ret); +} + +#define CPPC_MAX_PERF U8_MAX + +static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) +{ + int ret, prio; + u32 highest_perf; + + ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf); + if (ret) + return; + + cpudata->hw_prefcore = true; + /* check if CPPC preferred core feature is enabled*/ + if (highest_perf < CPPC_MAX_PERF) + prio = (int)highest_perf; + else { + pr_debug("AMD CPPC preferred core is unsupported!\n"); + cpudata->hw_prefcore = false; + return; + } + + if (!amd_pstate_prefcore) + return; + + /* + * The priorities can be set regardless of whether or not + * sched_set_itmt_support(true) has been called and it is valid to + * update them at any time after it has been called. + */ + sched_set_itmt_core_prio(prio, cpudata->cpu); + + schedule_work(&sched_prefcore_work); +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -727,6 +806,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; @@ -877,6 +958,17 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sysfs_emit(buf, "%u\n", perf); } +static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, + char *buf) +{ + bool hw_prefcore; + struct amd_cpudata *cpudata = policy->driver_data; + + hw_prefcore = READ_ONCE(cpudata->hw_prefcore); + + return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); +} + static ssize_t show_energy_performance_available_preferences( struct cpufreq_policy *policy, char *buf) { @@ -1074,18 +1166,27 @@ static ssize_t status_store(struct device *a, struct device_attribute *b, return ret < 0 ? ret : count; } +static ssize_t prefcore_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); +} + cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); static DEVICE_ATTR_RW(status); +static DEVICE_ATTR_RO(prefcore); static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_hw_prefcore, NULL, }; @@ -1093,6 +1194,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_hw_prefcore, &energy_performance_preference, &energy_performance_available_preferences, NULL, @@ -1100,6 +1202,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { static struct attribute *pstate_global_attributes[] = { &dev_attr_status.attr, + &dev_attr_prefcore.attr, NULL }; @@ -1151,6 +1254,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; cpudata->epp_policy = 0; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; @@ -1567,7 +1672,17 @@ static int __init amd_pstate_param(char *str) return amd_pstate_set_driver(mode_idx); } + +static int __init amd_prefcore_param(char *str) +{ + if (!strcmp(str, "disable")) + amd_pstate_prefcore = false; + + return 0; +} + early_param("amd_pstate", amd_pstate_param); +early_param("amd_prefcore", amd_prefcore_param); MODULE_AUTHOR("Huang Rui "); MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 6ad02ad9c7b4..68fc1bd8d851 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -52,6 +52,9 @@ struct amd_aperf_mperf { * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value * @boost_supported: check whether the Processor or SBIOS supports boost mode + * @hw_prefcore: check whether HW supports preferred core featue. + * Only when hw_prefcore and early prefcore param are true, + * AMD P-State driver supports preferred core featue. * @epp_policy: Last saved policy used to set energy-performance preference * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value @@ -85,6 +88,7 @@ struct amd_cpudata { u64 freq; bool boost_supported; + bool hw_prefcore; /* EPP feature related attributes*/ s16 epp_policy; From 9c4a13a08a9b7afa4bc33f57675358f0195e302c Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:04:59 +0800 Subject: [PATCH 0058/1038] ACPI: cpufreq: Add highest perf change notification Platform firmware sends notify 0x85 to inform the OS that the highest performance of a CPU has changed. This will be used by the AMD P-state driver to update the ranking of preferred cores and set the priority of cores accordingly. Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Link: https://uefi.org/specs/ACPI/6.5/05_ACPI_Software_Programming_Model.html#processor-device-notification-values [ rjw: New subject, changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_driver.c | 6 ++++++ include/linux/cpufreq.h | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 4bd16b3f0781..67db60eda370 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -27,6 +27,7 @@ #define ACPI_PROCESSOR_NOTIFY_PERFORMANCE 0x80 #define ACPI_PROCESSOR_NOTIFY_POWER 0x81 #define ACPI_PROCESSOR_NOTIFY_THROTTLING 0x82 +#define ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED 0x85 MODULE_AUTHOR("Paul Diefenbaugh"); MODULE_DESCRIPTION("ACPI Processor Driver"); @@ -83,6 +84,11 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data) acpi_bus_generate_netlink_event(device->pnp.device_class, dev_name(&device->dev), event, 0); break; + case ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED: + cpufreq_update_limits(pr->id); + acpi_bus_generate_netlink_event(device->pnp.device_class, + dev_name(&device->dev), event, 0); + break; default: acpi_handle_debug(handle, "Unsupported event [0x%x]\n", event); break; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index afda5f24d3dd..9bebeec24abb 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -263,6 +263,7 @@ static inline bool cpufreq_supports_freq_invariance(void) return false; } static inline void disable_cpufreq(void) { } +static inline void cpufreq_update_limits(unsigned int cpu) { } #endif #ifdef CONFIG_CPU_FREQ_STAT From e571a5e2068ef57945fcd5d0fb950f8f96da6dc8 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:00 +0800 Subject: [PATCH 0059/1038] cpufreq: amd-pstate: Update amd-pstate preferred core ranking dynamically Preferred core rankings can be changed dynamically by the platform based on the workload and platform conditions and accounting for thermals and aging. When this occurs, cpu priority need to be set. Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 52 ++++++++++++++++++++++++++++++++++++ include/linux/amd-pstate.h | 6 +++++ 2 files changed, 58 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7c533ad91f4f..08e112444c27 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -315,6 +315,7 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); + WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); return 0; } @@ -339,6 +340,7 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->lowest_nonlinear_perf, cppc_perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); + WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); if (cppc_state == AMD_PSTATE_ACTIVE) @@ -785,6 +787,40 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) schedule_work(&sched_prefcore_work); } +static void amd_pstate_update_limits(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct amd_cpudata *cpudata = policy->driver_data; + u32 prev_high = 0, cur_high = 0; + int ret; + bool highest_perf_changed = false; + + mutex_lock(&amd_pstate_driver_lock); + if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) + goto free_cpufreq_put; + + ret = amd_pstate_get_highest_perf(cpu, &cur_high); + if (ret) + goto free_cpufreq_put; + + prev_high = READ_ONCE(cpudata->prefcore_ranking); + if (prev_high != cur_high) { + highest_perf_changed = true; + WRITE_ONCE(cpudata->prefcore_ranking, cur_high); + + if (cur_high < CPPC_MAX_PERF) + sched_set_itmt_core_prio((int)cur_high, cpu); + } + +free_cpufreq_put: + cpufreq_cpu_put(policy); + + if (!highest_perf_changed) + cpufreq_update_policy(cpu); + + mutex_unlock(&amd_pstate_driver_lock); +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -958,6 +994,17 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sysfs_emit(buf, "%u\n", perf); } +static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, + char *buf) +{ + u32 perf; + struct amd_cpudata *cpudata = policy->driver_data; + + perf = READ_ONCE(cpudata->prefcore_ranking); + + return sysfs_emit(buf, "%u\n", perf); +} + static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, char *buf) { @@ -1176,6 +1223,7 @@ cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); @@ -1186,6 +1234,7 @@ static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, &amd_pstate_hw_prefcore, NULL, }; @@ -1194,6 +1243,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, &amd_pstate_hw_prefcore, &energy_performance_preference, &energy_performance_available_preferences, @@ -1537,6 +1587,7 @@ static struct cpufreq_driver amd_pstate_driver = { .suspend = amd_pstate_cpu_suspend, .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate", .attr = amd_pstate_attr, }; @@ -1551,6 +1602,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .online = amd_pstate_epp_cpu_online, .suspend = amd_pstate_epp_suspend, .resume = amd_pstate_epp_resume, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate-epp", .attr = amd_pstate_epp_attr, }; diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 68fc1bd8d851..d21838835abd 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -39,11 +39,16 @@ struct amd_aperf_mperf { * @cppc_req_cached: cached performance request hints * @highest_perf: the maximum performance an individual processor may reach, * assuming ideal conditions + * For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the fixed value as the highest_perf. * @nominal_perf: the maximum sustained performance level of the processor, * assuming ideal operating conditions * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power * savings are achieved * @lowest_perf: the absolute lowest performance level of the processor + * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher + * priority. * @max_freq: the frequency that mapped to highest_perf * @min_freq: the frequency that mapped to lowest_perf * @nominal_freq: the frequency that mapped to nominal_perf @@ -73,6 +78,7 @@ struct amd_cpudata { u32 nominal_perf; u32 lowest_nonlinear_perf; u32 lowest_perf; + u32 prefcore_ranking; u32 min_limit_perf; u32 max_limit_perf; u32 min_limit_freq; From 3a004e1fee4ba3d37976c1a9707869acc8d60b55 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:01 +0800 Subject: [PATCH 0060/1038] Documentation: amd-pstate: introduce amd-pstate preferred core Introduce amd-pstate preferred core. check preferred core state set by the kernel parameter: $ cat /sys/devices/system/cpu/amd-pstate/prefcore Tested-by: Oleksandr Natalenko Reviewed-by: Wyes Karny Reviewed-by: Mario Limonciello Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/amd-pstate.rst | 59 ++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 9eb26014d34b..0a3aa6b8ffd5 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -300,8 +300,8 @@ platforms. The AMD P-States mechanism is the more performance and energy efficiency frequency management method on AMD processors. -AMD Pstate Driver Operation Modes -================================= +``amd-pstate`` Driver Operation Modes +====================================== ``amd_pstate`` CPPC has 3 operation modes: autonomous (active) mode, non-autonomous (passive) mode and guided autonomous (guided) mode. @@ -353,6 +353,48 @@ is activated. In this mode, driver requests minimum and maximum performance level and the platform autonomously selects a performance level in this range and appropriate to the current workload. +``amd-pstate`` Preferred Core +================================= + +The core frequency is subjected to the process variation in semiconductors. +Not all cores are able to reach the maximum frequency respecting the +infrastructure limits. Consequently, AMD has redefined the concept of +maximum frequency of a part. This means that a fraction of cores can reach +maximum frequency. To find the best process scheduling policy for a given +scenario, OS needs to know the core ordering informed by the platform through +highest performance capability register of the CPPC interface. + +``amd-pstate`` preferred core enables the scheduler to prefer scheduling on +cores that can achieve a higher frequency with lower voltage. The preferred +core rankings can dynamically change based on the workload, platform conditions, +thermals and ageing. + +The priority metric will be initialized by the ``amd-pstate`` driver. The ``amd-pstate`` +driver will also determine whether or not ``amd-pstate`` preferred core is +supported by the platform. + +``amd-pstate`` driver will provide an initial core ordering when the system boots. +The platform uses the CPPC interfaces to communicate the core ranking to the +operating system and scheduler to make sure that OS is choosing the cores +with highest performance firstly for scheduling the process. When ``amd-pstate`` +driver receives a message with the highest performance change, it will +update the core ranking and set the cpu's priority. + +``amd-pstate`` Preferred Core Switch +================================= +Kernel Parameters +----------------- + +``amd-pstate`` peferred core`` has two states: enable and disable. +Enable/disable states can be chosen by different kernel parameters. +Default enable ``amd-pstate`` preferred core. + +``amd_prefcore=disable`` + +For systems that support ``amd-pstate`` preferred core, the core rankings will +always be advertised by the platform. But OS can choose to ignore that via the +kernel parameter ``amd_prefcore=disable``. + User Space Interface in ``sysfs`` - General =========================================== @@ -385,6 +427,19 @@ control its functionality at the system level. They are located in the to the operation mode represented by that string - or to be unregistered in the "disable" case. +``prefcore`` + Preferred core state of the driver: "enabled" or "disabled". + + "enabled" + Enable the ``amd-pstate`` preferred core. + + "disabled" + Disable the ``amd-pstate`` preferred core + + + This attribute is read-only to check the state of preferred core set + by the kernel parameter. + ``cpupower`` tool support for ``amd-pstate`` =============================================== From dfddf34a3f0d45483f5b3e46c2e7bda173796f1b Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:02 +0800 Subject: [PATCH 0061/1038] Documentation: introduce amd-pstate preferrd core mode kernel command line options amd-pstate driver support enable/disable preferred core. Default enabled on platforms supporting amd-pstate preferred core. Disable amd-pstate preferred core with "amd_prefcore=disable" added to the kernel command line. Signed-off-by: Meng Li Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Tested-by: Oleksandr Natalenko Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 31b3a25680d0..522530432548 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -374,6 +374,11 @@ selects a performance level in this range and appropriate to the current workload. + amd_prefcore= + [X86] + disable + Disable amd-pstate preferred core. + amijoy.map= [HW,JOY] Amiga joystick support Map of devices attached to JOY0DAT and JOY1DAT Format: , From e6c5812dc4d0b3e890608cb9c98597d1bed7e937 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Wed, 31 Jan 2024 11:07:27 -0600 Subject: [PATCH 0062/1038] spi: reorder spi_message struct member doc comments The members of `struct spi_message` were reordered in commit ae2ade4ba581 ("spi: Reorder fields in 'struct spi_message'") but the documentation comments were not updated to match. This commit updates the comments to match the new order. Signed-off-by: David Lechner Link: https://msgid.link/r/20240131170732.1665105-1-dlechner@baylibre.com Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index f306aececeaf..29c3e4dd5d93 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1113,16 +1113,16 @@ struct spi_transfer { * @spi: SPI device to which the transaction is queued * @is_dma_mapped: if true, the caller provided both DMA and CPU virtual * addresses for each transfer buffer + * @prepared: spi_prepare_message was called for the this message + * @status: zero for success, else negative errno * @complete: called to report transaction completions * @context: the argument to complete() when it's called * @frame_length: the total number of bytes in the message * @actual_length: the total number of bytes that were transferred in all * successful segments - * @status: zero for success, else negative errno * @queue: for use by whichever driver currently owns the message * @state: for use by whichever driver currently owns the message * @resources: for resource management when the SPI message is processed - * @prepared: spi_prepare_message was called for the this message * * A @spi_message is used to execute an atomic sequence of data transfers, * each represented by a struct spi_transfer. The sequence is "atomic" From 64db3e8d7be00462ad95dd81f11cf8e3f0968d70 Mon Sep 17 00:00:00 2001 From: Andrei Simion Date: Thu, 1 Feb 2024 18:15:17 +0200 Subject: [PATCH 0063/1038] regulator: dt-bindings: microchip,mcp16502: convert to YAML Convert devicetree binding mcp16502-regulator.txt to YAML format. Signed-off-by: Andrei Simion Reviewed-by: Krzysztof Kozlowski Link: https://msgid.link/r/20240201161517.492162-1-andrei.simion@microchip.com Signed-off-by: Mark Brown --- .../bindings/regulator/mcp16502-regulator.txt | 144 -------------- .../regulator/microchip,mcp16502.yaml | 180 ++++++++++++++++++ 2 files changed, 180 insertions(+), 144 deletions(-) delete mode 100644 Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt create mode 100644 Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml diff --git a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt b/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt deleted file mode 100644 index 451cc4e86b01..000000000000 --- a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt +++ /dev/null @@ -1,144 +0,0 @@ -MCP16502 PMIC - -Required properties: -- compatible: "microchip,mcp16502" -- reg: I2C slave address -- lpm-gpios: GPIO for LPM pin. Note that this GPIO *must* remain high during - suspend-to-ram, keeping the PMIC into HIBERNATE mode; this - property is optional; -- regulators: A node that houses a sub-node for each regulator within - the device. Each sub-node is identified using the node's - name. The content of each sub-node is defined by the - standard binding for regulators; see regulator.txt. - -Regulators of MCP16502 PMIC: -1) VDD_IO - Buck (1.2 - 3.7 V) -2) VDD_DDR - Buck (0.6 - 1.85 V) -3) VDD_CORE - Buck (0.6 - 1.85 V) -4) VDD_OTHER - BUCK (0.6 - 1.85 V) -5) LDO1 - LDO (1.2 - 3.7 V) -6) LDO2 - LDO (1.2 - 3.7 V) - -Regulator modes: -2 - FPWM: higher precision, higher consumption -4 - AutoPFM: lower precision, lower consumption - -Each regulator is defined using the standard binding for regulators. - -Example: - -mcp16502@5b { - compatible = "microchip,mcp16502"; - reg = <0x5b>; - status = "okay"; - lpm-gpios = <&pioBU 7 GPIO_ACTIVE_HIGH>; - - regulators { - VDD_IO { - regulator-name = "VDD_IO"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-off-in-suspend; - regulator-mode = <4>; - }; - }; - - VDD_DDR { - regulator-name = "VDD_DDR"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - }; - - VDD_CORE { - regulator-name = "VDD_CORE"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-off-in-suspend; - regulator-mode = <4>; - }; - }; - - VDD_OTHER { - regulator-name = "VDD_OTHER"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-off-in-suspend; - regulator-mode = <4>; - }; - }; - - LDO1 { - regulator-name = "LDO1"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - }; - - regulator-state-mem { - regulator-off-in-suspend; - }; - }; - - LDO2 { - regulator-name = "LDO2"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - }; - - regulator-state-mem { - regulator-off-in-suspend; - }; - }; - - }; -}; diff --git a/Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml b/Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml new file mode 100644 index 000000000000..1aca3646789e --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/microchip,mcp16502.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MCP16502 - High-Performance PMIC + +maintainers: + - Andrei Simion + +description: + The MCP16502 is an optimally integrated PMIC compatible + with Microchip's eMPUs(Embedded Microprocessor Units), + requiring Dynamic Voltage Scaling (DVS) with the use + of High-Performance mode (HPM). + +properties: + compatible: + const: microchip,mcp16502 + + lpm-gpios: + maxItems: 1 + description: GPIO for LPM pin. + Note that this GPIO must remain high during + suspend-to-ram, keeping the PMIC into HIBERNATE mode. + + reg: + maxItems: 1 + + regulators: + type: object + additionalProperties: false + description: List of regulators and its properties. + + patternProperties: + "^(VDD_(IO|CORE|DDR|OTHER)|LDO[1-2])$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + properties: + regulator-initial-mode: + enum: [2, 4] + default: 2 + description: Initial operating mode + + regulator-allowed-modes: + items: + enum: [2, 4] + description: Supported modes + 2 - FPWM higher precision, higher consumption + 4 - AutoPFM lower precision, lower consumption + +required: + - compatible + - reg + - regulators + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + pmic@5b { + compatible = "microchip,mcp16502"; + reg = <0x5b>; + + regulators { + VDD_IO { + regulator-name = "VDD_IO"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + VDD_DDR { + regulator-name = "VDD_DDR"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + }; + + VDD_CORE { + regulator-name = "VDD_CORE"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + VDD_OTHER { + regulator-name = "VDD_OTHER"; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1250000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + LDO1 { + regulator-name = "LDO1"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + }; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + LDO2 { + regulator-name = "LDO2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3700000>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + }; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; + }; From 1e942b5bb18e8a9709a5cb5293b15dd54ed79f98 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Fri, 2 Feb 2024 10:34:30 +0000 Subject: [PATCH 0064/1038] spi: cs42l43: Clean up of firmware node As we get a child node in the OF case, we should also clean up the reference, add code to do so. Suggested-by: Andy Shevchenko Reviewed-by: Andy Shevchenko Signed-off-by: Charles Keepax Link: https://msgid.link/r/20240202103430.951598-1-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/spi/spi-cs42l43.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/spi/spi-cs42l43.c b/drivers/spi/spi-cs42l43.c index b24190526ce9..d45d9e2e1611 100644 --- a/drivers/spi/spi-cs42l43.c +++ b/drivers/spi/spi-cs42l43.c @@ -202,6 +202,11 @@ static size_t cs42l43_spi_max_length(struct spi_device *spi) return CS42L43_SPI_MAX_LENGTH; } +static void cs42l43_release_of_node(void *data) +{ + fwnode_handle_put(data); +} + static int cs42l43_spi_probe(struct platform_device *pdev) { struct cs42l43 *cs42l43 = dev_get_drvdata(pdev->dev.parent); @@ -228,12 +233,6 @@ static int cs42l43_spi_probe(struct platform_device *pdev) priv->ctlr->transfer_one = cs42l43_transfer_one; priv->ctlr->set_cs = cs42l43_set_cs; priv->ctlr->max_transfer_size = cs42l43_spi_max_length; - - if (is_of_node(fwnode)) - fwnode = fwnode_get_named_child_node(fwnode, "spi"); - - device_set_node(&priv->ctlr->dev, fwnode); - priv->ctlr->mode_bits = SPI_3WIRE | SPI_MODE_X_MASK; priv->ctlr->flags = SPI_CONTROLLER_HALF_DUPLEX; priv->ctlr->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(16) | @@ -257,6 +256,17 @@ static int cs42l43_spi_probe(struct platform_device *pdev) regmap_write(priv->regmap, CS42L43_SPI_CONFIG3, 0); regmap_write(priv->regmap, CS42L43_SPI_CONFIG4, CS42L43_SPI_STALL_ENA_MASK); + if (is_of_node(fwnode)) { + fwnode = fwnode_get_named_child_node(fwnode, "spi"); + ret = devm_add_action(priv->dev, cs42l43_release_of_node, fwnode); + if (ret) { + fwnode_handle_put(fwnode); + return ret; + } + } + + device_set_node(&priv->ctlr->dev, fwnode); + ret = devm_spi_register_controller(priv->dev, priv->ctlr); if (ret) { dev_err(priv->dev, "Failed to register SPI controller: %d\n", ret); From ff3d9bfa25fabc4bf014d4e350ded1dc12b0a44a Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Thu, 1 Feb 2024 13:16:33 +0100 Subject: [PATCH 0065/1038] dt-bindings: gpio: renesas,rcar-gpio: Add r8a779h0 support Document support for GPIO controller blocks in the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Acked-by: Conor Dooley Signed-off-by: Bartosz Golaszewski --- Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml index aa424e2b95f8..cc7a950a6030 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml @@ -53,6 +53,7 @@ properties: - renesas,gpio-r8a779a0 # R-Car V3U - renesas,gpio-r8a779f0 # R-Car S4-8 - renesas,gpio-r8a779g0 # R-Car V4H + - renesas,gpio-r8a779h0 # R-Car V4M - const: renesas,rcar-gen4-gpio # R-Car Gen4 reg: From f57595788244a838deec2d3be375291327cbc035 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Wed, 24 Jan 2024 21:58:57 +0100 Subject: [PATCH 0066/1038] gpio: vf610: allow disabling the vf610 driver The vf610 gpio driver is enabled by default for all i.MX machines, without any option to disable it in a board-specific config file. Most i.MX chipsets have no hardware for this driver. Change the default to enable GPIO_VF610 for SOC_VF610 and disable it otherwise. Add a text description after the bool type, this makes the driver selectable by make config etc. Fixes: 30a35c07d9e9 ("gpio: vf610: drop the SOC_VF610 dependency for GPIO_VF610") Signed-off-by: Martin Kaiser Signed-off-by: Bartosz Golaszewski --- drivers/gpio/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 1301cec94f12..353af1a4d0ac 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -711,7 +711,8 @@ config GPIO_UNIPHIER Say yes here to support UniPhier GPIOs. config GPIO_VF610 - def_bool y + bool "VF610 GPIO support" + default y if SOC_VF610 depends on ARCH_MXC select GPIOLIB_IRQCHIP help From 3eac8bbed22e940ac1645a884f221bef408f675c Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Wed, 24 Jan 2024 21:58:58 +0100 Subject: [PATCH 0067/1038] gpio: vf610: enable COMPILE_TEST Enable COMPILE_TEST for the vf610 gpio driver to support test builds on systems without this hardware. Signed-off-by: Martin Kaiser Signed-off-by: Bartosz Golaszewski --- drivers/gpio/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 353af1a4d0ac..3081406ff57a 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -713,7 +713,7 @@ config GPIO_UNIPHIER config GPIO_VF610 bool "VF610 GPIO support" default y if SOC_VF610 - depends on ARCH_MXC + depends on ARCH_MXC || COMPILE_TEST select GPIOLIB_IRQCHIP help Say yes here to support i.MX or Vybrid vf610 GPIOs. From 26da9a8d279f30f1d0aa33cea0003a3d37fa051c Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 2 Oct 2023 16:54:58 +0300 Subject: [PATCH 0068/1038] ACPI: NFIT: Switch to use acpi_evaluate_dsm_typed() The acpi_evaluate_dsm_typed() provides a way to check the type of the object evaluated by _DSM call. Use it instead of open coded variant. Signed-off-by: Andy Shevchenko Reviewed-by: Dave Jiang Reviewed-by: Dan Williams Signed-off-by: Rafael J. Wysocki --- drivers/acpi/nfit/core.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 802f8a56d1fa..d4595d1985b1 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -1737,9 +1737,8 @@ __weak void nfit_intel_shutdown_status(struct nfit_mem *nfit_mem) if ((nfit_mem->dsm_mask & (1 << func)) == 0) return; - out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj); - if (!out_obj || out_obj->type != ACPI_TYPE_BUFFER - || out_obj->buffer.length < sizeof(smart)) { + out_obj = acpi_evaluate_dsm_typed(handle, guid, revid, func, &in_obj, ACPI_TYPE_BUFFER); + if (!out_obj || out_obj->buffer.length < sizeof(smart)) { dev_dbg(dev->parent, "%s: failed to retrieve initial health\n", dev_name(dev)); ACPI_FREE(out_obj); From b377252eeec91f347cd538011f956a4fe73794b3 Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Tue, 30 Jan 2024 12:12:33 +0100 Subject: [PATCH 0069/1038] thermal: core: Change governor name to const char pointer All users are already assigning a const char * to the `governor_name` member of struct thermal_zone_params and to the `name` member of struct thermal_governor. Even if users are technically wrong, it just makes more sense to change this member to be a const char pointer instead of doing the other way around. Signed-off-by: AngeloGioacchino Del Regno Signed-off-by: Rafael J. Wysocki --- include/linux/thermal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/thermal.h b/include/linux/thermal.h index b7a3deb372fd..65d8f92a9a0d 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -214,7 +214,7 @@ struct thermal_zone_device { * @governor_list: node in thermal_governor_list (in thermal_core.c) */ struct thermal_governor { - char name[THERMAL_NAME_LENGTH]; + const char *name; int (*bind_to_tz)(struct thermal_zone_device *tz); void (*unbind_from_tz)(struct thermal_zone_device *tz); int (*throttle)(struct thermal_zone_device *tz, @@ -226,7 +226,7 @@ struct thermal_governor { /* Structure to define Thermal Zone parameters */ struct thermal_zone_params { - char governor_name[THERMAL_NAME_LENGTH]; + const char *governor_name; /* * a boolean to indicate if the thermal to hwmon sysfs interface From e25ac87d3f831fed002c34aadddaf4ebb4ea45ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 26 Jan 2024 13:04:33 +0100 Subject: [PATCH 0070/1038] pwm: atmel-hlcdc: Fix clock imbalance related to suspend support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suspend callback disables the periph clock when the PWM is enabled and resume reenables this clock if the PWM was disabled before. Judging from the code comment it's suspend that is wrong here. Fix accordingly. Fixes: f9bb9da7c09d ("pwm: atmel-hlcdc: Implement the suspend/resume hooks") Reviewed-by: Claudiu Beznea Link: https://lore.kernel.org/r/b51ea92b0a45eff3dc83b08adefd43d930df996c.1706269232.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-hlcdc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-atmel-hlcdc.c b/drivers/pwm/pwm-atmel-hlcdc.c index 3f2c5031a3ba..1f6fc9a9fcf3 100644 --- a/drivers/pwm/pwm-atmel-hlcdc.c +++ b/drivers/pwm/pwm-atmel-hlcdc.c @@ -185,7 +185,7 @@ static int atmel_hlcdc_pwm_suspend(struct device *dev) struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); /* Keep the periph clock enabled if the PWM is still running. */ - if (pwm_is_enabled(&atmel->chip.pwms[0])) + if (!pwm_is_enabled(&atmel->chip.pwms[0])) clk_disable_unprepare(atmel->hlcdc->periph_clk); return 0; From 868206f1eba77fdced68395f8782a4b306e74f03 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 2 Feb 2024 16:23:51 -0600 Subject: [PATCH 0071/1038] dt-bindings: gpio: mvebu: Fix "unevaluatedProperties" to be false Schemas need "unevaluatedProperties" or "additionalProperties" to be false in order to disallow undefined properties. Only common or otherwise incomplete schemas should allow undefined properties. Signed-off-by: Rob Herring Signed-off-by: Bartosz Golaszewski --- Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml b/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml index f1bd1e6b2e1f..33d4e4716516 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml @@ -115,7 +115,7 @@ allOf: required: - reg -unevaluatedProperties: true +unevaluatedProperties: false examples: - | From fc449cefe69d19d3a56903ca7e0fbc91c48ca3f5 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Thu, 1 Feb 2024 11:24:20 +0100 Subject: [PATCH 0072/1038] dt-bindings: gpio: pca9570: Add label property Add a label property to allow a custom name to be used for identifying a device on the board. This is useful when multiple devices are present on the same board. Similar change was done by commit ffae65fb1ae4 ("dt-bindings: spi: spi-cadence: Add label property") or by commit a53faa6bfa3b ("dt-bindings: hwmon: ina2xx: Add label property"). Signed-off-by: Michal Simek Acked-by: Rob Herring Signed-off-by: Bartosz Golaszewski --- Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml index 452f8972a965..6f73961001b7 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml @@ -28,6 +28,9 @@ properties: minItems: 4 maxItems: 8 + label: + description: A descriptive name for this device. + required: - compatible - reg From a875746f603b86134bd1924b7289fc3542fd45e7 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 13:29:42 -0300 Subject: [PATCH 0073/1038] gpio: gpiolib: make gpio_bus_type const Now that the driver core can properly handle constant struct bus_type, move the gpio_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index d50a786f8176..24d046268a01 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -63,7 +63,7 @@ static int gpio_bus_match(struct device *dev, struct device_driver *drv) return 1; } -static struct bus_type gpio_bus_type = { +static const struct bus_type gpio_bus_type = { .name = "gpio", .match = gpio_bus_match, }; From bc88528cda2eddc3e5ea304fc3f147f1b4186aa4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:09:44 +0100 Subject: [PATCH 0074/1038] PM: sleep: stats: Use array of suspend step names Replace suspend_step_name() in the suspend statistics code with an array of suspend step names which has fewer lines of code and less overhead. While at it, remove two unnecessary line breaks in suspend_stats_show() and adjust some white space in there to the kernel coding style for a more consistent code layout. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- include/linux/suspend.h | 3 ++- kernel/power/main.c | 50 +++++++++++++++-------------------------- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index ef503088942d..58f7352af205 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -41,7 +41,8 @@ typedef int __bitwise suspend_state_t; #define PM_SUSPEND_MAX ((__force suspend_state_t) 4) enum suspend_stat_step { - SUSPEND_FREEZE = 1, + SUSPEND_WORKING = 0, + SUSPEND_FREEZE, SUSPEND_PREPARE, SUSPEND_SUSPEND, SUSPEND_SUSPEND_LATE, diff --git a/kernel/power/main.c b/kernel/power/main.c index b1ae9b677d03..dca14543dfed 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -319,25 +319,17 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, power_attr(pm_test); #endif /* CONFIG_PM_SLEEP_DEBUG */ -static char *suspend_step_name(enum suspend_stat_step step) -{ - switch (step) { - case SUSPEND_FREEZE: - return "freeze"; - case SUSPEND_PREPARE: - return "prepare"; - case SUSPEND_SUSPEND: - return "suspend"; - case SUSPEND_SUSPEND_NOIRQ: - return "suspend_noirq"; - case SUSPEND_RESUME_NOIRQ: - return "resume_noirq"; - case SUSPEND_RESUME: - return "resume"; - default: - return ""; - } -} +static const char * const suspend_step_names[] = { + [SUSPEND_WORKING] = "", + [SUSPEND_FREEZE] = "freeze", + [SUSPEND_PREPARE] = "prepare", + [SUSPEND_SUSPEND] = "suspend", + [SUSPEND_SUSPEND_LATE] = "suspend_late", + [SUSPEND_SUSPEND_NOIRQ] = "suspend_noirq", + [SUSPEND_RESUME_NOIRQ] = "resume_noirq", + [SUSPEND_RESUME_EARLY] = "resume_early", + [SUSPEND_RESUME] = "resume", +}; #define suspend_attr(_name, format_str) \ static ssize_t _name##_show(struct kobject *kobj, \ @@ -392,16 +384,14 @@ static struct kobj_attribute last_failed_errno = __ATTR_RO(last_failed_errno); static ssize_t last_failed_step_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - int index; enum suspend_stat_step step; - char *last_failed_step = NULL; + int index; index = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; index %= REC_FAILED_NUM; step = suspend_stats.failed_steps[index]; - last_failed_step = suspend_step_name(step); - return sprintf(buf, "%s\n", last_failed_step); + return sprintf(buf, "%s\n", suspend_step_names[step]); } static struct kobj_attribute last_failed_step = __ATTR_RO(last_failed_step); @@ -473,30 +463,26 @@ static int suspend_stats_show(struct seq_file *s, void *unused) "failed_resume_noirq", suspend_stats.failed_resume_noirq); seq_printf(s, "failures:\n last_failed_dev:\t%-s\n", - suspend_stats.failed_devs[last_dev]); + suspend_stats.failed_devs[last_dev]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_dev + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; - seq_printf(s, "\t\t\t%-s\n", - suspend_stats.failed_devs[index]); + seq_printf(s, "\t\t\t%-s\n", suspend_stats.failed_devs[index]); } seq_printf(s, " last_failed_errno:\t%-d\n", suspend_stats.errno[last_errno]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_errno + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; - seq_printf(s, "\t\t\t%-d\n", - suspend_stats.errno[index]); + seq_printf(s, "\t\t\t%-d\n", suspend_stats.errno[index]); } seq_printf(s, " last_failed_step:\t%-s\n", - suspend_step_name( - suspend_stats.failed_steps[last_step])); + suspend_step_names[suspend_stats.failed_steps[last_step]]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_step + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; seq_printf(s, "\t\t\t%-s\n", - suspend_step_name( - suspend_stats.failed_steps[index])); + suspend_step_names[suspend_stats.failed_steps[index]]); } return 0; From b730bab0b9c4204d7dda3f5bc8adf4292497fc39 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:11:57 +0100 Subject: [PATCH 0075/1038] PM: sleep: stats: Use an array of step failure counters Instead of using a set of individual struct suspend_stats fields representing suspend step failure counters, use an array of counters indexed by enum suspend_stat_step for this purpose, which allows dpm_save_failed_step() to increment the appropriate counter automatically, so that its callers don't need to do that directly. It also allows suspend_stats_show() to carry out a loop over the counters array to print their values. Because the counters cannot become negative, use unsigned int for representing them. The only user-observable impact of this change is a different ordering of entries in the suspend_stats debugfs file which is not expected to matter. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 22 +++++++---------- include/linux/suspend.h | 12 +++------ kernel/power/main.c | 51 +++++++++++++++++++++------------------ kernel/power/suspend.c | 1 - 4 files changed, 40 insertions(+), 46 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index a2cdef95d8c4..896450503d0d 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -686,7 +686,6 @@ Out: TRACE_RESUME(error); if (error) { - suspend_stats.failed_resume_noirq++; dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); @@ -817,7 +816,6 @@ Out: complete_all(&dev->power.completion); if (error) { - suspend_stats.failed_resume_early++; dpm_save_failed_step(SUSPEND_RESUME_EARLY); dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async early" : " early", error); @@ -974,7 +972,6 @@ static void device_resume(struct device *dev, pm_message_t state, bool async) TRACE_RESUME(error); if (error) { - suspend_stats.failed_resume++; dpm_save_failed_step(SUSPEND_RESUME); dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async" : "", error); @@ -1323,10 +1320,9 @@ static int dpm_noirq_suspend_devices(pm_message_t state) if (!error) error = async_error; - if (error) { - suspend_stats.failed_suspend_noirq++; + if (error) dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ); - } + dpm_show_time(starttime, state, error, "noirq"); trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false); return error; @@ -1509,8 +1505,8 @@ int dpm_suspend_late(pm_message_t state) async_synchronize_full(); if (!error) error = async_error; + if (error) { - suspend_stats.failed_suspend_late++; dpm_save_failed_step(SUSPEND_SUSPEND_LATE); dpm_resume_early(resume_event(state)); } @@ -1789,10 +1785,10 @@ int dpm_suspend(pm_message_t state) async_synchronize_full(); if (!error) error = async_error; - if (error) { - suspend_stats.failed_suspend++; + + if (error) dpm_save_failed_step(SUSPEND_SUSPEND); - } + dpm_show_time(starttime, state, error, NULL); trace_suspend_resume(TPS("dpm_suspend"), state.event, false); return error; @@ -1943,11 +1939,11 @@ int dpm_suspend_start(pm_message_t state) int error; error = dpm_prepare(state); - if (error) { - suspend_stats.failed_prepare++; + if (error) dpm_save_failed_step(SUSPEND_PREPARE); - } else + else error = dpm_suspend(state); + dpm_show_time(starttime, state, error, "start"); return error; } diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 58f7352af205..5e4c4d4aed95 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -52,17 +52,12 @@ enum suspend_stat_step { SUSPEND_RESUME }; +#define SUSPEND_NR_STEPS SUSPEND_RESUME + struct suspend_stats { + unsigned int step_failures[SUSPEND_NR_STEPS]; int success; int fail; - int failed_freeze; - int failed_prepare; - int failed_suspend; - int failed_suspend_late; - int failed_suspend_noirq; - int failed_resume; - int failed_resume_early; - int failed_resume_noirq; #define REC_FAILED_NUM 2 int last_failed_dev; char failed_devs[REC_FAILED_NUM][40]; @@ -95,6 +90,7 @@ static inline void dpm_save_failed_errno(int err) static inline void dpm_save_failed_step(enum suspend_stat_step step) { + suspend_stats.step_failures[step-1]++; suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; suspend_stats.last_failed_step++; suspend_stats.last_failed_step %= REC_FAILED_NUM; diff --git a/kernel/power/main.c b/kernel/power/main.c index dca14543dfed..d7a02105b183 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -341,18 +341,28 @@ static struct kobj_attribute _name = __ATTR_RO(_name) suspend_attr(success, "%d\n"); suspend_attr(fail, "%d\n"); -suspend_attr(failed_freeze, "%d\n"); -suspend_attr(failed_prepare, "%d\n"); -suspend_attr(failed_suspend, "%d\n"); -suspend_attr(failed_suspend_late, "%d\n"); -suspend_attr(failed_suspend_noirq, "%d\n"); -suspend_attr(failed_resume, "%d\n"); -suspend_attr(failed_resume_early, "%d\n"); -suspend_attr(failed_resume_noirq, "%d\n"); suspend_attr(last_hw_sleep, "%llu\n"); suspend_attr(total_hw_sleep, "%llu\n"); suspend_attr(max_hw_sleep, "%llu\n"); +#define suspend_step_attr(_name, step) \ +static ssize_t _name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%u\n", \ + suspend_stats.step_failures[step-1]); \ +} \ +static struct kobj_attribute _name = __ATTR_RO(_name) + +suspend_step_attr(failed_freeze, SUSPEND_FREEZE); +suspend_step_attr(failed_prepare, SUSPEND_PREPARE); +suspend_step_attr(failed_suspend, SUSPEND_SUSPEND); +suspend_step_attr(failed_suspend_late, SUSPEND_SUSPEND_LATE); +suspend_step_attr(failed_suspend_noirq, SUSPEND_SUSPEND_NOIRQ); +suspend_step_attr(failed_resume, SUSPEND_RESUME); +suspend_step_attr(failed_resume_early, SUSPEND_RESUME_EARLY); +suspend_step_attr(failed_resume_noirq, SUSPEND_RESUME_NOIRQ); + static ssize_t last_failed_dev_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -439,6 +449,7 @@ static const struct attribute_group suspend_attr_group = { static int suspend_stats_show(struct seq_file *s, void *unused) { int i, index, last_dev, last_errno, last_step; + enum suspend_stat_step step; last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1; last_dev %= REC_FAILED_NUM; @@ -446,22 +457,14 @@ static int suspend_stats_show(struct seq_file *s, void *unused) last_errno %= REC_FAILED_NUM; last_step = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; last_step %= REC_FAILED_NUM; - seq_printf(s, "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n" - "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n", - "success", suspend_stats.success, - "fail", suspend_stats.fail, - "failed_freeze", suspend_stats.failed_freeze, - "failed_prepare", suspend_stats.failed_prepare, - "failed_suspend", suspend_stats.failed_suspend, - "failed_suspend_late", - suspend_stats.failed_suspend_late, - "failed_suspend_noirq", - suspend_stats.failed_suspend_noirq, - "failed_resume", suspend_stats.failed_resume, - "failed_resume_early", - suspend_stats.failed_resume_early, - "failed_resume_noirq", - suspend_stats.failed_resume_noirq); + + seq_printf(s, "success: %d\nfail: %d\n", + suspend_stats.success, suspend_stats.fail); + + for (step = SUSPEND_FREEZE; step <= SUSPEND_NR_STEPS; step++) + seq_printf(s, "failed_%s: %u\n", suspend_step_names[step], + suspend_stats.step_failures[step-1]); + seq_printf(s, "failures:\n last_failed_dev:\t%-s\n", suspend_stats.failed_devs[last_dev]); for (i = 1; i < REC_FAILED_NUM; i++) { diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index fa3bf161d13f..07bde5bba49e 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -367,7 +367,6 @@ static int suspend_prepare(suspend_state_t state) if (!error) return 0; - suspend_stats.failed_freeze++; dpm_save_failed_step(SUSPEND_FREEZE); pm_notifier_call_chain(PM_POST_SUSPEND); Restore: From 2231f78d3e15e45abe534db1997bc6a2153dc01c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:13:14 +0100 Subject: [PATCH 0076/1038] PM: sleep: stats: Use unsigned int for success and failure counters Change the type of the "success" and "fail" fields in struct suspend_stats to unsigned int, because they cannot be negative. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- include/linux/suspend.h | 4 ++-- kernel/power/main.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 5e4c4d4aed95..216bae989535 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -56,8 +56,8 @@ enum suspend_stat_step { struct suspend_stats { unsigned int step_failures[SUSPEND_NR_STEPS]; - int success; - int fail; + unsigned int success; + unsigned int fail; #define REC_FAILED_NUM 2 int last_failed_dev; char failed_devs[REC_FAILED_NUM][40]; diff --git a/kernel/power/main.c b/kernel/power/main.c index d7a02105b183..d6b4a9258288 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -339,8 +339,8 @@ static ssize_t _name##_show(struct kobject *kobj, \ } \ static struct kobj_attribute _name = __ATTR_RO(_name) -suspend_attr(success, "%d\n"); -suspend_attr(fail, "%d\n"); +suspend_attr(success, "%u\n"); +suspend_attr(fail, "%u\n"); suspend_attr(last_hw_sleep, "%llu\n"); suspend_attr(total_hw_sleep, "%llu\n"); suspend_attr(max_hw_sleep, "%llu\n"); @@ -458,7 +458,7 @@ static int suspend_stats_show(struct seq_file *s, void *unused) last_step = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; last_step %= REC_FAILED_NUM; - seq_printf(s, "success: %d\nfail: %d\n", + seq_printf(s, "success: %u\nfail: %u\n", suspend_stats.success, suspend_stats.fail); for (step = SUSPEND_FREEZE; step <= SUSPEND_NR_STEPS; step++) From 9ff544fa5f94fe07f99a36d2138075b322067546 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:30:44 +0100 Subject: [PATCH 0077/1038] PM: sleep: stats: Define suspend_stats next to the code using it It is not necessary to define struct suspend_stats in a header file and the suspend_stats variable in the core device system-wide PM code. They both can be defined in kernel/power/main.c, next to the sysfs and debugfs code accessing suspend_stats, which can be static. Modify the code in question in accordance with the above observation and replace the static inline functions manipulating suspend_stats with regular ones defined in kernel/power/main.c. While at it, move the enum suspend_stat_step to the end of suspend.h which is a more suitable place for it. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 1 - include/linux/suspend.h | 71 ++++++++---------------------------- kernel/power/main.c | 76 ++++++++++++++++++++++++++++++++------- kernel/power/power.h | 2 ++ kernel/power/suspend.c | 7 +--- 5 files changed, 81 insertions(+), 76 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 896450503d0d..761b03b4edb1 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -60,7 +60,6 @@ static LIST_HEAD(dpm_suspended_list); static LIST_HEAD(dpm_late_early_list); static LIST_HEAD(dpm_noirq_list); -struct suspend_stats suspend_stats; static DEFINE_MUTEX(dpm_list_mtx); static pm_message_t pm_transition; diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 216bae989535..da6ebca3ff77 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -40,62 +40,6 @@ typedef int __bitwise suspend_state_t; #define PM_SUSPEND_MIN PM_SUSPEND_TO_IDLE #define PM_SUSPEND_MAX ((__force suspend_state_t) 4) -enum suspend_stat_step { - SUSPEND_WORKING = 0, - SUSPEND_FREEZE, - SUSPEND_PREPARE, - SUSPEND_SUSPEND, - SUSPEND_SUSPEND_LATE, - SUSPEND_SUSPEND_NOIRQ, - SUSPEND_RESUME_NOIRQ, - SUSPEND_RESUME_EARLY, - SUSPEND_RESUME -}; - -#define SUSPEND_NR_STEPS SUSPEND_RESUME - -struct suspend_stats { - unsigned int step_failures[SUSPEND_NR_STEPS]; - unsigned int success; - unsigned int fail; -#define REC_FAILED_NUM 2 - int last_failed_dev; - char failed_devs[REC_FAILED_NUM][40]; - int last_failed_errno; - int errno[REC_FAILED_NUM]; - int last_failed_step; - u64 last_hw_sleep; - u64 total_hw_sleep; - u64 max_hw_sleep; - enum suspend_stat_step failed_steps[REC_FAILED_NUM]; -}; - -extern struct suspend_stats suspend_stats; - -static inline void dpm_save_failed_dev(const char *name) -{ - strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], - name, - sizeof(suspend_stats.failed_devs[0])); - suspend_stats.last_failed_dev++; - suspend_stats.last_failed_dev %= REC_FAILED_NUM; -} - -static inline void dpm_save_failed_errno(int err) -{ - suspend_stats.errno[suspend_stats.last_failed_errno] = err; - suspend_stats.last_failed_errno++; - suspend_stats.last_failed_errno %= REC_FAILED_NUM; -} - -static inline void dpm_save_failed_step(enum suspend_stat_step step) -{ - suspend_stats.step_failures[step-1]++; - suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; - suspend_stats.last_failed_step++; - suspend_stats.last_failed_step %= REC_FAILED_NUM; -} - /** * struct platform_suspend_ops - Callbacks for managing platform dependent * system sleep states. @@ -623,4 +567,19 @@ static inline void queue_up_suspend_work(void) {} #endif /* !CONFIG_PM_AUTOSLEEP */ +enum suspend_stat_step { + SUSPEND_WORKING = 0, + SUSPEND_FREEZE, + SUSPEND_PREPARE, + SUSPEND_SUSPEND, + SUSPEND_SUSPEND_LATE, + SUSPEND_SUSPEND_NOIRQ, + SUSPEND_RESUME_NOIRQ, + SUSPEND_RESUME_EARLY, + SUSPEND_RESUME +}; + +void dpm_save_failed_dev(const char *name); +void dpm_save_failed_step(enum suspend_stat_step step); + #endif /* _LINUX_SUSPEND_H */ diff --git a/kernel/power/main.c b/kernel/power/main.c index d6b4a9258288..8c4bf5a54805 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -95,19 +95,6 @@ int unregister_pm_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(unregister_pm_notifier); -void pm_report_hw_sleep_time(u64 t) -{ - suspend_stats.last_hw_sleep = t; - suspend_stats.total_hw_sleep += t; -} -EXPORT_SYMBOL_GPL(pm_report_hw_sleep_time); - -void pm_report_max_hw_sleep(u64 t) -{ - suspend_stats.max_hw_sleep = t; -} -EXPORT_SYMBOL_GPL(pm_report_max_hw_sleep); - int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down) { int ret; @@ -319,6 +306,69 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, power_attr(pm_test); #endif /* CONFIG_PM_SLEEP_DEBUG */ +#define SUSPEND_NR_STEPS SUSPEND_RESUME +#define REC_FAILED_NUM 2 + +struct suspend_stats { + unsigned int step_failures[SUSPEND_NR_STEPS]; + unsigned int success; + unsigned int fail; + int last_failed_dev; + char failed_devs[REC_FAILED_NUM][40]; + int last_failed_errno; + int errno[REC_FAILED_NUM]; + int last_failed_step; + u64 last_hw_sleep; + u64 total_hw_sleep; + u64 max_hw_sleep; + enum suspend_stat_step failed_steps[REC_FAILED_NUM]; +}; + +static struct suspend_stats suspend_stats; + +void dpm_save_failed_dev(const char *name) +{ + strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], + name, sizeof(suspend_stats.failed_devs[0])); + suspend_stats.last_failed_dev++; + suspend_stats.last_failed_dev %= REC_FAILED_NUM; +} + +void dpm_save_failed_step(enum suspend_stat_step step) +{ + suspend_stats.step_failures[step-1]++; + suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; + suspend_stats.last_failed_step++; + suspend_stats.last_failed_step %= REC_FAILED_NUM; +} + +void dpm_save_errno(int err) +{ + if (!err) { + suspend_stats.success++; + return; + } + + suspend_stats.fail++; + + suspend_stats.errno[suspend_stats.last_failed_errno] = err; + suspend_stats.last_failed_errno++; + suspend_stats.last_failed_errno %= REC_FAILED_NUM; +} + +void pm_report_hw_sleep_time(u64 t) +{ + suspend_stats.last_hw_sleep = t; + suspend_stats.total_hw_sleep += t; +} +EXPORT_SYMBOL_GPL(pm_report_hw_sleep_time); + +void pm_report_max_hw_sleep(u64 t) +{ + suspend_stats.max_hw_sleep = t; +} +EXPORT_SYMBOL_GPL(pm_report_max_hw_sleep); + static const char * const suspend_step_names[] = { [SUSPEND_WORKING] = "", [SUSPEND_FREEZE] = "freeze", diff --git a/kernel/power/power.h b/kernel/power/power.h index 8499a39c62f4..4e03046b9c4d 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -327,3 +327,5 @@ static inline void pm_sleep_enable_secondary_cpus(void) suspend_enable_secondary_cpus(); cpuidle_resume(); } + +void dpm_save_errno(int err); diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 07bde5bba49e..742eb26618cc 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -616,12 +616,7 @@ int pm_suspend(suspend_state_t state) pr_info("suspend entry (%s)\n", mem_sleep_labels[state]); error = enter_state(state); - if (error) { - suspend_stats.fail++; - dpm_save_failed_errno(error); - } else { - suspend_stats.success++; - } + dpm_save_errno(error); pr_info("suspend exit\n"); return error; } From 4add3e72f0fca0c66a8c9de8f58997eb9a3e3320 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:24:04 +0100 Subject: [PATCH 0078/1038] PM: sleep: stats: Call dpm_save_failed_step() at most once per phase If the handling of two or more devices fails in one suspend-resume phase, it should be counted once in the statistics which is not guaranteed to happen during system-wide resume of devices due to the possible asynchronous execution of device callbacks. Address this by using the async_error static variable during system-wide device resume to indicate that there has been a device resume error and the given suspend-resume phase should be counted as failing. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 761b03b4edb1..e75769af5e3e 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -685,7 +685,7 @@ Out: TRACE_RESUME(error); if (error) { - dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); + async_error = error; dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); } @@ -705,6 +705,9 @@ static void dpm_noirq_resume_devices(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true); + + async_error = 0; + mutex_lock(&dpm_list_mtx); pm_transition = state; @@ -734,6 +737,9 @@ static void dpm_noirq_resume_devices(pm_message_t state) mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, "noirq"); + if (async_error) + dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); + trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false); } @@ -815,7 +821,7 @@ Out: complete_all(&dev->power.completion); if (error) { - dpm_save_failed_step(SUSPEND_RESUME_EARLY); + async_error = error; dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async early" : " early", error); } @@ -839,6 +845,9 @@ void dpm_resume_early(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume_early"), state.event, true); + + async_error = 0; + mutex_lock(&dpm_list_mtx); pm_transition = state; @@ -868,6 +877,9 @@ void dpm_resume_early(pm_message_t state) mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, "early"); + if (async_error) + dpm_save_failed_step(SUSPEND_RESUME_EARLY); + trace_suspend_resume(TPS("dpm_resume_early"), state.event, false); } @@ -971,7 +983,7 @@ static void device_resume(struct device *dev, pm_message_t state, bool async) TRACE_RESUME(error); if (error) { - dpm_save_failed_step(SUSPEND_RESUME); + async_error = error; dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async" : "", error); } @@ -1030,6 +1042,8 @@ void dpm_resume(pm_message_t state) mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, NULL); + if (async_error) + dpm_save_failed_step(SUSPEND_RESUME); cpufreq_resume(); devfreq_resume(); From a6d38e991dc4f1b4a86137177435660df53951c5 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:24:30 +0100 Subject: [PATCH 0079/1038] PM: sleep: stats: Use locking in dpm_save_failed_dev() Because dpm_save_failed_dev() may be called simultaneously by multiple failing device PM functions, the state of the suspend_stats fields updated by it may become inconsistent. Prevent that from happening by using a lock in dpm_save_failed_dev(). Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- kernel/power/main.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/power/main.c b/kernel/power/main.c index 8c4bf5a54805..a9e0693aaf69 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -325,13 +325,18 @@ struct suspend_stats { }; static struct suspend_stats suspend_stats; +static DEFINE_MUTEX(suspend_stats_lock); void dpm_save_failed_dev(const char *name) { + mutex_lock(&suspend_stats_lock); + strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], name, sizeof(suspend_stats.failed_devs[0])); suspend_stats.last_failed_dev++; suspend_stats.last_failed_dev %= REC_FAILED_NUM; + + mutex_unlock(&suspend_stats_lock); } void dpm_save_failed_step(enum suspend_stat_step step) From ac6f87aaa26fa88680db23051a5d2b13a08d13b8 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:25:48 +0100 Subject: [PATCH 0080/1038] PM: sleep: stats: Log errors right after running suspend callbacks The error logging and failure statistics updates are carried out in two places in each system-wide device suspend phase, which is unnecessary code duplication, so do that in one place in each phase, right after invoking device suspend callbacks. While at it, add "noirq" or "late" to the "async" string printed when the failing device callback in the "noirq" or "late" suspend phase, respectively, was run asynchronously. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 49 +++++++++++---------------------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index e75769af5e3e..892d706af353 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1244,6 +1244,8 @@ Run: error = dpm_run_callback(callback, dev, state, info); if (error) { async_error = error; + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); goto Complete; } @@ -1273,14 +1275,8 @@ Complete: static void async_suspend_noirq(void *data, async_cookie_t cookie) { struct device *dev = data; - int error; - - error = __device_suspend_noirq(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + __device_suspend_noirq(dev, pm_transition, true); put_device(dev); } @@ -1312,12 +1308,8 @@ static int dpm_noirq_suspend_devices(pm_message_t state) mutex_lock(&dpm_list_mtx); - if (error) { - pm_dev_err(dev, state, " noirq", error); - dpm_save_failed_dev(dev_name(dev)); - } else if (!list_empty(&dev->power.entry)) { + if (!error && !list_empty(&dev->power.entry)) list_move(&dev->power.entry, &dpm_noirq_list); - } mutex_unlock(&dpm_list_mtx); @@ -1437,6 +1429,8 @@ Run: error = dpm_run_callback(callback, dev, state, info); if (error) { async_error = error; + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async late" : " late", error); goto Complete; } dpm_propagate_wakeup_to_parent(dev); @@ -1453,13 +1447,8 @@ Complete: static void async_suspend_late(void *data, async_cookie_t cookie) { struct device *dev = data; - int error; - error = __device_suspend_late(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + __device_suspend_late(dev, pm_transition, true); put_device(dev); } @@ -1500,11 +1489,6 @@ int dpm_suspend_late(pm_message_t state) if (!list_empty(&dev->power.entry)) list_move(&dev->power.entry, &dpm_late_early_list); - if (error) { - pm_dev_err(dev, state, " late", error); - dpm_save_failed_dev(dev_name(dev)); - } - mutex_unlock(&dpm_list_mtx); put_device(dev); @@ -1719,8 +1703,11 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) dpm_watchdog_clear(&wd); Complete: - if (error) + if (error) { async_error = error; + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async" : "", error); + } complete_all(&dev->power.completion); TRACE_SUSPEND(error); @@ -1730,14 +1717,8 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) static void async_suspend(void *data, async_cookie_t cookie) { struct device *dev = data; - int error; - - error = __device_suspend(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + __device_suspend(dev, pm_transition, true); put_device(dev); } @@ -1778,12 +1759,8 @@ int dpm_suspend(pm_message_t state) mutex_lock(&dpm_list_mtx); - if (error) { - pm_dev_err(dev, state, "", error); - dpm_save_failed_dev(dev_name(dev)); - } else if (!list_empty(&dev->power.entry)) { + if (!error && !list_empty(&dev->power.entry)) list_move(&dev->power.entry, &dpm_suspended_list); - } mutex_unlock(&dpm_list_mtx); From a4b64b893428734fc872a162ca7d92aa1ca1c429 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:27:33 +0100 Subject: [PATCH 0081/1038] PM: sleep: Move some assignments from under a lock The async_error and pm_transition variables are set under dpm_list_mtx in multiple places in the system-wide device PM core code, which is unnecessary and confusing, so rearrange the code so that the variables in question are set before acquiring the lock. While at it, add some empty code lines around locking to improve the consistency of the code. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 892d706af353..89a3ddd642c9 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -707,9 +707,9 @@ static void dpm_noirq_resume_devices(pm_message_t state) trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true); async_error = 0; + pm_transition = state; mutex_lock(&dpm_list_mtx); - pm_transition = state; /* * Trigger the resume of "async" devices upfront so they don't have to @@ -847,9 +847,9 @@ void dpm_resume_early(pm_message_t state) trace_suspend_resume(TPS("dpm_resume_early"), state.event, true); async_error = 0; + pm_transition = state; mutex_lock(&dpm_list_mtx); - pm_transition = state; /* * Trigger the resume of "async" devices upfront so they don't have to @@ -1012,10 +1012,11 @@ void dpm_resume(pm_message_t state) trace_suspend_resume(TPS("dpm_resume"), state.event, true); might_sleep(); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; + mutex_lock(&dpm_list_mtx); + /* * Trigger the resume of "async" devices upfront so they don't have to * wait for the "non-async" ones they don't depend on. @@ -1294,10 +1295,12 @@ static int dpm_noirq_suspend_devices(pm_message_t state) int error = 0; trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true); - mutex_lock(&dpm_list_mtx); + pm_transition = state; async_error = 0; + mutex_lock(&dpm_list_mtx); + while (!list_empty(&dpm_late_early_list)) { struct device *dev = to_device(dpm_late_early_list.prev); @@ -1320,7 +1323,9 @@ static int dpm_noirq_suspend_devices(pm_message_t state) if (error || async_error) break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); if (!error) error = async_error; @@ -1470,11 +1475,14 @@ int dpm_suspend_late(pm_message_t state) int error = 0; trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true); - wake_up_all_idle_cpus(); - mutex_lock(&dpm_list_mtx); + pm_transition = state; async_error = 0; + wake_up_all_idle_cpus(); + + mutex_lock(&dpm_list_mtx); + while (!list_empty(&dpm_suspended_list)) { struct device *dev = to_device(dpm_suspended_list.prev); @@ -1498,7 +1506,9 @@ int dpm_suspend_late(pm_message_t state) if (error || async_error) break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); if (!error) error = async_error; @@ -1745,9 +1755,11 @@ int dpm_suspend(pm_message_t state) devfreq_suspend(); cpufreq_suspend(); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; + + mutex_lock(&dpm_list_mtx); + while (!list_empty(&dpm_prepared_list)) { struct device *dev = to_device(dpm_prepared_list.prev); @@ -1771,7 +1783,9 @@ int dpm_suspend(pm_message_t state) if (error || async_error) break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); if (!error) error = async_error; From 96db0f947a14df754586ac38bc0d50ab8dae013c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:28:37 +0100 Subject: [PATCH 0082/1038] PM: sleep: Move devices to new lists earlier in each suspend phase During a system-wide suspend of devices, dpm_noirq_suspend_devices(), dpm_suspend_late() and dpm_suspend() move devices from one list to another. They do it with each device after its PM callback in the given suspend phase has run or has been scheduled for asynchronous execution, in case it is deleted from the current list in the meantime. However, devices can be moved to a new list before invoking their PM callbacks (which usually is the case for the devices whose callbacks are executed asynchronously anyway), because doing so does not affect the ordering of that list. In either case, each device is moved to the new list after the previous device has been moved to it or gone away, and if a device is removed, it does not matter which list it is in at that point, because deleting an entry from a list does not change the ordering of the other entries in it. Accordingly, modify the functions mentioned above to move devices to new lists without waiting for their PM callbacks to run regardless of whether or not they run asynchronously. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 89a3ddd642c9..bcd043c1d385 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1304,18 +1304,12 @@ static int dpm_noirq_suspend_devices(pm_message_t state) while (!list_empty(&dpm_late_early_list)) { struct device *dev = to_device(dpm_late_early_list.prev); + list_move(&dev->power.entry, &dpm_noirq_list); get_device(dev); mutex_unlock(&dpm_list_mtx); error = device_suspend_noirq(dev); - mutex_lock(&dpm_list_mtx); - - if (!error && !list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_noirq_list); - - mutex_unlock(&dpm_list_mtx); - put_device(dev); mutex_lock(&dpm_list_mtx); @@ -1486,19 +1480,13 @@ int dpm_suspend_late(pm_message_t state) while (!list_empty(&dpm_suspended_list)) { struct device *dev = to_device(dpm_suspended_list.prev); + list_move(&dev->power.entry, &dpm_late_early_list); get_device(dev); mutex_unlock(&dpm_list_mtx); error = device_suspend_late(dev); - mutex_lock(&dpm_list_mtx); - - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_late_early_list); - - mutex_unlock(&dpm_list_mtx); - put_device(dev); mutex_lock(&dpm_list_mtx); @@ -1763,19 +1751,13 @@ int dpm_suspend(pm_message_t state) while (!list_empty(&dpm_prepared_list)) { struct device *dev = to_device(dpm_prepared_list.prev); + list_move(&dev->power.entry, &dpm_suspended_list); get_device(dev); mutex_unlock(&dpm_list_mtx); error = device_suspend(dev); - mutex_lock(&dpm_list_mtx); - - if (!error && !list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_suspended_list); - - mutex_unlock(&dpm_list_mtx); - put_device(dev); mutex_lock(&dpm_list_mtx); From 86205785443bd6eff65152a24501ca58f301ab41 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 29 Jan 2024 17:29:41 +0100 Subject: [PATCH 0083/1038] PM: sleep: Call dpm_async_fn() directly in each suspend phase Simplify the system-wide suspend of devices by invoking dpm_async_fn() directly from the main loop in each suspend phase instead of using an additional wrapper function for running it. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Ulf Hansson --- drivers/base/power/main.c | 61 ++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index bcd043c1d385..5679f966f676 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1192,7 +1192,7 @@ static void dpm_superior_set_must_resume(struct device *dev) } /** - * __device_suspend_noirq - Execute a "noirq suspend" callback for given device. + * device_suspend_noirq - Execute a "noirq suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. @@ -1200,7 +1200,7 @@ static void dpm_superior_set_must_resume(struct device *dev) * The driver of @dev will not receive interrupts while this function is being * executed. */ -static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async) +static int device_suspend_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1277,18 +1277,10 @@ static void async_suspend_noirq(void *data, async_cookie_t cookie) { struct device *dev = data; - __device_suspend_noirq(dev, pm_transition, true); + device_suspend_noirq(dev, pm_transition, true); put_device(dev); } -static int device_suspend_noirq(struct device *dev) -{ - if (dpm_async_fn(dev, async_suspend_noirq)) - return 0; - - return __device_suspend_noirq(dev, pm_transition, false); -} - static int dpm_noirq_suspend_devices(pm_message_t state) { ktime_t starttime = ktime_get(); @@ -1305,10 +1297,15 @@ static int dpm_noirq_suspend_devices(pm_message_t state) struct device *dev = to_device(dpm_late_early_list.prev); list_move(&dev->power.entry, &dpm_noirq_list); + + if (dpm_async_fn(dev, async_suspend_noirq)) + continue; + get_device(dev); + mutex_unlock(&dpm_list_mtx); - error = device_suspend_noirq(dev); + error = device_suspend_noirq(dev, state, false); put_device(dev); @@ -1369,14 +1366,14 @@ static void dpm_propagate_wakeup_to_parent(struct device *dev) } /** - * __device_suspend_late - Execute a "late suspend" callback for given device. + * device_suspend_late - Execute a "late suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. * * Runtime PM is disabled for @dev while this function is being executed. */ -static int __device_suspend_late(struct device *dev, pm_message_t state, bool async) +static int device_suspend_late(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1447,18 +1444,10 @@ static void async_suspend_late(void *data, async_cookie_t cookie) { struct device *dev = data; - __device_suspend_late(dev, pm_transition, true); + device_suspend_late(dev, pm_transition, true); put_device(dev); } -static int device_suspend_late(struct device *dev) -{ - if (dpm_async_fn(dev, async_suspend_late)) - return 0; - - return __device_suspend_late(dev, pm_transition, false); -} - /** * dpm_suspend_late - Execute "late suspend" callbacks for all devices. * @state: PM transition of the system being carried out. @@ -1481,11 +1470,15 @@ int dpm_suspend_late(pm_message_t state) struct device *dev = to_device(dpm_suspended_list.prev); list_move(&dev->power.entry, &dpm_late_early_list); + + if (dpm_async_fn(dev, async_suspend_late)) + continue; + get_device(dev); mutex_unlock(&dpm_list_mtx); - error = device_suspend_late(dev); + error = device_suspend_late(dev, state, false); put_device(dev); @@ -1582,12 +1575,12 @@ static void dpm_clear_superiors_direct_complete(struct device *dev) } /** - * __device_suspend - Execute "suspend" callbacks for given device. + * device_suspend - Execute "suspend" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. */ -static int __device_suspend(struct device *dev, pm_message_t state, bool async) +static int device_suspend(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1716,18 +1709,10 @@ static void async_suspend(void *data, async_cookie_t cookie) { struct device *dev = data; - __device_suspend(dev, pm_transition, true); + device_suspend(dev, pm_transition, true); put_device(dev); } -static int device_suspend(struct device *dev) -{ - if (dpm_async_fn(dev, async_suspend)) - return 0; - - return __device_suspend(dev, pm_transition, false); -} - /** * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices. * @state: PM transition of the system being carried out. @@ -1752,11 +1737,15 @@ int dpm_suspend(pm_message_t state) struct device *dev = to_device(dpm_prepared_list.prev); list_move(&dev->power.entry, &dpm_suspended_list); + + if (dpm_async_fn(dev, async_suspend)) + continue; + get_device(dev); mutex_unlock(&dpm_list_mtx); - error = device_suspend(dev); + error = device_suspend(dev, state, false); put_device(dev); From 89a807625f9701154167bf6bf136adfa1be4d849 Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 22 Jan 2024 18:45:25 +0530 Subject: [PATCH 0084/1038] PM: hibernate: Rename lzo* to make it generic Renaming lzo* to generic names, except for lzo_xxx() APIs. This is used in the next patch where we move to crypto based APIs for compression. There are no functional changes introduced by this approach. Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki --- kernel/power/swap.c | 120 ++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 6053ddddaf65..35c62f91c13b 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -515,23 +515,23 @@ static int swap_writer_finish(struct swap_map_handle *handle, } /* We need to remember how much compressed data we need to read. */ -#define LZO_HEADER sizeof(size_t) +#define CMP_HEADER sizeof(size_t) /* Number of pages/bytes we'll compress at one time. */ -#define LZO_UNC_PAGES 32 -#define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE) +#define UNC_PAGES 32 +#define UNC_SIZE (UNC_PAGES * PAGE_SIZE) -/* Number of pages/bytes we need for compressed data (worst case). */ -#define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \ - LZO_HEADER, PAGE_SIZE) -#define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE) +/* Number of pages we need for compressed data (worst case). */ +#define CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(UNC_SIZE) + \ + CMP_HEADER, PAGE_SIZE) +#define CMP_SIZE (CMP_PAGES * PAGE_SIZE) /* Maximum number of threads for compression/decompression. */ -#define LZO_THREADS 3 +#define CMP_THREADS 3 /* Minimum/maximum number of pages for read buffering. */ -#define LZO_MIN_RD_PAGES 1024 -#define LZO_MAX_RD_PAGES 8192 +#define CMP_MIN_RD_PAGES 1024 +#define CMP_MAX_RD_PAGES 8192 /** @@ -593,8 +593,8 @@ struct crc_data { wait_queue_head_t go; /* start crc update */ wait_queue_head_t done; /* crc update done */ u32 *crc32; /* points to handle's crc32 */ - size_t *unc_len[LZO_THREADS]; /* uncompressed lengths */ - unsigned char *unc[LZO_THREADS]; /* uncompressed data */ + size_t *unc_len[CMP_THREADS]; /* uncompressed lengths */ + unsigned char *unc[CMP_THREADS]; /* uncompressed data */ }; /* @@ -625,7 +625,7 @@ static int crc32_threadfn(void *data) return 0; } /* - * Structure used for LZO data compression. + * Structure used for data compression. */ struct cmp_data { struct task_struct *thr; /* thread */ @@ -636,15 +636,15 @@ struct cmp_data { wait_queue_head_t done; /* compression done */ size_t unc_len; /* uncompressed length */ size_t cmp_len; /* compressed length */ - unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */ - unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ + unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ + unsigned char cmp[CMP_SIZE]; /* compressed buffer */ unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */ }; /* * Compression function that runs in its own thread. */ -static int lzo_compress_threadfn(void *data) +static int compress_threadfn(void *data) { struct cmp_data *d = data; @@ -661,7 +661,7 @@ static int lzo_compress_threadfn(void *data) atomic_set(&d->ready, 0); d->ret = lzo1x_1_compress(d->unc, d->unc_len, - d->cmp + LZO_HEADER, &d->cmp_len, + d->cmp + CMP_HEADER, &d->cmp_len, d->wrk); atomic_set_release(&d->stop, 1); wake_up(&d->done); @@ -670,14 +670,14 @@ static int lzo_compress_threadfn(void *data) } /** - * save_image_lzo - Save the suspend image data compressed with LZO. + * save_compressed_image - Save the suspend image data after compression. * @handle: Swap map handle to use for saving the image. * @snapshot: Image to read data from. * @nr_to_write: Number of pages to save. */ -static int save_image_lzo(struct swap_map_handle *handle, - struct snapshot_handle *snapshot, - unsigned int nr_to_write) +static int save_compressed_image(struct swap_map_handle *handle, + struct snapshot_handle *snapshot, + unsigned int nr_to_write) { unsigned int m; int ret = 0; @@ -699,18 +699,18 @@ static int save_image_lzo(struct swap_map_handle *handle, * footprint. */ nr_threads = num_online_cpus() - 1; - nr_threads = clamp_val(nr_threads, 1, LZO_THREADS); + nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); if (!page) { - pr_err("Failed to allocate LZO page\n"); + pr_err("Failed to allocate compression page\n"); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate LZO data\n"); + pr_err("Failed to allocate compression data\n"); ret = -ENOMEM; goto out_clean; } @@ -729,7 +729,7 @@ static int save_image_lzo(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); - data[thr].thr = kthread_run(lzo_compress_threadfn, + data[thr].thr = kthread_run(compress_threadfn, &data[thr], "image_compress/%u", thr); if (IS_ERR(data[thr].thr)) { @@ -777,7 +777,7 @@ static int save_image_lzo(struct swap_map_handle *handle, start = ktime_get(); for (;;) { for (thr = 0; thr < nr_threads; thr++) { - for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) { + for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) { ret = snapshot_read_next(snapshot); if (ret < 0) goto out_finish; @@ -817,14 +817,14 @@ static int save_image_lzo(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("LZO compression failed\n"); + pr_err("compression failed\n"); goto out_finish; } if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > lzo1x_worst_compress(data[thr].unc_len))) { - pr_err("Invalid LZO compressed length\n"); + pr_err("Invalid compressed length\n"); ret = -1; goto out_finish; } @@ -840,7 +840,7 @@ static int save_image_lzo(struct swap_map_handle *handle, * read it. */ for (off = 0; - off < LZO_HEADER + data[thr].cmp_len; + off < CMP_HEADER + data[thr].cmp_len; off += PAGE_SIZE) { memcpy(page, data[thr].cmp + off, PAGE_SIZE); @@ -942,7 +942,7 @@ int swsusp_write(unsigned int flags) if (!error) { error = (flags & SF_NOCOMPRESS_MODE) ? save_image(&handle, &snapshot, pages - 1) : - save_image_lzo(&handle, &snapshot, pages - 1); + save_compressed_image(&handle, &snapshot, pages - 1); } out_finish: error = swap_writer_finish(&handle, flags, error); @@ -1109,7 +1109,7 @@ static int load_image(struct swap_map_handle *handle, } /* - * Structure used for LZO data decompression. + * Structure used for data decompression. */ struct dec_data { struct task_struct *thr; /* thread */ @@ -1120,14 +1120,14 @@ struct dec_data { wait_queue_head_t done; /* decompression done */ size_t unc_len; /* uncompressed length */ size_t cmp_len; /* compressed length */ - unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */ - unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ + unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ + unsigned char cmp[CMP_SIZE]; /* compressed buffer */ }; /* * Decompression function that runs in its own thread. */ -static int lzo_decompress_threadfn(void *data) +static int decompress_threadfn(void *data) { struct dec_data *d = data; @@ -1143,9 +1143,9 @@ static int lzo_decompress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->unc_len = LZO_UNC_SIZE; - d->ret = lzo1x_decompress_safe(d->cmp + LZO_HEADER, d->cmp_len, - d->unc, &d->unc_len); + d->unc_len = UNC_SIZE; + d->ret = lzo1x_decompress_safe(d->cmp + CMP_HEADER, d->cmp_len, + d->unc, &d->unc_len); if (clean_pages_on_decompress) flush_icache_range((unsigned long)d->unc, (unsigned long)d->unc + d->unc_len); @@ -1157,14 +1157,14 @@ static int lzo_decompress_threadfn(void *data) } /** - * load_image_lzo - Load compressed image data and decompress them with LZO. + * load_compressed_image - Load compressed image data and decompress it. * @handle: Swap map handle to use for loading data. * @snapshot: Image to copy uncompressed data into. * @nr_to_read: Number of pages to load. */ -static int load_image_lzo(struct swap_map_handle *handle, - struct snapshot_handle *snapshot, - unsigned int nr_to_read) +static int load_compressed_image(struct swap_map_handle *handle, + struct snapshot_handle *snapshot, + unsigned int nr_to_read) { unsigned int m; int ret = 0; @@ -1189,18 +1189,18 @@ static int load_image_lzo(struct swap_map_handle *handle, * footprint. */ nr_threads = num_online_cpus() - 1; - nr_threads = clamp_val(nr_threads, 1, LZO_THREADS); + nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); - page = vmalloc(array_size(LZO_MAX_RD_PAGES, sizeof(*page))); + page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page))); if (!page) { - pr_err("Failed to allocate LZO page\n"); + pr_err("Failed to allocate compression page\n"); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate LZO data\n"); + pr_err("Failed to allocate compression data\n"); ret = -ENOMEM; goto out_clean; } @@ -1221,7 +1221,7 @@ static int load_image_lzo(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); - data[thr].thr = kthread_run(lzo_decompress_threadfn, + data[thr].thr = kthread_run(decompress_threadfn, &data[thr], "image_decompress/%u", thr); if (IS_ERR(data[thr].thr)) { @@ -1262,18 +1262,18 @@ static int load_image_lzo(struct swap_map_handle *handle, */ if (low_free_pages() > snapshot_get_image_size()) read_pages = (low_free_pages() - snapshot_get_image_size()) / 2; - read_pages = clamp_val(read_pages, LZO_MIN_RD_PAGES, LZO_MAX_RD_PAGES); + read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES); for (i = 0; i < read_pages; i++) { - page[i] = (void *)__get_free_page(i < LZO_CMP_PAGES ? + page[i] = (void *)__get_free_page(i < CMP_PAGES ? GFP_NOIO | __GFP_HIGH : GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); if (!page[i]) { - if (i < LZO_CMP_PAGES) { + if (i < CMP_PAGES) { ring_size = i; - pr_err("Failed to allocate LZO pages\n"); + pr_err("Failed to allocate compression pages\n"); ret = -ENOMEM; goto out_clean; } else { @@ -1344,13 +1344,13 @@ static int load_image_lzo(struct swap_map_handle *handle, data[thr].cmp_len = *(size_t *)page[pg]; if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(LZO_UNC_SIZE))) { - pr_err("Invalid LZO compressed length\n"); + lzo1x_worst_compress(UNC_SIZE))) { + pr_err("Invalid compressed length\n"); ret = -1; goto out_finish; } - need = DIV_ROUND_UP(data[thr].cmp_len + LZO_HEADER, + need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER, PAGE_SIZE); if (need > have) { if (eof > 1) { @@ -1361,7 +1361,7 @@ static int load_image_lzo(struct swap_map_handle *handle, } for (off = 0; - off < LZO_HEADER + data[thr].cmp_len; + off < CMP_HEADER + data[thr].cmp_len; off += PAGE_SIZE) { memcpy(data[thr].cmp + off, page[pg], PAGE_SIZE); @@ -1378,7 +1378,7 @@ static int load_image_lzo(struct swap_map_handle *handle, /* * Wait for more data while we are decompressing. */ - if (have < LZO_CMP_PAGES && asked) { + if (have < CMP_PAGES && asked) { ret = hib_wait_io(&hb); if (ret) goto out_finish; @@ -1396,14 +1396,14 @@ static int load_image_lzo(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("LZO decompression failed\n"); + pr_err("decompression failed\n"); goto out_finish; } if (unlikely(!data[thr].unc_len || - data[thr].unc_len > LZO_UNC_SIZE || - data[thr].unc_len & (PAGE_SIZE - 1))) { - pr_err("Invalid LZO uncompressed length\n"); + data[thr].unc_len > UNC_SIZE || + data[thr].unc_len & (PAGE_SIZE - 1))) { + pr_err("Invalid uncompressed length\n"); ret = -1; goto out_finish; } @@ -1500,7 +1500,7 @@ int swsusp_read(unsigned int *flags_p) if (!error) { error = (*flags_p & SF_NOCOMPRESS_MODE) ? load_image(&handle, &snapshot, header->pages - 1) : - load_image_lzo(&handle, &snapshot, header->pages - 1); + load_compressed_image(&handle, &snapshot, header->pages - 1); } swap_reader_finish(&handle); end: From a06c6f5d3cc90b3b070d7b99979d57238db77a86 Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 22 Jan 2024 18:45:26 +0530 Subject: [PATCH 0085/1038] PM: hibernate: Move to crypto APIs for LZO compression Currently for hibernation, LZO is the only compression algorithm available and uses the existing LZO library calls. However, there is no flexibility to switch to other algorithms which provides better results. The main idea is that different compression algorithms have different characteristics and hibernation may benefit when it uses alternate algorithms. By moving to crypto based APIs, it lays a foundation to use other compression algorithms for hibernation. There are no functional changes introduced by this approach. Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki --- kernel/power/Kconfig | 21 +++++++- kernel/power/hibernate.c | 33 +++++++++++++ kernel/power/power.h | 5 ++ kernel/power/swap.c | 101 +++++++++++++++++++++++++++++---------- 4 files changed, 132 insertions(+), 28 deletions(-) diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 4b31629c5be4..d4167159bae8 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -39,9 +39,9 @@ config HIBERNATION bool "Hibernation (aka 'suspend to disk')" depends on SWAP && ARCH_HIBERNATION_POSSIBLE select HIBERNATE_CALLBACKS - select LZO_COMPRESS - select LZO_DECOMPRESS select CRC32 + select CRYPTO + select CRYPTO_LZO help Enable the suspend to disk (STD) functionality, which is usually called "hibernation" in user interfaces. STD checkpoints the @@ -92,6 +92,23 @@ config HIBERNATION_SNAPSHOT_DEV If in doubt, say Y. +choice + prompt "Default compressor" + default HIBERNATION_COMP_LZO + depends on HIBERNATION + +config HIBERNATION_COMP_LZO + bool "lzo" + depends on CRYPTO_LZO + +endchoice + +config HIBERNATION_DEF_COMP + string + default "lzo" if HIBERNATION_COMP_LZO + help + Default compressor to be used for hibernation. + config PM_STD_PARTITION string "Default resume partition" depends on HIBERNATION diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 4b0b7cf2e019..76b7ff619c90 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -47,6 +47,15 @@ dev_t swsusp_resume_device; sector_t swsusp_resume_block; __visible int in_suspend __nosavedata; +static const char *default_compressor = CONFIG_HIBERNATION_DEF_COMP; + +/* + * Compression/decompression algorithm to be used while saving/loading + * image to/from disk. This would later be used in 'kernel/power/swap.c' + * to allocate comp streams. + */ +char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; + enum { HIBERNATION_INVALID, HIBERNATION_PLATFORM, @@ -732,6 +741,17 @@ int hibernate(void) return -EPERM; } + /* + * Query for the compression algorithm support if compression is enabled. + */ + if (!nocompress) { + strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { + pr_err("%s compression is not available\n", hib_comp_algo); + return -EOPNOTSUPP; + } + } + sleep_flags = lock_system_sleep(); /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { @@ -955,6 +975,19 @@ static int software_resume(void) if (error) goto Unlock; + /* + * Check if the hibernation image is compressed. If so, query for + * the algorithm support. + */ + if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) { + strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { + pr_err("%s compression is not available\n", hib_comp_algo); + error = -EOPNOTSUPP; + goto Unlock; + } + } + /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { error = -EBUSY; diff --git a/kernel/power/power.h b/kernel/power/power.h index 4e03046b9c4d..5efa2c987057 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -6,6 +6,7 @@ #include #include #include +#include struct swsusp_info { struct new_utsname uts; @@ -54,6 +55,10 @@ asmlinkage int swsusp_save(void); /* kernel/power/hibernate.c */ extern bool freezer_test_done; +extern char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; + +/* kernel/power/swap.c */ +extern unsigned int swsusp_header_flags; extern int hibernation_snapshot(int platform_mode); extern int hibernation_restore(int platform_mode); diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 35c62f91c13b..6513035f2f7f 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -339,6 +338,13 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) return error; } +/* + * Hold the swsusp_header flag. This is used in software_resume() in + * 'kernel/power/hibernate' to check if the image is compressed and query + * for the compression algorithm support(if so). + */ +unsigned int swsusp_header_flags; + /** * swsusp_swap_check - check if the resume device is a swap device * and get its index (if so) @@ -514,6 +520,12 @@ static int swap_writer_finish(struct swap_map_handle *handle, return error; } +/* + * Bytes we need for compressed data in worst case. We assume(limitation) + * this is the worst of all the compression algorithms. + */ +#define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2) + /* We need to remember how much compressed data we need to read. */ #define CMP_HEADER sizeof(size_t) @@ -522,7 +534,7 @@ static int swap_writer_finish(struct swap_map_handle *handle, #define UNC_SIZE (UNC_PAGES * PAGE_SIZE) /* Number of pages we need for compressed data (worst case). */ -#define CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(UNC_SIZE) + \ +#define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \ CMP_HEADER, PAGE_SIZE) #define CMP_SIZE (CMP_PAGES * PAGE_SIZE) @@ -533,7 +545,6 @@ static int swap_writer_finish(struct swap_map_handle *handle, #define CMP_MIN_RD_PAGES 1024 #define CMP_MAX_RD_PAGES 8192 - /** * save_image - save the suspend image data */ @@ -629,6 +640,7 @@ static int crc32_threadfn(void *data) */ struct cmp_data { struct task_struct *thr; /* thread */ + struct crypto_comp *cc; /* crypto compressor stream */ atomic_t ready; /* ready to start flag */ atomic_t stop; /* ready to stop flag */ int ret; /* return code */ @@ -638,15 +650,18 @@ struct cmp_data { size_t cmp_len; /* compressed length */ unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ unsigned char cmp[CMP_SIZE]; /* compressed buffer */ - unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */ }; +/* Indicates the image size after compression */ +static atomic_t compressed_size = ATOMIC_INIT(0); + /* * Compression function that runs in its own thread. */ static int compress_threadfn(void *data) { struct cmp_data *d = data; + unsigned int cmp_len = 0; while (1) { wait_event(d->go, atomic_read_acquire(&d->ready) || @@ -660,9 +675,13 @@ static int compress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->ret = lzo1x_1_compress(d->unc, d->unc_len, - d->cmp + CMP_HEADER, &d->cmp_len, - d->wrk); + cmp_len = CMP_SIZE - CMP_HEADER; + d->ret = crypto_comp_compress(d->cc, d->unc, d->unc_len, + d->cmp + CMP_HEADER, + &cmp_len); + d->cmp_len = cmp_len; + + atomic_set(&compressed_size, atomic_read(&compressed_size) + d->cmp_len); atomic_set_release(&d->stop, 1); wake_up(&d->done); } @@ -694,6 +713,8 @@ static int save_compressed_image(struct swap_map_handle *handle, hib_init_batch(&hb); + atomic_set(&compressed_size, 0); + /* * We'll limit the number of threads for compression to limit memory * footprint. @@ -703,14 +724,14 @@ static int save_compressed_image(struct swap_map_handle *handle, page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); if (!page) { - pr_err("Failed to allocate compression page\n"); + pr_err("Failed to allocate %s page\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate compression data\n"); + pr_err("Failed to allocate %s data\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } @@ -729,6 +750,13 @@ static int save_compressed_image(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); + data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); + if (IS_ERR_OR_NULL(data[thr].cc)) { + pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + ret = -EFAULT; + goto out_clean; + } + data[thr].thr = kthread_run(compress_threadfn, &data[thr], "image_compress/%u", thr); @@ -767,7 +795,7 @@ static int save_compressed_image(struct swap_map_handle *handle, */ handle->reqd_free_pages = reqd_free_pages(); - pr_info("Using %u thread(s) for compression\n", nr_threads); + pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo); pr_info("Compressing and saving image data (%u pages)...\n", nr_to_write); m = nr_to_write / 10; @@ -817,14 +845,14 @@ static int save_compressed_image(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("compression failed\n"); + pr_err("%s compression failed\n", hib_comp_algo); goto out_finish; } if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(data[thr].unc_len))) { - pr_err("Invalid compressed length\n"); + bytes_worst_compress(data[thr].unc_len))) { + pr_err("Invalid %s compressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -862,6 +890,9 @@ out_finish: if (!ret) pr_info("Image saving done\n"); swsusp_show_speed(start, stop, nr_to_write, "Wrote"); + pr_info("Image size after compression: %d kbytes\n", + (atomic_read(&compressed_size) / 1024)); + out_clean: hib_finish_batch(&hb); if (crc) { @@ -870,9 +901,12 @@ out_clean: kfree(crc); } if (data) { - for (thr = 0; thr < nr_threads; thr++) + for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); + if (data[thr].cc) + crypto_free_comp(data[thr].cc); + } vfree(data); } if (page) free_page((unsigned long)page); @@ -1113,6 +1147,7 @@ static int load_image(struct swap_map_handle *handle, */ struct dec_data { struct task_struct *thr; /* thread */ + struct crypto_comp *cc; /* crypto compressor stream */ atomic_t ready; /* ready to start flag */ atomic_t stop; /* ready to stop flag */ int ret; /* return code */ @@ -1130,6 +1165,7 @@ struct dec_data { static int decompress_threadfn(void *data) { struct dec_data *d = data; + unsigned int unc_len = 0; while (1) { wait_event(d->go, atomic_read_acquire(&d->ready) || @@ -1143,9 +1179,11 @@ static int decompress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->unc_len = UNC_SIZE; - d->ret = lzo1x_decompress_safe(d->cmp + CMP_HEADER, d->cmp_len, - d->unc, &d->unc_len); + unc_len = UNC_SIZE; + d->ret = crypto_comp_decompress(d->cc, d->cmp + CMP_HEADER, d->cmp_len, + d->unc, &unc_len); + d->unc_len = unc_len; + if (clean_pages_on_decompress) flush_icache_range((unsigned long)d->unc, (unsigned long)d->unc + d->unc_len); @@ -1193,14 +1231,14 @@ static int load_compressed_image(struct swap_map_handle *handle, page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page))); if (!page) { - pr_err("Failed to allocate compression page\n"); + pr_err("Failed to allocate %s page\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate compression data\n"); + pr_err("Failed to allocate %s data\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } @@ -1221,6 +1259,13 @@ static int load_compressed_image(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); + data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); + if (IS_ERR_OR_NULL(data[thr].cc)) { + pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + ret = -EFAULT; + goto out_clean; + } + data[thr].thr = kthread_run(decompress_threadfn, &data[thr], "image_decompress/%u", thr); @@ -1273,7 +1318,7 @@ static int load_compressed_image(struct swap_map_handle *handle, if (!page[i]) { if (i < CMP_PAGES) { ring_size = i; - pr_err("Failed to allocate compression pages\n"); + pr_err("Failed to allocate %s pages\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } else { @@ -1283,7 +1328,7 @@ static int load_compressed_image(struct swap_map_handle *handle, } want = ring_size = i; - pr_info("Using %u thread(s) for decompression\n", nr_threads); + pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo); pr_info("Loading and decompressing image data (%u pages)...\n", nr_to_read); m = nr_to_read / 10; @@ -1344,8 +1389,8 @@ static int load_compressed_image(struct swap_map_handle *handle, data[thr].cmp_len = *(size_t *)page[pg]; if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(UNC_SIZE))) { - pr_err("Invalid compressed length\n"); + bytes_worst_compress(UNC_SIZE))) { + pr_err("Invalid %s compressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -1396,14 +1441,14 @@ static int load_compressed_image(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("decompression failed\n"); + pr_err("%s decompression failed\n", hib_comp_algo); goto out_finish; } if (unlikely(!data[thr].unc_len || data[thr].unc_len > UNC_SIZE || data[thr].unc_len & (PAGE_SIZE - 1))) { - pr_err("Invalid uncompressed length\n"); + pr_err("Invalid %s uncompressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -1464,9 +1509,12 @@ out_clean: kfree(crc); } if (data) { - for (thr = 0; thr < nr_threads; thr++) + for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); + if (data[thr].cc) + crypto_free_comp(data[thr].cc); + } vfree(data); } vfree(page); @@ -1535,6 +1583,7 @@ int swsusp_check(bool exclusive) if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); + swsusp_header_flags = swsusp_header->flags; /* Reset swap signature now */ error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC, swsusp_resume_block, From 8bc29736357e7f9a6bd0d16b57b5612197e1924b Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 22 Jan 2024 18:45:27 +0530 Subject: [PATCH 0086/1038] PM: hibernate: Add support for LZ4 compression for hibernation Extend the support for LZ4 compression to be used with hibernation. The main idea is that different compression algorithms have different characteristics and hibernation may benefit when it uses any of these algorithms: a default algorithm, having higher compression rate but is slower(compression/decompression) and a secondary algorithm, that is faster(compression/decompression) but has lower compression rate. LZ4 algorithm has better decompression speeds over LZO. This reduces the hibernation image restore time. As per test results: LZO LZ4 Size before Compression(bytes) 682696704 682393600 Size after Compression(bytes) 146502402 155993547 Decompression Rate 335.02 MB/s 501.05 MB/s Restore time 4.4s 3.8s LZO is the default compression algorithm used for hibernation. Enable CONFIG_HIBERNATION_COMP_LZ4 to set the default compressor as LZ4. Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki --- kernel/power/Kconfig | 5 +++++ kernel/power/hibernate.c | 25 ++++++++++++++++++++++--- kernel/power/power.h | 14 ++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index d4167159bae8..afce8130d8b9 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -101,11 +101,16 @@ config HIBERNATION_COMP_LZO bool "lzo" depends on CRYPTO_LZO +config HIBERNATION_COMP_LZ4 + bool "lz4" + depends on CRYPTO_LZ4 + endchoice config HIBERNATION_DEF_COMP string default "lzo" if HIBERNATION_COMP_LZO + default "lz4" if HIBERNATION_COMP_LZ4 help Default compressor to be used for hibernation. diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 76b7ff619c90..219191d6d0e8 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -727,6 +727,9 @@ static int load_image_and_restore(void) return error; } +#define COMPRESSION_ALGO_LZO "lzo" +#define COMPRESSION_ALGO_LZ4 "lz4" + /** * hibernate - Carry out system hibernation, including saving the image. */ @@ -786,11 +789,24 @@ int hibernate(void) if (hibernation_mode == HIBERNATION_PLATFORM) flags |= SF_PLATFORM_MODE; - if (nocompress) + if (nocompress) { flags |= SF_NOCOMPRESS_MODE; - else + } else { flags |= SF_CRC32_MODE; + /* + * By default, LZO compression is enabled. Use SF_COMPRESSION_ALG_LZ4 + * to override this behaviour and use LZ4. + * + * Refer kernel/power/power.h for more details + */ + + if (!strcmp(hib_comp_algo, COMPRESSION_ALGO_LZ4)) + flags |= SF_COMPRESSION_ALG_LZ4; + else + flags |= SF_COMPRESSION_ALG_LZO; + } + pm_pr_dbg("Writing hibernation image.\n"); error = swsusp_write(flags); swsusp_free(); @@ -980,7 +996,10 @@ static int software_resume(void) * the algorithm support. */ if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) { - strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4) + strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4, sizeof(hib_comp_algo)); + else + strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { pr_err("%s compression is not available\n", hib_comp_algo); error = -EOPNOTSUPP; diff --git a/kernel/power/power.h b/kernel/power/power.h index 5efa2c987057..518349272848 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -167,11 +167,25 @@ extern int swsusp_swap_in_use(void); * Flags that can be passed from the hibernatig hernel to the "boot" kernel in * the image header. */ +#define SF_COMPRESSION_ALG_LZO 0 /* dummy, details given below */ #define SF_PLATFORM_MODE 1 #define SF_NOCOMPRESS_MODE 2 #define SF_CRC32_MODE 4 #define SF_HW_SIG 8 +/* + * Bit to indicate the compression algorithm to be used(for LZ4). The same + * could be checked while saving/loading image to/from disk to use the + * corresponding algorithms. + * + * By default, LZO compression is enabled if SF_CRC32_MODE is set. Use + * SF_COMPRESSION_ALG_LZ4 to override this behaviour and use LZ4. + * + * SF_CRC32_MODE, SF_COMPRESSION_ALG_LZO(dummy) -> Compression, LZO + * SF_CRC32_MODE, SF_COMPRESSION_ALG_LZ4 -> Compression, LZ4 + */ +#define SF_COMPRESSION_ALG_LZ4 16 + /* kernel/power/hibernate.c */ int swsusp_check(bool exclusive); extern void swsusp_free(void); From 0ec74ad3c157bd4bcbcc8b294777733687e8cd2a Mon Sep 17 00:00:00 2001 From: Jan Dakinevich Date: Fri, 26 Jan 2024 23:08:36 +0300 Subject: [PATCH 0087/1038] regmap: rework ->max_register handling When regmap consists of single register, 'regmap' subsystem is unable to understand whether ->max_register is set or not, because in both cases it is equal to zero. It leads to that the logic based on value of ->max_register doesn't work. For example using of REGCACHE_FLAT fails. This patch introduces an extra parameter to regmap config, indicating that zero value in ->max_register is authentic. Signed-off-by: Jan Dakinevich Link: https://lore.kernel.org/r/20240126200836.1829995-1-jan.dakinevich@salutedevices.com Signed-off-by: Mark Brown --- drivers/base/regmap/internal.h | 1 + drivers/base/regmap/regcache-flat.c | 2 +- drivers/base/regmap/regcache.c | 4 +++- drivers/base/regmap/regmap.c | 10 ++++++---- include/linux/regmap.h | 5 +++++ 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h index 583dd5d7d46b..bcdb25bec77c 100644 --- a/drivers/base/regmap/internal.h +++ b/drivers/base/regmap/internal.h @@ -93,6 +93,7 @@ struct regmap { #endif unsigned int max_register; + bool max_register_is_set; bool (*writeable_reg)(struct device *dev, unsigned int reg); bool (*readable_reg)(struct device *dev, unsigned int reg); bool (*volatile_reg)(struct device *dev, unsigned int reg); diff --git a/drivers/base/regmap/regcache-flat.c b/drivers/base/regmap/regcache-flat.c index b7e4b2464102..9b17c77dec9d 100644 --- a/drivers/base/regmap/regcache-flat.c +++ b/drivers/base/regmap/regcache-flat.c @@ -23,7 +23,7 @@ static int regcache_flat_init(struct regmap *map) int i; unsigned int *cache; - if (!map || map->reg_stride_order < 0 || !map->max_register) + if (!map || map->reg_stride_order < 0 || !map->max_register_is_set) return -EINVAL; map->cache = kcalloc(regcache_flat_get_index(map, map->max_register) diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index ac63a73ccdaa..2e41cb12b8e2 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -187,8 +187,10 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) return 0; } - if (!map->max_register && map->num_reg_defaults_raw) + if (!map->max_register_is_set && map->num_reg_defaults_raw) { map->max_register = (map->num_reg_defaults_raw - 1) * map->reg_stride; + map->max_register_is_set = true; + } if (map->cache_ops->init) { dev_dbg(map->dev, "Initializing %s cache\n", diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 6db77d8e45f9..5cb425f6f02d 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -89,7 +89,7 @@ EXPORT_SYMBOL_GPL(regmap_check_range_table); bool regmap_writeable(struct regmap *map, unsigned int reg) { - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->writeable_reg) @@ -112,7 +112,7 @@ bool regmap_cached(struct regmap *map, unsigned int reg) if (!map->cache_ops) return false; - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; map->lock(map->lock_arg); @@ -129,7 +129,7 @@ bool regmap_readable(struct regmap *map, unsigned int reg) if (!map->reg_read) return false; - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->format.format_write) @@ -787,6 +787,7 @@ struct regmap *__regmap_init(struct device *dev, map->bus = bus; map->bus_context = bus_context; map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->wr_table = config->wr_table; map->rd_table = config->rd_table; map->volatile_table = config->volatile_table; @@ -1412,6 +1413,7 @@ int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config) regmap_debugfs_exit(map); map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->writeable_reg = config->writeable_reg; map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; @@ -3383,7 +3385,7 @@ EXPORT_SYMBOL_GPL(regmap_get_val_bytes); */ int regmap_get_max_register(struct regmap *map) { - return map->max_register ? map->max_register : -EINVAL; + return map->max_register_is_set ? map->max_register : -EINVAL; } EXPORT_SYMBOL_GPL(regmap_get_max_register); diff --git a/include/linux/regmap.h b/include/linux/regmap.h index c9182a47736e..b743241cfb7c 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -332,6 +332,10 @@ typedef void (*regmap_unlock)(void *); * @io_port: Support IO port accessors. Makes sense only when MMIO vs. IO port * access can be distinguished. * @max_register: Optional, specifies the maximum valid register address. + * @max_register_is_0: Optional, specifies that zero value in @max_register + * should be taken into account. This is a workaround to + * apply handling of @max_register for regmap that contains + * only one register. * @wr_table: Optional, points to a struct regmap_access_table specifying * valid ranges for write access. * @rd_table: As above, for read access. @@ -422,6 +426,7 @@ struct regmap_config { bool io_port; unsigned int max_register; + bool max_register_is_0; const struct regmap_access_table *wr_table; const struct regmap_access_table *rd_table; const struct regmap_access_table *volatile_table; From f156743c526281ddcc19511e9073f8c987506913 Mon Sep 17 00:00:00 2001 From: "andy.shevchenko@gmail.com" Date: Sun, 4 Feb 2024 22:29:18 +0200 Subject: [PATCH 0088/1038] spi: fsl-dspi: Preserve error code returned by dmaengine_slave_config() dmaengine_slave_config() may return different error codes based on the circumstances. Preserve it instead of shadowing to -EINVAL. Suggested-by: Vladimir Oltean Signed-off-by: Andy Shevchenko Reviewed-by: Vladimir Oltean Link: https://lore.kernel.org/r/20240204203127.1186621-2-andy.shevchenko@gmail.com Signed-off-by: Mark Brown --- drivers/spi/spi-fsl-dspi.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index c9eae046f66c..0b5ea7a7da71 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -542,7 +542,6 @@ static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) ret = dmaengine_slave_config(dma->chan_rx, &cfg); if (ret) { dev_err(dev, "can't configure rx dma channel\n"); - ret = -EINVAL; goto err_slave_config; } @@ -550,7 +549,6 @@ static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) ret = dmaengine_slave_config(dma->chan_tx, &cfg); if (ret) { dev_err(dev, "can't configure tx dma channel\n"); - ret = -EINVAL; goto err_slave_config; } From 51b8e79c45d5a42891c6196dcd3f73cbb599940a Mon Sep 17 00:00:00 2001 From: "andy.shevchenko@gmail.com" Date: Sun, 4 Feb 2024 22:29:19 +0200 Subject: [PATCH 0089/1038] spi: fsl-dspi: Unify error messaging in dspi_request_dma() Use dev_err_probe() for all messages in dspi_request_dma() for the sake of making them uniform. While at it, fix indentation issue reported by Vladimir Oltean. Signed-off-by: Andy Shevchenko Reviewed-by: Vladimir Oltean Link: https://lore.kernel.org/r/20240204203127.1186621-3-andy.shevchenko@gmail.com Signed-off-by: Mark Brown --- drivers/spi/spi-fsl-dspi.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index 0b5ea7a7da71..38defdcf9370 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -502,15 +502,12 @@ static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) return -ENOMEM; dma->chan_rx = dma_request_chan(dev, "rx"); - if (IS_ERR(dma->chan_rx)) { - return dev_err_probe(dev, PTR_ERR(dma->chan_rx), - "rx dma channel not available\n"); - } + if (IS_ERR(dma->chan_rx)) + return dev_err_probe(dev, PTR_ERR(dma->chan_rx), "rx dma channel not available\n"); dma->chan_tx = dma_request_chan(dev, "tx"); if (IS_ERR(dma->chan_tx)) { - ret = PTR_ERR(dma->chan_tx); - dev_err_probe(dev, ret, "tx dma channel not available\n"); + ret = dev_err_probe(dev, PTR_ERR(dma->chan_tx), "tx dma channel not available\n"); goto err_tx_channel; } @@ -541,14 +538,14 @@ static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) cfg.direction = DMA_DEV_TO_MEM; ret = dmaengine_slave_config(dma->chan_rx, &cfg); if (ret) { - dev_err(dev, "can't configure rx dma channel\n"); + dev_err_probe(dev, ret, "can't configure rx dma channel\n"); goto err_slave_config; } cfg.direction = DMA_MEM_TO_DEV; ret = dmaengine_slave_config(dma->chan_tx, &cfg); if (ret) { - dev_err(dev, "can't configure tx dma channel\n"); + dev_err_probe(dev, ret, "can't configure tx dma channel\n"); goto err_slave_config; } From c8bec3355f08ddb887d5c13b7095dfa79e6db108 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Fri, 26 Jan 2024 15:23:57 -0600 Subject: [PATCH 0090/1038] spi: move split xfers for CS_WORD emulation This moves splitting transfers for CS_WORD software emulation to the same place where we split transfers for controller-specific reasons. This fixes a few subtle bugs. The calculation for maxsize was wrong for bit sizes between 17 and 24. This is fixed by making use of spi_split_transfers_maxwords() which already has the correct calculation. Also, since this indirectly calls spi_res_alloc(), to avoid leaking resources, spi_finalize_current_message() would need to be called on all error paths in __spi_validate() and callers of __spi_validate() would need to do the same. This is fixed by moving the call to __spi_pump_transfer_message() where it is already splitting transfers for other reasons and correctly releases resources in the subsequent error paths. Fixes: cbaa62e0094a ("spi: add software implementation for SPI_CS_WORD") Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20240126212358.3916280-2-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 63 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 6610aeced765..517558e01559 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1743,13 +1743,37 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, trace_spi_message_start(msg); - ret = spi_split_transfers_maxsize(ctlr, msg, - spi_max_transfer_size(msg->spi), - GFP_KERNEL | GFP_DMA); - if (ret) { - msg->status = ret; - spi_finalize_current_message(ctlr); - return ret; + /* + * If an SPI controller does not support toggling the CS line on each + * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO + * for the CS line, we can emulate the CS-per-word hardware function by + * splitting transfers into one-word transfers and ensuring that + * cs_change is set for each transfer. + */ + if ((msg->spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || + spi_is_csgpiod(msg->spi))) { + ret = spi_split_transfers_maxwords(ctlr, msg, 1, GFP_KERNEL); + if (ret) { + msg->status = ret; + spi_finalize_current_message(ctlr); + return ret; + } + + list_for_each_entry(xfer, &msg->transfers, transfer_list) { + /* Don't change cs_change on the last entry in the list */ + if (list_is_last(&xfer->transfer_list, &msg->transfers)) + break; + xfer->cs_change = 1; + } + } else { + ret = spi_split_transfers_maxsize(ctlr, msg, + spi_max_transfer_size(msg->spi), + GFP_KERNEL | GFP_DMA); + if (ret) { + msg->status = ret; + spi_finalize_current_message(ctlr); + return ret; + } } if (ctlr->prepare_message) { @@ -4061,31 +4085,6 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) message->spi = spi; - /* - * If an SPI controller does not support toggling the CS line on each - * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO - * for the CS line, we can emulate the CS-per-word hardware function by - * splitting transfers into one-word transfers and ensuring that - * cs_change is set for each transfer. - */ - if ((spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || - spi_is_csgpiod(spi))) { - size_t maxsize = BITS_TO_BYTES(spi->bits_per_word); - int ret; - - ret = spi_split_transfers_maxsize(ctlr, message, maxsize, - GFP_KERNEL); - if (ret) - return ret; - - list_for_each_entry(xfer, &message->transfers, transfer_list) { - /* Don't change cs_change on the last entry in the list */ - if (list_is_last(&xfer->transfer_list, &message->transfers)) - break; - xfer->cs_change = 1; - } - } - /* * Half-duplex links include original MicroWire, and ones with * only one data pin like SPI_3WIRE (switches direction) or where From 2733092baa3e8a1f05fd16088808be17c98990b4 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Fri, 26 Jan 2024 16:00:23 -0600 Subject: [PATCH 0091/1038] spi: bcm2835: implement ctlr->max_transfer_size The core SPI code will handle splitting transfers if needed as long as ctlr->max_transfer_size is implemented. It does this in __spi_pump_transfer_message() immediately before calling ctlr->prepare_message. So effectively, this change does not alter the behavior of the driver. Also, several peripheral drivers make use of spi_max_transfer_size(), so this should improve compatibility with those drivers. Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20240126220024.3926403-2-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-bcm2835.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/spi/spi-bcm2835.c b/drivers/spi/spi-bcm2835.c index e709887eb2a9..e1b9b1235787 100644 --- a/drivers/spi/spi-bcm2835.c +++ b/drivers/spi/spi-bcm2835.c @@ -1117,19 +1117,6 @@ static int bcm2835_spi_prepare_message(struct spi_controller *ctlr, struct spi_device *spi = msg->spi; struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr); struct bcm2835_spidev *target = spi_get_ctldata(spi); - int ret; - - if (ctlr->can_dma) { - /* - * DMA transfers are limited to 16 bit (0 to 65535 bytes) by - * the SPI HW due to DLEN. Split up transfers (32-bit FIFO - * aligned) if the limit is exceeded. - */ - ret = spi_split_transfers_maxsize(ctlr, msg, 65532, - GFP_KERNEL | GFP_DMA); - if (ret) - return ret; - } /* * Set up clock polarity before spi_transfer_one_message() asserts @@ -1219,6 +1206,19 @@ static int bcm2835_spi_setup_dma(struct spi_controller *ctlr, return 0; } +static size_t bcm2835_spi_max_transfer_size(struct spi_device *spi) +{ + /* + * DMA transfers are limited to 16 bit (0 to 65535 bytes) by + * the SPI HW due to DLEN. Split up transfers (32-bit FIFO + * aligned) if the limit is exceeded. + */ + if (spi->controller->can_dma) + return 65532; + + return SIZE_MAX; +} + static int bcm2835_spi_setup(struct spi_device *spi) { struct spi_controller *ctlr = spi->controller; @@ -1348,6 +1348,7 @@ static int bcm2835_spi_probe(struct platform_device *pdev) ctlr->mode_bits = BCM2835_SPI_MODE_BITS; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); ctlr->num_chipselect = 3; + ctlr->max_transfer_size = bcm2835_spi_max_transfer_size; ctlr->setup = bcm2835_spi_setup; ctlr->cleanup = bcm2835_spi_cleanup; ctlr->transfer_one = bcm2835_spi_transfer_one; From 88c2b56c2690061121cad03f0f551db465287575 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Fri, 2 Feb 2024 15:31:32 -0600 Subject: [PATCH 0092/1038] spi: axi-spi-engine: use common AXI macros This avoid duplicating the same macros in multiple drivers by reusing the common AXI macros for the version register. Signed-off-by: David Lechner Reviewed-by: Nuno Sa Link: https://lore.kernel.org/r/20240202213132.3863124-2-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index 9ace259d2d29..6b0c72bf3395 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -15,12 +16,6 @@ #include #include -#define SPI_ENGINE_VERSION_MAJOR(x) ((x >> 16) & 0xff) -#define SPI_ENGINE_VERSION_MINOR(x) ((x >> 8) & 0xff) -#define SPI_ENGINE_VERSION_PATCH(x) (x & 0xff) - -#define SPI_ENGINE_REG_VERSION 0x00 - #define SPI_ENGINE_REG_RESET 0x40 #define SPI_ENGINE_REG_INT_ENABLE 0x80 @@ -661,12 +656,12 @@ static int spi_engine_probe(struct platform_device *pdev) if (IS_ERR(spi_engine->base)) return PTR_ERR(spi_engine->base); - version = readl(spi_engine->base + SPI_ENGINE_REG_VERSION); - if (SPI_ENGINE_VERSION_MAJOR(version) != 1) { + version = readl(spi_engine->base + ADI_AXI_REG_VERSION); + if (ADI_AXI_PCORE_VER_MAJOR(version) != 1) { dev_err(&pdev->dev, "Unsupported peripheral version %u.%u.%c\n", - SPI_ENGINE_VERSION_MAJOR(version), - SPI_ENGINE_VERSION_MINOR(version), - SPI_ENGINE_VERSION_PATCH(version)); + ADI_AXI_PCORE_VER_MAJOR(version), + ADI_AXI_PCORE_VER_MINOR(version), + ADI_AXI_PCORE_VER_PATCH(version)); return -ENODEV; } From e58db3bcd93b9e0bf5068a29f7e1a97c29926830 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Thu, 1 Feb 2024 14:16:37 +0200 Subject: [PATCH 0093/1038] spi: intel: Add default partition and name to the second chip This should make it easier to identify the second chip and also allows using "mtdparts=" and the like with this chip too. Signed-off-by: Mika Westerberg Link: https://lore.kernel.org/r/20240201121638.207632-1-mika.westerberg@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi-intel.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c index 3654ae35d2db..467d39bf00b4 100644 --- a/drivers/spi/spi-intel.c +++ b/drivers/spi/spi-intel.c @@ -1346,6 +1346,7 @@ static int intel_spi_read_desc(struct intel_spi *ispi) static int intel_spi_populate_chip(struct intel_spi *ispi) { struct flash_platform_data *pdata; + struct mtd_partition *parts; struct spi_board_info chip; int ret; @@ -1376,7 +1377,23 @@ static int intel_spi_populate_chip(struct intel_spi *ispi) if (ispi->host->num_chipselect < 2) return 0; - chip.platform_data = NULL; + pdata = devm_kzalloc(ispi->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return -ENOMEM; + + pdata->name = devm_kasprintf(ispi->dev, GFP_KERNEL, "%s-chip1", + dev_name(ispi->dev)); + pdata->nr_parts = 1; + parts = devm_kcalloc(ispi->dev, pdata->nr_parts, sizeof(*parts), + GFP_KERNEL); + if (!parts) + return -ENOMEM; + + parts[0].size = MTDPART_SIZ_FULL; + parts[0].name = "BIOS1"; + pdata->parts = parts; + + chip.platform_data = pdata; chip.chip_select = 1; if (!spi_new_device(ispi->host, &chip)) From 83c9c7ec8b914faf91567132ab197c54253c277f Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Thu, 1 Feb 2024 14:16:38 +0200 Subject: [PATCH 0094/1038] spi: intel: Keep the BIOS partition inside the first chip If there are two flash chips connected flash regions can refer to the second chip too. In this case we may see the following warning: mtd: partition "BIOS" extends beyond the end of device "0000:00:1f.5" -- size truncated to 0x400000 For this reason, check the BIOS partition size against the chip size and make sure it stays within the that. Signed-off-by: Mika Westerberg Link: https://lore.kernel.org/r/20240201121638.207632-2-mika.westerberg@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi-intel.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c index 467d39bf00b4..3e5dcf2b3c8a 100644 --- a/drivers/spi/spi-intel.c +++ b/drivers/spi/spi-intel.c @@ -1254,6 +1254,13 @@ static void intel_spi_fill_partition(struct intel_spi *ispi, if (end > part->size) part->size = end; } + + /* + * Regions can refer to the second chip too so in this case we + * just make the BIOS partition to occupy the whole chip. + */ + if (ispi->chip0_size && part->size > ispi->chip0_size) + part->size = MTDPART_SIZ_FULL; } static int intel_spi_read_desc(struct intel_spi *ispi) @@ -1350,6 +1357,10 @@ static int intel_spi_populate_chip(struct intel_spi *ispi) struct spi_board_info chip; int ret; + ret = intel_spi_read_desc(ispi); + if (ret) + return ret; + pdata = devm_kzalloc(ispi->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; @@ -1369,10 +1380,6 @@ static int intel_spi_populate_chip(struct intel_spi *ispi) if (!spi_new_device(ispi->host, &chip)) return -ENODEV; - ret = intel_spi_read_desc(ispi); - if (ret) - return ret; - /* Add the second chip if present */ if (ispi->host->num_chipselect < 2) return 0; From 5c018e378f916b56503bba65cbdec66ad16f20b7 Mon Sep 17 00:00:00 2001 From: Luis de Arquer Date: Fri, 26 Jan 2024 20:46:22 +0100 Subject: [PATCH 0095/1038] spi: spi-rockchip: Fix out of bounds array access Since spi-rockchip enables use_gpio_descriptors and the SPI_CONTROLLER_GPIO_SS flag, the spi subsytem may call set_cs() for spi devices with indexes above ROCKCHIP_SPI_MAX_CS_NUM Remove array cs_asserted[] which held a shadow copy of the state of the chip select lines with the only purpose of optimizing out rewriting a chip select line to the current state (no-op) This case is already handled by spi.c Signed-off-by: Luis de Arquer Link: https://lore.kernel.org/r/d0a0c4b94f933f7f43973c34765214303ee82b77.camel@gmail.com Signed-off-by: Mark Brown --- drivers/spi/spi-rockchip.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index 4b9669da2cf3..2fb2d65bd5f4 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -192,8 +192,6 @@ struct rockchip_spi { u8 n_bytes; u8 rsd; - bool cs_asserted[ROCKCHIP_SPI_MAX_CS_NUM]; - bool target_abort; bool cs_inactive; /* spi target tansmition stop when cs inactive */ bool cs_high_supported; /* native CS supports active-high polarity */ @@ -245,10 +243,6 @@ static void rockchip_spi_set_cs(struct spi_device *spi, bool enable) struct rockchip_spi *rs = spi_controller_get_devdata(ctlr); bool cs_asserted = spi->mode & SPI_CS_HIGH ? enable : !enable; - /* Return immediately for no-op */ - if (cs_asserted == rs->cs_asserted[spi_get_chipselect(spi, 0)]) - return; - if (cs_asserted) { /* Keep things powered as long as CS is asserted */ pm_runtime_get_sync(rs->dev); @@ -268,8 +262,6 @@ static void rockchip_spi_set_cs(struct spi_device *spi, bool enable) /* Drop reference from when we first asserted CS */ pm_runtime_put(rs->dev); } - - rs->cs_asserted[spi_get_chipselect(spi, 0)] = cs_asserted; } static void rockchip_spi_handle_err(struct spi_controller *ctlr, From 07d6749317bf8cc416fb7eb8f997a884bd610f8d Mon Sep 17 00:00:00 2001 From: Luis de Arquer Date: Fri, 26 Jan 2024 20:46:44 +0100 Subject: [PATCH 0096/1038] spi: spi-rockchip: Fix max_native_cs Signed-off-by: Luis de Arquer Link: https://lore.kernel.org/r/4d18808e85b85077761c5655083f20ebfd7d3770.camel@gmail.com Signed-off-by: Mark Brown --- drivers/spi/spi-rockchip.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index 2fb2d65bd5f4..e1ecd96c7858 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -160,8 +160,7 @@ */ #define ROCKCHIP_SPI_MAX_TRANLEN 0xffff -/* 2 for native cs, 2 for cs-gpio */ -#define ROCKCHIP_SPI_MAX_CS_NUM 4 +#define ROCKCHIP_SPI_MAX_NATIVE_CS_NUM 2 #define ROCKCHIP_SPI_VER2_TYPE1 0x05EC0002 #define ROCKCHIP_SPI_VER2_TYPE2 0x00110002 @@ -839,7 +838,7 @@ static int rockchip_spi_probe(struct platform_device *pdev) ctlr->target_abort = rockchip_spi_target_abort; } else { ctlr->flags = SPI_CONTROLLER_GPIO_SS; - ctlr->max_native_cs = ROCKCHIP_SPI_MAX_CS_NUM; + ctlr->max_native_cs = ROCKCHIP_SPI_MAX_NATIVE_CS_NUM; /* * rk spi0 has two native cs, spi1..5 one cs only * if num-cs is missing in the dts, default to 1 From e3c5df138169b5494a16b20e74db1ee14b1901ce Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 13:10:18 -0300 Subject: [PATCH 0097/1038] firmware: coreboot: make coreboot_bus_type const Now that the driver core can properly handle constant struct bus_type, move the coreboot_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240204-bus_cleanup-firmware-v1-1-d1bff946606d@marliere.net Signed-off-by: Tzung-Bi Shih --- drivers/firmware/google/coreboot_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 2a4469bf1b81..fcdc387ae7aa 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -53,7 +53,7 @@ static void coreboot_bus_remove(struct device *dev) driver->remove(device); } -static struct bus_type coreboot_bus_type = { +static const struct bus_type coreboot_bus_type = { .name = "coreboot", .match = coreboot_bus_match, .probe = coreboot_bus_probe, From a9dde2823c3330503fb6906d18c7bcced7b280af Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 12:23:29 -0300 Subject: [PATCH 0098/1038] ata: pata_parport: make pata_parport_bus_type const Now that the driver core can properly handle constant struct bus_type, move the pata_parport_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Damien Le Moal Reviewed-by: Greg Kroah-Hartman Signed-off-by: Niklas Cassel --- drivers/ata/pata_parport/pata_parport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ata/pata_parport/pata_parport.c b/drivers/ata/pata_parport/pata_parport.c index a7adfdcb5e27..9a2cb9ca9d1d 100644 --- a/drivers/ata/pata_parport/pata_parport.c +++ b/drivers/ata/pata_parport/pata_parport.c @@ -464,7 +464,7 @@ static void pata_parport_bus_release(struct device *dev) /* nothing to do here but required to avoid warning on device removal */ } -static struct bus_type pata_parport_bus_type = { +static const struct bus_type pata_parport_bus_type = { .name = DRV_NAME, }; From 7f7f2b7624d4770978c15c0ecd720dc703f7ace3 Mon Sep 17 00:00:00 2001 From: Hari Prasath Gujulan Elango Date: Mon, 5 Feb 2024 16:22:01 +0530 Subject: [PATCH 0099/1038] dt-bindings: ata: atmel: remove at91 compact flash documentation The compatible "at91rm9200-cf" is not used by any driver, hence remove the corresponding documentation. Signed-off-by: Hari Prasath Gujulan Elango Acked-by: Rob Herring Acked-by: Nicolas Ferre [cassel: add missing space in commit log] Signed-off-by: Niklas Cassel --- .../devicetree/bindings/ata/atmel-at91_cf.txt | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 Documentation/devicetree/bindings/ata/atmel-at91_cf.txt diff --git a/Documentation/devicetree/bindings/ata/atmel-at91_cf.txt b/Documentation/devicetree/bindings/ata/atmel-at91_cf.txt deleted file mode 100644 index c1d22b3ae134..000000000000 --- a/Documentation/devicetree/bindings/ata/atmel-at91_cf.txt +++ /dev/null @@ -1,19 +0,0 @@ -Atmel AT91RM9200 CompactFlash - -Required properties: -- compatible : "atmel,at91rm9200-cf". -- reg : should specify localbus address and size used. -- gpios : specifies the gpio pins to control the CF device. Detect - and reset gpio's are mandatory while irq and vcc gpio's are - optional and may be set to 0 if not present. - -Example: -compact-flash@50000000 { - compatible = "atmel,at91rm9200-cf"; - reg = <0x50000000 0x30000000>; - gpios = <&pioC 13 0 /* irq */ - &pioC 15 0 /* detect */ - 0 /* vcc */ - &pioC 5 0 /* reset */ - >; -}; From fc67d663aabdfddbba8e16a40c3f6a973face509 Mon Sep 17 00:00:00 2001 From: Adam Ford Date: Sat, 3 Feb 2024 10:52:43 -0600 Subject: [PATCH 0100/1038] dt-bindings: soc: imx: add missing clock and power-domains to imx8mp-hdmi-blk-ctrl Per guidance from the NXP downstream kernel, if the clock is disabled before HDMI/LCDIF probe, LCDIF will not get pixel clock from HDMI PHY and throw an error: [CRTC:39:crtc-2] vblank wait timed out WARNING: CPU: 2 PID: 9 at drivers/gpu/drm/drm_atomic_helper.c: 1634 drm_atomic_helper_wait_for_vblanks.part.0+0x23c/0x260 Fix this by adding the fdcc clock to the hdmi_blk_ctrl. This should be safe, since neither this power domain nor the dependent HDMI and LCDIF drivers been enabled or added to the SoC device tree yet. According to Sandor Yu from NXP, "the FDCC clock is not for HDMITX in desgin, but it is part of HDMI domain that needed by HDMITX. So I think it is reasonable added it to the power domain driver." The driver also supports two power domains which are missing from the binding that also fix an issue with resuming from suspend. Signed-off-by: Adam Ford Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240203165307.7806-4-aford173@gmail.com Signed-off-by: Ulf Hansson --- .../soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml index 1be4ce2a45e8..bd1cdaa4f54b 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml @@ -27,8 +27,8 @@ properties: const: 1 power-domains: - minItems: 8 - maxItems: 8 + minItems: 10 + maxItems: 10 power-domain-names: items: @@ -40,10 +40,12 @@ properties: - const: trng - const: hdmi-tx - const: hdmi-tx-phy + - const: hdcp + - const: hrv clocks: - minItems: 4 - maxItems: 4 + minItems: 5 + maxItems: 5 clock-names: items: @@ -51,6 +53,7 @@ properties: - const: axi - const: ref_266m - const: ref_24m + - const: fdcc interconnects: maxItems: 3 @@ -82,12 +85,15 @@ examples: clocks = <&clk IMX8MP_CLK_HDMI_APB>, <&clk IMX8MP_CLK_HDMI_ROOT>, <&clk IMX8MP_CLK_HDMI_REF_266M>, - <&clk IMX8MP_CLK_HDMI_24M>; - clock-names = "apb", "axi", "ref_266m", "ref_24m"; + <&clk IMX8MP_CLK_HDMI_24M>, + <&clk IMX8MP_CLK_HDMI_FDCC_TST>; + clock-names = "apb", "axi", "ref_266m", "ref_24m", "fdcc"; power-domains = <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, - <&pgc_hdmimix>, <&pgc_hdmi_phy>; + <&pgc_hdmimix>, <&pgc_hdmi_phy>, + <&pgc_hdmimix>, <&pgc_hdmimix>; power-domain-names = "bus", "irqsteer", "lcdif", "pai", "pvi", "trng", - "hdmi-tx", "hdmi-tx-phy"; + "hdmi-tx", "hdmi-tx-phy", + "hdcp", "hrv"; #power-domain-cells = <1>; }; From c0c0293cf7a0f21ef461956d44e4add718574f3f Mon Sep 17 00:00:00 2001 From: David Lechner Date: Tue, 6 Feb 2024 14:06:46 -0600 Subject: [PATCH 0101/1038] spi: drop gpf arg from __spi_split_transfer_maxsize() The __spi_split_transfer_maxsize() function has a gpf argument to allow callers to specify the type of memory allocation that needs to be used. However, this function only allocates struct spi_transfer and is not intended to be used from atomic contexts so this type should always be GFP_KERNEL, so we can just drop the argument. Some callers of these functions also passed GFP_DMA, but since only struct spi_transfer is allocated and not any tx/rx buffers, this is not actually necessary and is removed in this commit. Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20240206200648.1782234-1-dlechner@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-stm32.c | 4 +--- drivers/spi/spi.c | 22 ++++++++-------------- include/linux/spi/spi.h | 6 ++---- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index e61302ef3c21..c32e57bb38bd 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1170,9 +1170,7 @@ static int stm32_spi_prepare_msg(struct spi_controller *ctrl, if (spi->cfg->set_number_of_data) { int ret; - ret = spi_split_transfers_maxwords(ctrl, msg, - spi->t_size_max, - GFP_KERNEL | GFP_DMA); + ret = spi_split_transfers_maxwords(ctrl, msg, spi->t_size_max); if (ret) return ret; } diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 517558e01559..3270a1337a5b 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1752,7 +1752,7 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, */ if ((msg->spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || spi_is_csgpiod(msg->spi))) { - ret = spi_split_transfers_maxwords(ctlr, msg, 1, GFP_KERNEL); + ret = spi_split_transfers_maxwords(ctlr, msg, 1); if (ret) { msg->status = ret; spi_finalize_current_message(ctlr); @@ -1767,8 +1767,7 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, } } else { ret = spi_split_transfers_maxsize(ctlr, msg, - spi_max_transfer_size(msg->spi), - GFP_KERNEL | GFP_DMA); + spi_max_transfer_size(msg->spi)); if (ret) { msg->status = ret; spi_finalize_current_message(ctlr); @@ -3707,8 +3706,7 @@ static struct spi_replaced_transfers *spi_replace_transfers( static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, struct spi_message *msg, struct spi_transfer **xferp, - size_t maxsize, - gfp_t gfp) + size_t maxsize) { struct spi_transfer *xfer = *xferp, *xfers; struct spi_replaced_transfers *srt; @@ -3719,7 +3717,7 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, count = DIV_ROUND_UP(xfer->len, maxsize); /* Create replacement */ - srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, gfp); + srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, GFP_KERNEL); if (IS_ERR(srt)) return PTR_ERR(srt); xfers = srt->inserted_transfers; @@ -3779,14 +3777,12 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, * @ctlr: the @spi_controller for this transfer * @msg: the @spi_message to transform * @maxsize: the maximum when to apply this - * @gfp: GFP allocation flags * * Return: status of transformation */ int spi_split_transfers_maxsize(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxsize, - gfp_t gfp) + size_t maxsize) { struct spi_transfer *xfer; int ret; @@ -3801,7 +3797,7 @@ int spi_split_transfers_maxsize(struct spi_controller *ctlr, list_for_each_entry(xfer, &msg->transfers, transfer_list) { if (xfer->len > maxsize) { ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer, - maxsize, gfp); + maxsize); if (ret) return ret; } @@ -3819,14 +3815,12 @@ EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize); * @ctlr: the @spi_controller for this transfer * @msg: the @spi_message to transform * @maxwords: the number of words to limit each transfer to - * @gfp: GFP allocation flags * * Return: status of transformation */ int spi_split_transfers_maxwords(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxwords, - gfp_t gfp) + size_t maxwords) { struct spi_transfer *xfer; @@ -3844,7 +3838,7 @@ int spi_split_transfers_maxwords(struct spi_controller *ctlr, maxsize = maxwords * roundup_pow_of_two(BITS_TO_BYTES(xfer->bits_per_word)); if (xfer->len > maxsize) { ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer, - maxsize, gfp); + maxsize); if (ret) return ret; } diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 29c3e4dd5d93..9339c8ed1f8f 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1365,12 +1365,10 @@ struct spi_replaced_transfers { extern int spi_split_transfers_maxsize(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxsize, - gfp_t gfp); + size_t maxsize); extern int spi_split_transfers_maxwords(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxwords, - gfp_t gfp); + size_t maxwords); /*---------------------------------------------------------------------------*/ From 531860e12da76a444e0ecfd37a9d786e7986957a Mon Sep 17 00:00:00 2001 From: David Lechner Date: Wed, 7 Feb 2024 08:51:24 -0600 Subject: [PATCH 0102/1038] spi: axi-spi-engine: remove use of ida for sync id Profiling has shown that ida_alloc_range() accounts for about 10% of the time spent in spi_sync() when using the AXI SPI Engine controller. This call is used to create a unique id for each SPI message to match to an IRQ when the message is complete. Since the core SPI code serializes messages in a message queue, we can only have one message in flight at a time, namely host->cur_msg. This means that we can use a fixed value instead of a unique id for each message since there can never be more than one message pending at a time. This patch removes the use of ida for the sync id and replaces it with a constant value. This simplifies the driver and improves performance. Reviewed-by: Nuno Sa Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20240207-axi-spi-engine-round-2-1-v2-1-40c0b4e85352@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index 6b0c72bf3395..606389566129 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -57,6 +56,9 @@ #define SPI_ENGINE_TRANSFER_WRITE 0x1 #define SPI_ENGINE_TRANSFER_READ 0x2 +/* Arbitrary sync ID for use by host->cur_msg */ +#define AXI_SPI_ENGINE_CUR_MSG_SYNC_ID 0x1 + #define SPI_ENGINE_CMD(inst, arg1, arg2) \ (((inst) << 12) | ((arg1) << 8) | (arg2)) @@ -98,8 +100,6 @@ struct spi_engine_message_state { unsigned int rx_length; /** @rx_buf: Bytes not yet written to the RX FIFO. */ uint8_t *rx_buf; - /** @sync_id: ID to correlate SYNC interrupts with this message. */ - u8 sync_id; }; struct spi_engine { @@ -109,7 +109,6 @@ struct spi_engine { spinlock_t lock; void __iomem *base; - struct ida sync_ida; struct timer_list watchdog_timer; struct spi_controller *controller; @@ -483,9 +482,7 @@ static irqreturn_t spi_engine_irq(int irq, void *devid) } if (pending & SPI_ENGINE_INT_SYNC && msg) { - struct spi_engine_message_state *st = msg->state; - - if (completed_id == st->sync_id) { + if (completed_id == AXI_SPI_ENGINE_CUR_MSG_SYNC_ID) { if (timer_delete_sync(&spi_engine->watchdog_timer)) { msg->status = 0; msg->actual_length = msg->frame_length; @@ -510,10 +507,8 @@ static int spi_engine_prepare_message(struct spi_controller *host, struct spi_message *msg) { struct spi_engine_program p_dry, *p; - struct spi_engine *spi_engine = spi_controller_get_devdata(host); struct spi_engine_message_state *st; size_t size; - int ret; st = kzalloc(sizeof(*st), GFP_KERNEL); if (!st) @@ -531,18 +526,10 @@ static int spi_engine_prepare_message(struct spi_controller *host, return -ENOMEM; } - ret = ida_alloc_range(&spi_engine->sync_ida, 0, U8_MAX, GFP_KERNEL); - if (ret < 0) { - kfree(p); - kfree(st); - return ret; - } - - st->sync_id = ret; - spi_engine_compile_message(msg, false, p); - spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC(st->sync_id)); + spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC( + AXI_SPI_ENGINE_CUR_MSG_SYNC_ID)); st->p = p; st->cmd_buf = p->instructions; @@ -555,10 +542,8 @@ static int spi_engine_prepare_message(struct spi_controller *host, static int spi_engine_unprepare_message(struct spi_controller *host, struct spi_message *msg) { - struct spi_engine *spi_engine = spi_controller_get_devdata(host); struct spi_engine_message_state *st = msg->state; - ida_free(&spi_engine->sync_ida, st->sync_id); kfree(st->p); kfree(st); @@ -640,7 +625,6 @@ static int spi_engine_probe(struct platform_device *pdev) spi_engine = spi_controller_get_devdata(host); spin_lock_init(&spi_engine->lock); - ida_init(&spi_engine->sync_ida); timer_setup(&spi_engine->watchdog_timer, spi_engine_timeout, TIMER_IRQSAFE); spi_engine->controller = host; From abb4b46c43689dd1f4d80c41e49127ca0ede75b3 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Wed, 7 Feb 2024 08:51:25 -0600 Subject: [PATCH 0103/1038] spi: axi-spi-engine: move msg finalization out of irq handler As a general principal, it is best to do as little as possible in an interrupt handler. This patch reworks the AXI SPI Engine driver to move timer_delete_sync() and spi_finalize_current_message() out of the interrupt handler. Instead, spi_finalize_current_message() is moved to the transfer_one_message function (similar to nearly all other SPI controllers). A completion is now used to wait for the sync interrupt that indicates that the message is complete. The watchdog timer is no longer needed since we can use the wait_for_completion_timeout() function to wait for the message to complete with the same effect. As a bonus, these changes also improve throughput of the SPI controller. For example, this was tested on a ZynqMP with a 80MHz SCLK reading 4 byte samples from an ADC. The max measured throughput increased from 26k to 28k samples per second. Reviewed-by: Nuno Sa Signed-off-by: David Lechner Link: https://lore.kernel.org/r/20240207-axi-spi-engine-round-2-1-v2-2-40c0b4e85352@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 40 ++++++++++++-------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index 606389566129..ca66d202f0e2 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -13,7 +14,6 @@ #include #include #include -#include #define SPI_ENGINE_REG_RESET 0x40 @@ -109,9 +109,7 @@ struct spi_engine { spinlock_t lock; void __iomem *base; - struct timer_list watchdog_timer; - struct spi_controller *controller; - + struct completion msg_complete; unsigned int int_enable; }; @@ -483,11 +481,9 @@ static irqreturn_t spi_engine_irq(int irq, void *devid) if (pending & SPI_ENGINE_INT_SYNC && msg) { if (completed_id == AXI_SPI_ENGINE_CUR_MSG_SYNC_ID) { - if (timer_delete_sync(&spi_engine->watchdog_timer)) { - msg->status = 0; - msg->actual_length = msg->frame_length; - spi_finalize_current_message(host); - } + msg->status = 0; + msg->actual_length = msg->frame_length; + complete(&spi_engine->msg_complete); disable_int |= SPI_ENGINE_INT_SYNC; } } @@ -558,7 +554,7 @@ static int spi_engine_transfer_one_message(struct spi_controller *host, unsigned int int_enable = 0; unsigned long flags; - mod_timer(&spi_engine->watchdog_timer, jiffies + msecs_to_jiffies(5000)); + reinit_completion(&spi_engine->msg_complete); spin_lock_irqsave(&spi_engine->lock, flags); @@ -580,21 +576,16 @@ static int spi_engine_transfer_one_message(struct spi_controller *host, spi_engine->int_enable = int_enable; spin_unlock_irqrestore(&spi_engine->lock, flags); - return 0; -} + if (!wait_for_completion_timeout(&spi_engine->msg_complete, + msecs_to_jiffies(5000))) { + dev_err(&host->dev, + "Timeout occurred while waiting for transfer to complete. Hardware is probably broken.\n"); + msg->status = -ETIMEDOUT; + } -static void spi_engine_timeout(struct timer_list *timer) -{ - struct spi_engine *spi_engine = from_timer(spi_engine, timer, watchdog_timer); - struct spi_controller *host = spi_engine->controller; - - if (WARN_ON(!host->cur_msg)) - return; - - dev_err(&host->dev, - "Timeout occurred while waiting for transfer to complete. Hardware is probably broken.\n"); - host->cur_msg->status = -ETIMEDOUT; spi_finalize_current_message(host); + + return msg->status; } static void spi_engine_release_hw(void *p) @@ -625,8 +616,7 @@ static int spi_engine_probe(struct platform_device *pdev) spi_engine = spi_controller_get_devdata(host); spin_lock_init(&spi_engine->lock); - timer_setup(&spi_engine->watchdog_timer, spi_engine_timeout, TIMER_IRQSAFE); - spi_engine->controller = host; + init_completion(&spi_engine->msg_complete); spi_engine->clk = devm_clk_get_enabled(&pdev->dev, "s_axi_aclk"); if (IS_ERR(spi_engine->clk)) From 5b73e706f00f3553e1a4efbb31951ce9fe18f2dc Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:50 +0106 Subject: [PATCH 0104/1038] printk: nbcon: Relocate 32bit seq macros The macros __seq_to_nbcon_seq() and __nbcon_seq_to_seq() are used to provide support for atomic handling of sequence numbers on 32bit systems. Until now this was only used by nbcon.c, which is why they were located in nbcon.c and include nbcon in the name. In a follow-up commit this functionality is also needed by printk_ringbuffer. Rather than duplicating the functionality, relocate the macros to printk_ringbuffer.h. Also, since the macros will be no longer nbcon-specific, rename them to __u64seq_to_ulseq() and __ulseq_to_u64seq(). This does not result in any functional change. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-2-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 41 +++---------------------------- kernel/printk/printk_ringbuffer.h | 33 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index b96077152f49..c8093bcc01fe 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -140,39 +140,6 @@ static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_sta return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); } -#ifdef CONFIG_64BIT - -#define __seq_to_nbcon_seq(seq) (seq) -#define __nbcon_seq_to_seq(seq) (seq) - -#else /* CONFIG_64BIT */ - -#define __seq_to_nbcon_seq(seq) ((u32)seq) - -static inline u64 __nbcon_seq_to_seq(u32 nbcon_seq) -{ - u64 seq; - u64 rb_next_seq; - - /* - * The provided sequence is only the lower 32 bits of the ringbuffer - * sequence. It needs to be expanded to 64bit. Get the next sequence - * number from the ringbuffer and fold it. - * - * Having a 32bit representation in the console is sufficient. - * If a console ever gets more than 2^31 records behind - * the ringbuffer then this is the least of the problems. - * - * Also the access to the ring buffer is always safe. - */ - rb_next_seq = prb_next_seq(prb); - seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq); - - return seq; -} - -#endif /* CONFIG_64BIT */ - /** * nbcon_seq_read - Read the current console sequence * @con: Console to read the sequence of @@ -183,7 +150,7 @@ u64 nbcon_seq_read(struct console *con) { unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq)); - return __nbcon_seq_to_seq(nbcon_seq); + return __ulseq_to_u64seq(prb, nbcon_seq); } /** @@ -204,7 +171,7 @@ void nbcon_seq_force(struct console *con, u64 seq) */ u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb)); - atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __seq_to_nbcon_seq(valid_seq)); + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq)); /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */ con->seq = 0; @@ -223,11 +190,11 @@ void nbcon_seq_force(struct console *con, u64 seq) */ static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) { - unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq); + unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq); struct console *con = ctxt->console; if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq, - __seq_to_nbcon_seq(new_seq))) { + __u64seq_to_ulseq(new_seq))) { ctxt->seq = new_seq; } else { ctxt->seq = nbcon_seq_read(con); diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index 18cd25e489b8..b82a96dc2ea2 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -381,4 +381,37 @@ bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, u64 prb_first_valid_seq(struct printk_ringbuffer *rb); u64 prb_next_seq(struct printk_ringbuffer *rb); +#ifdef CONFIG_64BIT + +#define __u64seq_to_ulseq(u64seq) (u64seq) +#define __ulseq_to_u64seq(rb, ulseq) (ulseq) + +#else /* CONFIG_64BIT */ + +#define __u64seq_to_ulseq(u64seq) ((u32)u64seq) + +static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) +{ + u64 seq; + u64 rb_next_seq; + + /* + * The provided sequence is only the lower 32 bits of the ringbuffer + * sequence. It needs to be expanded to 64bit. Get the next sequence + * number from the ringbuffer and fold it. + * + * Having a 32bit representation in the console is sufficient. + * If a console ever gets more than 2^31 records behind + * the ringbuffer then this is the least of the problems. + * + * Also the access to the ring buffer is always safe. + */ + rb_next_seq = prb_next_seq(rb); + seq = rb_next_seq - ((u32)rb_next_seq - ulseq); + + return seq; +} + +#endif /* CONFIG_64BIT */ + #endif /* _KERNEL_PRINTK_RINGBUFFER_H */ From 418ec1961c07d84293cc3cd54d67b90bbeba7feb Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 7 Feb 2024 14:46:51 +0106 Subject: [PATCH 0105/1038] printk: Adjust mapping for 32bit seq macros Note: This change only applies to 32bit architectures. On 64bit architectures the macros are NOPs. __ulseq_to_u64seq() computes the upper 32 bits of the passed argument value (@ulseq). The upper bits are derived from a base value (@rb_next_seq) in a way that assumes @ulseq represents a 64bit number that is less than or equal to @rb_next_seq. Until now this mapping has been correct for all call sites. However, in a follow-up commit, values of @ulseq will be passed in that are higher than the base value. This requires a change to how the 32bit value is mapped to a 64bit sequence number. Rather than mapping @ulseq such that the base value is the end of a 32bit block, map @ulseq such that the base value is in the middle of a 32bit block. This allows supporting 31 bits before and after the base value, which is deemed acceptable for the console sequence number during runtime. Here is an example to illustrate the previous and new mappings. For a base value (@rb_next_seq) of 2 2000 0000... Before this change the range of possible return values was: 1 2000 0001 to 2 2000 0000 __ulseq_to_u64seq(1fff ffff) => 2 1fff ffff __ulseq_to_u64seq(2000 0000) => 2 2000 0000 __ulseq_to_u64seq(2000 0001) => 1 2000 0001 __ulseq_to_u64seq(9fff ffff) => 1 9fff ffff __ulseq_to_u64seq(a000 0000) => 1 a000 0000 __ulseq_to_u64seq(a000 0001) => 1 a000 0001 After this change the range of possible return values are: 1 a000 0001 to 2 a000 0000 __ulseq_to_u64seq(1fff ffff) => 2 1fff ffff __ulseq_to_u64seq(2000 0000) => 2 2000 0000 __ulseq_to_u64seq(2000 0001) => 2 2000 0001 __ulseq_to_u64seq(9fff ffff) => 2 9fff ffff __ulseq_to_u64seq(a000 0000) => 2 a000 0000 __ulseq_to_u64seq(a000 0001) => 1 a000 0001 [ john.ogness: Rewrite commit message. ] Reported-by: Francesco Dolcini Reported-by: kernel test robot Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-3-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index b82a96dc2ea2..12f60c782e46 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -407,7 +407,7 @@ static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) * Also the access to the ring buffer is always safe. */ rb_next_seq = prb_next_seq(rb); - seq = rb_next_seq - ((u32)rb_next_seq - ulseq); + seq = rb_next_seq - (s32)((u32)rb_next_seq - ulseq); return seq; } From 90ad525c2d9a8a6591ab822234a94b82871ef8e0 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:52 +0106 Subject: [PATCH 0106/1038] printk: Use prb_first_seq() as base for 32bit seq macros Note: This change only applies to 32bit architectures. On 64bit architectures the macros are NOPs. Currently prb_next_seq() is used as the base for the 32bit seq macros __u64seq_to_ulseq() and __ulseq_to_u64seq(). However, in a follow-up commit, prb_next_seq() will need to make use of the 32bit seq macros. Use prb_first_seq() as the base for the 32bit seq macros instead because it is guaranteed to return 64bit sequence numbers without relying on any 32bit seq macros. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-4-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.c | 2 +- kernel/printk/printk_ringbuffer.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index fde338606ce8..49a82ccce8e9 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -1832,7 +1832,7 @@ static int prb_read(struct printk_ringbuffer *rb, u64 seq, } /* Get the sequence number of the tail descriptor. */ -static u64 prb_first_seq(struct printk_ringbuffer *rb) +u64 prb_first_seq(struct printk_ringbuffer *rb) { struct prb_desc_ring *desc_ring = &rb->desc_ring; enum desc_state d_state; diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index 12f60c782e46..ee294aaf4aeb 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -378,6 +378,7 @@ bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, struct printk_info *info, unsigned int *line_count); +u64 prb_first_seq(struct printk_ringbuffer *rb); u64 prb_first_valid_seq(struct printk_ringbuffer *rb); u64 prb_next_seq(struct printk_ringbuffer *rb); @@ -392,12 +393,12 @@ u64 prb_next_seq(struct printk_ringbuffer *rb); static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) { + u64 rb_first_seq = prb_first_seq(rb); u64 seq; - u64 rb_next_seq; /* * The provided sequence is only the lower 32 bits of the ringbuffer - * sequence. It needs to be expanded to 64bit. Get the next sequence + * sequence. It needs to be expanded to 64bit. Get the first sequence * number from the ringbuffer and fold it. * * Having a 32bit representation in the console is sufficient. @@ -406,8 +407,7 @@ static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) * * Also the access to the ring buffer is always safe. */ - rb_next_seq = prb_next_seq(rb); - seq = rb_next_seq - (s32)((u32)rb_next_seq - ulseq); + seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq); return seq; } From 5f72e52ba959e50680b8d83599da1368cd7a6ee2 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:53 +0106 Subject: [PATCH 0107/1038] printk: ringbuffer: Do not skip non-finalized records with prb_next_seq() Commit f244b4dc53e5 ("printk: ringbuffer: Improve prb_next_seq() performance") introduced an optimization for prb_next_seq() by using best-effort to track recently finalized records. However, the order of finalization does not necessarily match the order of the records. The optimization changed prb_next_seq() to return inconsistent results, possibly yielding sequence numbers that are not available to readers because they are preceded by non-finalized records or they are not yet visible to the reader CPU. Rather than simply best-effort tracking recently finalized records, force the committing writer to read records and increment the last "contiguous block" of finalized records. In order to do this, the sequence number instead of ID must be stored because ID's cannot be directly compared. A new memory barrier pair is introduced to guarantee that a reader can always read the records up until the sequence number returned by prb_next_seq() (unless the records have since been overwritten in the ringbuffer). This restores the original functionality of prb_next_seq() while also keeping the optimization. For 32bit systems, only the lower 32 bits of the sequence number are stored. When reading the value, it is expanded to the full 64bit sequence number using the 32bit seq macros, which fold in the value returned by prb_first_seq(). Fixes: f244b4dc53e5 ("printk: ringbuffer: Improve prb_next_seq() performance") Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-5-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.c | 164 +++++++++++++++++++++++------- kernel/printk/printk_ringbuffer.h | 4 +- 2 files changed, 127 insertions(+), 41 deletions(-) diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 49a82ccce8e9..04c26cca546f 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -6,6 +6,7 @@ #include #include #include "printk_ringbuffer.h" +#include "internal.h" /** * DOC: printk_ringbuffer overview @@ -303,6 +304,9 @@ * * desc_push_tail:B / desc_reserve:D * set descriptor reusable (state), then push descriptor tail (id) + * + * desc_update_last_finalized:A / desc_last_finalized_seq:A + * store finalized record, then set new highest finalized sequence number */ #define DATA_SIZE(data_ring) _DATA_SIZE((data_ring)->size_bits) @@ -1441,20 +1445,118 @@ fail_reopen: return false; } +/* + * @last_finalized_seq value guarantees that all records up to and including + * this sequence number are finalized and can be read. The only exception are + * too old records which have already been overwritten. + * + * It is also guaranteed that @last_finalized_seq only increases. + * + * Be aware that finalized records following non-finalized records are not + * reported because they are not yet available to the reader. For example, + * a new record stored via printk() will not be available to a printer if + * it follows a record that has not been finalized yet. However, once that + * non-finalized record becomes finalized, @last_finalized_seq will be + * appropriately updated and the full set of finalized records will be + * available to the printer. And since each printk() caller will either + * directly print or trigger deferred printing of all available unprinted + * records, all printk() messages will get printed. + */ +static u64 desc_last_finalized_seq(struct printk_ringbuffer *rb) +{ + struct prb_desc_ring *desc_ring = &rb->desc_ring; + unsigned long ulseq; + + /* + * Guarantee the sequence number is loaded before loading the + * associated record in order to guarantee that the record can be + * seen by this CPU. This pairs with desc_update_last_finalized:A. + */ + ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq + ); /* LMM(desc_last_finalized_seq:A) */ + + return __ulseq_to_u64seq(rb, ulseq); +} + +static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, + struct printk_record *r, unsigned int *line_count); + +/* + * Check if there are records directly following @last_finalized_seq that are + * finalized. If so, update @last_finalized_seq to the latest of these + * records. It is not allowed to skip over records that are not yet finalized. + */ +static void desc_update_last_finalized(struct printk_ringbuffer *rb) +{ + struct prb_desc_ring *desc_ring = &rb->desc_ring; + u64 old_seq = desc_last_finalized_seq(rb); + unsigned long oldval; + unsigned long newval; + u64 finalized_seq; + u64 try_seq; + +try_again: + finalized_seq = old_seq; + try_seq = finalized_seq + 1; + + /* Try to find later finalized records. */ + while (_prb_read_valid(rb, &try_seq, NULL, NULL)) { + finalized_seq = try_seq; + try_seq++; + } + + /* No update needed if no later finalized record was found. */ + if (finalized_seq == old_seq) + return; + + oldval = __u64seq_to_ulseq(old_seq); + newval = __u64seq_to_ulseq(finalized_seq); + + /* + * Set the sequence number of a later finalized record that has been + * seen. + * + * Guarantee the record data is visible to other CPUs before storing + * its sequence number. This pairs with desc_last_finalized_seq:A. + * + * Memory barrier involvement: + * + * If desc_last_finalized_seq:A reads from + * desc_update_last_finalized:A, then desc_read:A reads from + * _prb_commit:B. + * + * Relies on: + * + * RELEASE from _prb_commit:B to desc_update_last_finalized:A + * matching + * ACQUIRE from desc_last_finalized_seq:A to desc_read:A + * + * Note: _prb_commit:B and desc_update_last_finalized:A can be + * different CPUs. However, the desc_update_last_finalized:A + * CPU (which performs the release) must have previously seen + * _prb_commit:B. + */ + if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq, + &oldval, newval)) { /* LMM(desc_update_last_finalized:A) */ + old_seq = __ulseq_to_u64seq(rb, oldval); + goto try_again; + } +} + /* * Attempt to finalize a specified descriptor. If this fails, the descriptor * is either already final or it will finalize itself when the writer commits. */ -static void desc_make_final(struct prb_desc_ring *desc_ring, unsigned long id) +static void desc_make_final(struct printk_ringbuffer *rb, unsigned long id) { + struct prb_desc_ring *desc_ring = &rb->desc_ring; unsigned long prev_state_val = DESC_SV(id, desc_committed); struct prb_desc *d = to_desc(desc_ring, id); - atomic_long_cmpxchg_relaxed(&d->state_var, prev_state_val, - DESC_SV(id, desc_finalized)); /* LMM(desc_make_final:A) */ - - /* Best effort to remember the last finalized @id. */ - atomic_long_set(&desc_ring->last_finalized_id, id); + if (atomic_long_try_cmpxchg_relaxed(&d->state_var, &prev_state_val, + DESC_SV(id, desc_finalized))) { /* LMM(desc_make_final:A) */ + desc_update_last_finalized(rb); + } } /** @@ -1550,7 +1652,7 @@ bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb, * readers. (For seq==0 there is no previous descriptor.) */ if (info->seq > 0) - desc_make_final(desc_ring, DESC_ID(id - 1)); + desc_make_final(rb, DESC_ID(id - 1)); r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id); /* If text data allocation fails, a data-less record is committed. */ @@ -1643,7 +1745,7 @@ void prb_commit(struct prb_reserved_entry *e) */ head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */ if (head_id != e->id) - desc_make_final(desc_ring, e->id); + desc_make_final(e->rb, e->id); } /** @@ -1663,12 +1765,9 @@ void prb_commit(struct prb_reserved_entry *e) */ void prb_final_commit(struct prb_reserved_entry *e) { - struct prb_desc_ring *desc_ring = &e->rb->desc_ring; - _prb_commit(e, desc_finalized); - /* Best effort to remember the last finalized @id. */ - atomic_long_set(&desc_ring->last_finalized_id, e->id); + desc_update_last_finalized(e->rb); } /* @@ -2008,7 +2107,9 @@ u64 prb_first_valid_seq(struct printk_ringbuffer *rb) * newest sequence number available to readers will be. * * This provides readers a sequence number to jump to if all currently - * available records should be skipped. + * available records should be skipped. It is guaranteed that all records + * previous to the returned value have been finalized and are (or were) + * available to the reader. * * Context: Any context. * Return: The sequence number of the next newest (not yet available) record @@ -2016,34 +2117,19 @@ u64 prb_first_valid_seq(struct printk_ringbuffer *rb) */ u64 prb_next_seq(struct printk_ringbuffer *rb) { - struct prb_desc_ring *desc_ring = &rb->desc_ring; - enum desc_state d_state; - unsigned long id; u64 seq; - /* Check if the cached @id still points to a valid @seq. */ - id = atomic_long_read(&desc_ring->last_finalized_id); - d_state = desc_read(desc_ring, id, NULL, &seq, NULL); + seq = desc_last_finalized_seq(rb); - if (d_state == desc_finalized || d_state == desc_reusable) { - /* - * Begin searching after the last finalized record. - * - * On 0, the search must begin at 0 because of hack#2 - * of the bootstrapping phase it is not known if a - * record at index 0 exists. - */ - if (seq != 0) - seq++; - } else { - /* - * The information about the last finalized sequence number - * has gone. It should happen only when there is a flood of - * new messages and the ringbuffer is rapidly recycled. - * Give up and start from the beginning. - */ - seq = 0; - } + /* + * Begin searching after the last finalized record. + * + * On 0, the search must begin at 0 because of hack#2 + * of the bootstrapping phase it is not known if a + * record at index 0 exists. + */ + if (seq != 0) + seq++; /* * The information about the last finalized @seq might be inaccurate. @@ -2085,7 +2171,7 @@ void prb_init(struct printk_ringbuffer *rb, rb->desc_ring.infos = infos; atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits)); atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits)); - atomic_long_set(&rb->desc_ring.last_finalized_id, DESC0_ID(descbits)); + atomic_long_set(&rb->desc_ring.last_finalized_seq, 0); rb->text_data_ring.size_bits = textbits; rb->text_data_ring.data = text_buf; diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index ee294aaf4aeb..2d948cc82b5b 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -75,7 +75,7 @@ struct prb_desc_ring { struct printk_info *infos; atomic_long_t head_id; atomic_long_t tail_id; - atomic_long_t last_finalized_id; + atomic_long_t last_finalized_seq; }; /* @@ -259,7 +259,7 @@ static struct printk_ringbuffer name = { \ .infos = &_##name##_infos[0], \ .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \ .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \ - .last_finalized_id = ATOMIC_INIT(DESC0_ID(descbits)), \ + .last_finalized_seq = ATOMIC_INIT(0), \ }, \ .text_data_ring = { \ .size_bits = (avgtextbits) + (descbits), \ From 5113cf5f4c53eb2901ddb8fa70b72e8b6f4084ac Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:54 +0106 Subject: [PATCH 0108/1038] printk: ringbuffer: Clarify special lpos values For empty line records, no data blocks are created. Instead, these valid records are identified by special logical position values (in fields of @prb_desc.text_blk_lpos). Currently the macro NO_LPOS is used for empty line records. This name is confusing because it does not imply _why_ there is no data block. Rename NO_LPOS to EMPTY_LINE_LPOS so that it is clear why there is no data block. Also add comments explaining the use of EMPTY_LINE_LPOS as well as clarification to the values used to represent data-less blocks. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-6-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.c | 20 ++++++++++++++++---- kernel/printk/printk_ringbuffer.h | 16 +++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 04c26cca546f..244d991ffd73 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -1034,9 +1034,13 @@ static char *data_alloc(struct printk_ringbuffer *rb, unsigned int size, unsigned long next_lpos; if (size == 0) { - /* Specify a data-less block. */ - blk_lpos->begin = NO_LPOS; - blk_lpos->next = NO_LPOS; + /* + * Data blocks are not created for empty lines. Instead, the + * reader will recognize these special lpos values and handle + * it appropriately. + */ + blk_lpos->begin = EMPTY_LINE_LPOS; + blk_lpos->next = EMPTY_LINE_LPOS; return NULL; } @@ -1214,10 +1218,18 @@ static const char *get_data(struct prb_data_ring *data_ring, /* Data-less data block description. */ if (BLK_DATALESS(blk_lpos)) { - if (blk_lpos->begin == NO_LPOS && blk_lpos->next == NO_LPOS) { + /* + * Records that are just empty lines are also valid, even + * though they do not have a data block. For such records + * explicitly return empty string data to signify success. + */ + if (blk_lpos->begin == EMPTY_LINE_LPOS && + blk_lpos->next == EMPTY_LINE_LPOS) { *data_size = 0; return ""; } + + /* Data lost, invalid, or otherwise unavailable. */ return NULL; } diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index 2d948cc82b5b..d49460f7578e 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -127,8 +127,22 @@ enum desc_state { #define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id) #define DESC_ID_MASK (~DESC_FLAGS_MASK) #define DESC_ID(sv) ((sv) & DESC_ID_MASK) + +/* + * Special data block logical position values (for fields of + * @prb_desc.text_blk_lpos). + * + * - Bit0 is used to identify if the record has no data block. (Implemented in + * the LPOS_DATALESS() macro.) + * + * - Bit1 specifies the reason for not having a data block. + * + * These special values could never be real lpos values because of the + * meta data and alignment padding of data blocks. (See to_blk_size() for + * details.) + */ #define FAILED_LPOS 0x1 -#define NO_LPOS 0x3 +#define EMPTY_LINE_LPOS 0x3 #define FAILED_BLK_LPOS \ { \ From 0ab7cdd00491b532591ef065be706301de7e448f Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:55 +0106 Subject: [PATCH 0109/1038] printk: For @suppress_panic_printk check for other CPU in panic Currently @suppress_panic_printk is checked along with non-matching @panic_cpu and current CPU. This works because @suppress_panic_printk is only set when panic_in_progress() is true. Rather than relying on the @suppress_panic_printk semantics, use the concise helper function other_cpu_in_progress(). The helper function exists to avoid open coding such tests. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-7-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index be95a6851164..22bb45d8f2f2 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2270,8 +2270,7 @@ asmlinkage int vprintk_emit(int facility, int level, if (unlikely(suppress_printk)) return 0; - if (unlikely(suppress_panic_printk) && - atomic_read(&panic_cpu) != raw_smp_processor_id()) + if (unlikely(suppress_panic_printk) && other_cpu_in_panic()) return 0; if (level == LOGLEVEL_SCHED) { From 36652d0f3bf34899e82d31a5fa9e2bdd02fd6381 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:56 +0106 Subject: [PATCH 0110/1038] printk: Add this_cpu_in_panic() There is already panic_in_progress() and other_cpu_in_panic(), but checking if the current CPU is the panic CPU must still be open coded. Add this_cpu_in_panic() to complete the set. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-8-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/internal.h | 1 + kernel/printk/printk.c | 43 +++++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 6c2afee5ef62..ac2d9750e5f8 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -130,6 +130,7 @@ struct printk_message { }; bool other_cpu_in_panic(void); +bool this_cpu_in_panic(void); bool printk_get_next_message(struct printk_message *pmsg, u64 seq, bool is_extended, bool may_supress); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 22bb45d8f2f2..82dc2c7949b7 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -347,6 +347,29 @@ static bool panic_in_progress(void) return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID); } +/* Return true if a panic is in progress on the current CPU. */ +bool this_cpu_in_panic(void) +{ + /* + * We can use raw_smp_processor_id() here because it is impossible for + * the task to be migrated to the panic_cpu, or away from it. If + * panic_cpu has already been set, and we're not currently executing on + * that CPU, then we never will be. + */ + return unlikely(atomic_read(&panic_cpu) == raw_smp_processor_id()); +} + +/* + * Return true if a panic is in progress on a remote CPU. + * + * On true, the local CPU should immediately release any printing resources + * that may be needed by the panic CPU. + */ +bool other_cpu_in_panic(void) +{ + return (panic_in_progress() && !this_cpu_in_panic()); +} + /* * This is used for debugging the mess that is the VT code by * keeping track if we have the console semaphore held. It's @@ -2592,26 +2615,6 @@ static int console_cpu_notify(unsigned int cpu) return 0; } -/* - * Return true if a panic is in progress on a remote CPU. - * - * On true, the local CPU should immediately release any printing resources - * that may be needed by the panic CPU. - */ -bool other_cpu_in_panic(void) -{ - if (!panic_in_progress()) - return false; - - /* - * We can use raw_smp_processor_id() here because it is impossible for - * the task to be migrated to the panic_cpu, or away from it. If - * panic_cpu has already been set, and we're not currently executing on - * that CPU, then we never will be. - */ - return atomic_read(&panic_cpu) != raw_smp_processor_id(); -} - /** * console_lock - block the console subsystem from printing * From 584528d621459d1a5c31da7a591218ad3bb96d6c Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:57 +0106 Subject: [PATCH 0111/1038] printk: ringbuffer: Cleanup reader terminology With the lockless ringbuffer, it is allowed that multiple CPUs/contexts write simultaneously into the buffer. This creates an ambiguity as some writers will finalize sooner. The documentation for the prb_read functions is not clear as it refers to "not yet written" and "no data available". Clarify the return values and language to be in terms of the reader: records available for reading. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-9-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 244d991ffd73..67ee1c62fcd6 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -1987,11 +1987,13 @@ u64 prb_first_seq(struct printk_ringbuffer *rb) } /* - * Non-blocking read of a record. Updates @seq to the last finalized record - * (which may have no data available). + * Non-blocking read of a record. * - * See the description of prb_read_valid() and prb_read_valid_info() - * for details. + * On success @seq is updated to the record that was read and (if provided) + * @r and @line_count will contain the read/calculated data. + * + * On failure @seq is updated to a record that is not yet available to the + * reader, but it will be the next record available to the reader. */ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, struct printk_record *r, unsigned int *line_count) @@ -2010,7 +2012,7 @@ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, *seq = tail_seq; } else if (err == -ENOENT) { - /* Record exists, but no data available. Skip. */ + /* Record exists, but the data was lost. Skip. */ (*seq)++; } else { @@ -2043,7 +2045,7 @@ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, * On success, the reader must check r->info.seq to see which record was * actually read. This allows the reader to detect dropped records. * - * Failure means @seq refers to a not yet written record. + * Failure means @seq refers to a record not yet available to the reader. */ bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, struct printk_record *r) @@ -2073,7 +2075,7 @@ bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, * On success, the reader must check info->seq to see which record meta data * was actually read. This allows the reader to detect dropped records. * - * Failure means @seq refers to a not yet written record. + * Failure means @seq refers to a record not yet available to the reader. */ bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, struct printk_info *info, unsigned int *line_count) From ac7d7844c64d15603daa3e905a311ddcfbb4bc91 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:58 +0106 Subject: [PATCH 0112/1038] printk: Wait for all reserved records with pr_flush() Currently pr_flush() will only wait for records that were available to readers at the time of the call (using prb_next_seq()). But there may be more records (non-finalized) that have following finalized records. pr_flush() should wait for these to print as well. Particularly because any trailing finalized records may be the messages that the calling context wants to ensure are printed. Add a new ringbuffer function prb_next_reserve_seq() to return the sequence number following the most recently reserved record. This guarantees that pr_flush() will wait until all current printk() messages (completed or in progress) have been printed. Fixes: 3b604ca81202 ("printk: add pr_flush()") Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-10-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 2 +- kernel/printk/printk_ringbuffer.c | 105 ++++++++++++++++++++++++++++++ kernel/printk/printk_ringbuffer.h | 1 + 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 82dc2c7949b7..f3a7f5a6f6f8 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3755,7 +3755,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre might_sleep(); - seq = prb_next_seq(prb); + seq = prb_next_reserve_seq(prb); /* Flush the consoles so that records up to @seq are printed. */ console_lock(); diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 67ee1c62fcd6..24b484c221e8 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -1986,6 +1986,111 @@ u64 prb_first_seq(struct printk_ringbuffer *rb) return seq; } +/** + * prb_next_reserve_seq() - Get the sequence number after the most recently + * reserved record. + * + * @rb: The ringbuffer to get the sequence number from. + * + * This is the public function available to readers to see what sequence + * number will be assigned to the next reserved record. + * + * Note that depending on the situation, this value can be equal to or + * higher than the sequence number returned by prb_next_seq(). + * + * Context: Any context. + * Return: The sequence number that will be assigned to the next record + * reserved. + */ +u64 prb_next_reserve_seq(struct printk_ringbuffer *rb) +{ + struct prb_desc_ring *desc_ring = &rb->desc_ring; + unsigned long last_finalized_id; + atomic_long_t *state_var; + u64 last_finalized_seq; + unsigned long head_id; + struct prb_desc desc; + unsigned long diff; + struct prb_desc *d; + int err; + + /* + * It may not be possible to read a sequence number for @head_id. + * So the ID of @last_finailzed_seq is used to calculate what the + * sequence number of @head_id will be. + */ + +try_again: + last_finalized_seq = desc_last_finalized_seq(rb); + + /* + * @head_id is loaded after @last_finalized_seq to ensure that + * it points to the record with @last_finalized_seq or newer. + * + * Memory barrier involvement: + * + * If desc_last_finalized_seq:A reads from + * desc_update_last_finalized:A, then + * prb_next_reserve_seq:A reads from desc_reserve:D. + * + * Relies on: + * + * RELEASE from desc_reserve:D to desc_update_last_finalized:A + * matching + * ACQUIRE from desc_last_finalized_seq:A to prb_next_reserve_seq:A + * + * Note: desc_reserve:D and desc_update_last_finalized:A can be + * different CPUs. However, the desc_update_last_finalized:A CPU + * (which performs the release) must have previously seen + * desc_read:C, which implies desc_reserve:D can be seen. + */ + head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_next_reserve_seq:A) */ + + d = to_desc(desc_ring, last_finalized_seq); + state_var = &d->state_var; + + /* Extract the ID, used to specify the descriptor to read. */ + last_finalized_id = DESC_ID(atomic_long_read(state_var)); + + /* Ensure @last_finalized_id is correct. */ + err = desc_read_finalized_seq(desc_ring, last_finalized_id, last_finalized_seq, &desc); + + if (err == -EINVAL) { + if (last_finalized_seq == 0) { + /* + * No record has been finalized or even reserved yet. + * + * The @head_id is initialized such that the first + * increment will yield the first record (seq=0). + * Handle it separately to avoid a negative @diff + * below. + */ + if (head_id == DESC0_ID(desc_ring->count_bits)) + return 0; + + /* + * One or more descriptors are already reserved. Use + * the descriptor ID of the first one (@seq=0) for + * the @diff below. + */ + last_finalized_id = DESC0_ID(desc_ring->count_bits) + 1; + } else { + /* Record must have been overwritten. Try again. */ + goto try_again; + } + } + + /* Diff of known descriptor IDs to compute related sequence numbers. */ + diff = head_id - last_finalized_id; + + /* + * @head_id points to the most recently reserved record, but this + * function returns the sequence number that will be assigned to the + * next (not yet reserved) record. Thus +1 is needed. + */ + return (last_finalized_seq + diff + 1); +} + /* * Non-blocking read of a record. * diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index d49460f7578e..52626d0f1fa3 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -395,6 +395,7 @@ bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, u64 prb_first_seq(struct printk_ringbuffer *rb); u64 prb_first_valid_seq(struct printk_ringbuffer *rb); u64 prb_next_seq(struct printk_ringbuffer *rb); +u64 prb_next_reserve_seq(struct printk_ringbuffer *rb); #ifdef CONFIG_64BIT From b1c4c67a5e90db8fbdb5b5504fe16e17b564cca8 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:46:59 +0106 Subject: [PATCH 0113/1038] printk: ringbuffer: Skip non-finalized records in panic Normally a reader will stop once reaching a non-finalized record. However, when a panic happens, writers from other CPUs (or an interrupted context on the panic CPU) may have been writing a record and were unable to finalize it. The panic CPU will reserve/commit/finalize its panic records, but these will be located after the non-finalized records. This results in panic() not flushing the panic messages. Extend _prb_read_valid() to skip over non-finalized records if on the panic CPU. Fixes: 896fbe20b4e2 ("printk: use the lockless ringbuffer") Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-11-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk_ringbuffer.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 24b484c221e8..88e8f3a61922 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -2099,6 +2099,10 @@ try_again: * * On failure @seq is updated to a record that is not yet available to the * reader, but it will be the next record available to the reader. + * + * Note: When the current CPU is in panic, this function will skip over any + * non-existent/non-finalized records in order to allow the panic CPU + * to print any and all records that have been finalized. */ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, struct printk_record *r, unsigned int *line_count) @@ -2121,8 +2125,28 @@ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, (*seq)++; } else { - /* Non-existent/non-finalized record. Must stop. */ - return false; + /* + * Non-existent/non-finalized record. Must stop. + * + * For panic situations it cannot be expected that + * non-finalized records will become finalized. But + * there may be other finalized records beyond that + * need to be printed for a panic situation. If this + * is the panic CPU, skip this + * non-existent/non-finalized record unless it is + * at or beyond the head, in which case it is not + * possible to continue. + * + * Note that new messages printed on panic CPU are + * finalized when we are here. The only exception + * might be the last message without trailing newline. + * But it would have the sequence number returned + * by "prb_next_reserve_seq() - 1". + */ + if (this_cpu_in_panic() && ((*seq + 1) < prb_next_reserve_seq(rb))) + (*seq)++; + else + return false; } } From d04d5882cd678b898a9d7c5aee6afbe9e6e77fcd Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Wed, 7 Feb 2024 14:47:00 +0106 Subject: [PATCH 0114/1038] printk: Disable passing console lock owner completely during panic() The commit d51507098ff91 ("printk: disable optimistic spin during panic") added checks to avoid becoming a console waiter if a panic is in progress. However, the transition to panic can occur while there is already a waiter. The current owner should not pass the lock to the waiter because it might get stopped or blocked anytime. Also the panic context might pass the console lock owner to an already stopped waiter by mistake. It might happen when console_flush_on_panic() ignores the current lock owner, for example: CPU0 CPU1 ---- ---- console_lock_spinning_enable() console_trylock_spinning() [CPU1 now console waiter] NMI: panic() panic_other_cpus_shutdown() [stopped as console waiter] console_flush_on_panic() console_lock_spinning_enable() [print 1 record] console_lock_spinning_disable_and_check() [handover to stopped CPU1] This results in panic() not flushing the panic messages. Fix these problems by disabling all spinning operations completely during panic(). Another advantage is that it prevents possible deadlocks caused by "console_owner_lock". The panic() context does not need to take it any longer. The lockless checks are safe because the functions become NOPs when they see the panic in progress. All operations manipulating the state are still synchronized by the lock even when non-panic CPUs would notice the panic synchronously. The current owner might stay spinning. But non-panic() CPUs would get stopped anyway and the panic context will never start spinning. Fixes: dbdda842fe96 ("printk: Add console owner and waiter logic to load balance console writes") Signed-off-by: John Ogness Link: https://lore.kernel.org/r/20240207134103.1357162-12-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index f3a7f5a6f6f8..cb99c854a648 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1869,10 +1869,23 @@ static bool console_waiter; */ static void console_lock_spinning_enable(void) { + /* + * Do not use spinning in panic(). The panic CPU wants to keep the lock. + * Non-panic CPUs abandon the flush anyway. + * + * Just keep the lockdep annotation. The panic-CPU should avoid + * taking console_owner_lock because it might cause a deadlock. + * This looks like the easiest way how to prevent false lockdep + * reports without handling races a lockless way. + */ + if (panic_in_progress()) + goto lockdep; + raw_spin_lock(&console_owner_lock); console_owner = current; raw_spin_unlock(&console_owner_lock); +lockdep: /* The waiter may spin on us after setting console_owner */ spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_); } @@ -1897,6 +1910,22 @@ static int console_lock_spinning_disable_and_check(int cookie) { int waiter; + /* + * Ignore spinning waiters during panic() because they might get stopped + * or blocked at any time, + * + * It is safe because nobody is allowed to start spinning during panic + * in the first place. If there has been a waiter then non panic CPUs + * might stay spinning. They would get stopped anyway. The panic context + * will never start spinning and an interrupted spin on panic CPU will + * never continue. + */ + if (panic_in_progress()) { + /* Keep lockdep happy. */ + spin_release(&console_owner_dep_map, _THIS_IP_); + return 0; + } + raw_spin_lock(&console_owner_lock); waiter = READ_ONCE(console_waiter); console_owner = NULL; From 779dbc2e78d746250f07c463e13ac603b349f6f3 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:47:01 +0106 Subject: [PATCH 0115/1038] printk: Avoid non-panic CPUs writing to ringbuffer Commit 13fb0f74d702 ("printk: Avoid livelock with heavy printk during panic") introduced a mechanism to silence non-panic CPUs if too many messages are being dropped. Aside from trying to workaround the livelock bugs of legacy consoles, it was also intended to avoid losing panic messages. However, if non-panic CPUs are writing to the ringbuffer, then reacting to dropped messages is too late. Another motivation is that non-finalized messages already might be skipped in panic(). In other words, random messages from non-panic CPUs might already get lost. It is better to ignore all to avoid confusion. To avoid losing panic CPU messages, silence non-panic CPUs immediately on panic. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-13-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/printk/printk.c | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index cb99c854a648..1685a71f3f71 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -462,12 +462,6 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; static DEFINE_MUTEX(syslog_lock); #ifdef CONFIG_PRINTK -/* - * During panic, heavy printk by other CPUs can delay the - * panic and risk deadlock on console resources. - */ -static int __read_mostly suppress_panic_printk; - DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ /* the next printk record to read by syslog(READ) or /proc/kmsg */ @@ -2322,7 +2316,12 @@ asmlinkage int vprintk_emit(int facility, int level, if (unlikely(suppress_printk)) return 0; - if (unlikely(suppress_panic_printk) && other_cpu_in_panic()) + /* + * The messages on the panic CPU are the most important. If + * non-panic CPUs are generating any messages, they will be + * silently dropped. + */ + if (other_cpu_in_panic()) return 0; if (level == LOGLEVEL_SCHED) { @@ -2799,8 +2798,6 @@ void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) bool printk_get_next_message(struct printk_message *pmsg, u64 seq, bool is_extended, bool may_suppress) { - static int panic_console_dropped; - struct printk_buffers *pbufs = pmsg->pbufs; const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf); const size_t outbuf_sz = sizeof(pbufs->outbuf); @@ -2828,17 +2825,6 @@ bool printk_get_next_message(struct printk_message *pmsg, u64 seq, pmsg->seq = r.info->seq; pmsg->dropped = r.info->seq - seq; - /* - * Check for dropped messages in panic here so that printk - * suppression can occur as early as possible if necessary. - */ - if (pmsg->dropped && - panic_in_progress() && - panic_console_dropped++ > 10) { - suppress_panic_printk = 1; - pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n"); - } - /* Skip record that has level above the console loglevel. */ if (may_suppress && suppress_message_printing(r.info->level)) goto out; From d988d9a9b9d180bfd5c1d353b3b176cb90d6861b Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:47:02 +0106 Subject: [PATCH 0116/1038] panic: Flush kernel log buffer at the end If the kernel crashes in a context where printk() calls always defer printing (such as in NMI or inside a printk_safe section) then the final panic messages will be deferred to irq_work. But if irq_work is not available, the messages will not get printed unless explicitly flushed. The result is that the final "end Kernel panic" banner does not get printed. Add one final flush after the last printk() call to make sure the final panic messages make it out as well. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-14-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- kernel/panic.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/panic.c b/kernel/panic.c index 2807639aab51..f22d8f33ea14 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -446,6 +446,14 @@ void panic(const char *fmt, ...) /* Do not scroll important messages printed above */ suppress_printk = 1; + + /* + * The final messages may not have been printed if in a context that + * defers printing (such as NMI) and irq_work is not available. + * Explicitly flush the kernel log buffer one last time. + */ + console_flush_on_panic(CONSOLE_FLUSH_PENDING); + local_irq_enable(); for (i = 0; ; i += PANIC_TIMER_STEP) { touch_softlockup_watchdog(); From 7412dc6d55eed6b76180e40ac3601412ebde29bd Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 7 Feb 2024 14:47:03 +0106 Subject: [PATCH 0117/1038] dump_stack: Do not get cpu_sync for panic CPU dump_stack() is called in panic(). If for some reason another CPU is holding the printk_cpu_sync and is unable to release it, the panic CPU will be unable to continue and print the stacktrace. Since non-panic CPUs are not allowed to store new printk messages anyway, there is no need to synchronize the stacktrace output in a panic situation. For the panic CPU, do not get the printk_cpu_sync because it is not needed and avoids a potential deadlock scenario in panic(). Link: https://lore.kernel.org/lkml/ZcIGKU8sxti38Kok@alley Signed-off-by: John Ogness Reviewed-by: Petr Mladek Link: https://lore.kernel.org/r/20240207134103.1357162-15-john.ogness@linutronix.de Signed-off-by: Petr Mladek --- include/linux/printk.h | 2 ++ kernel/printk/internal.h | 1 - lib/dump_stack.c | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/include/linux/printk.h b/include/linux/printk.h index 8ef499ab3c1e..955e31860095 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -273,6 +273,8 @@ static inline void printk_trigger_flush(void) } #endif +bool this_cpu_in_panic(void); + #ifdef CONFIG_SMP extern int __printk_cpu_sync_try_get(void); extern void __printk_cpu_sync_wait(void); diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index ac2d9750e5f8..6c2afee5ef62 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -130,7 +130,6 @@ struct printk_message { }; bool other_cpu_in_panic(void); -bool this_cpu_in_panic(void); bool printk_get_next_message(struct printk_message *pmsg, u64 seq, bool is_extended, bool may_supress); diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 83471e81501a..222c6d6c8281 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -96,15 +96,25 @@ static void __dump_stack(const char *log_lvl) */ asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { + bool in_panic = this_cpu_in_panic(); unsigned long flags; /* * Permit this cpu to perform nested stack dumps while serialising - * against other CPUs + * against other CPUs, unless this CPU is in panic. + * + * When in panic, non-panic CPUs are not permitted to store new + * printk messages so there is no need to synchronize the output. + * This avoids potential deadlock in panic() if another CPU is + * holding and unable to release the printk_cpu_sync. */ - printk_cpu_sync_get_irqsave(flags); + if (!in_panic) + printk_cpu_sync_get_irqsave(flags); + __dump_stack(log_lvl); - printk_cpu_sync_put_irqrestore(flags); + + if (!in_panic) + printk_cpu_sync_put_irqrestore(flags); } EXPORT_SYMBOL(dump_stack_lvl); From 5f623835584f1c8d1030666796f40c47a448ce0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Sun, 4 Feb 2024 22:20:43 +0100 Subject: [PATCH 0118/1038] pwm: sti: Fix capture for st,pwm-num-chan < st,capture-num-chan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver only used the number of pwm channels to set the pwm_chip's npwm member. The result is that if there are more capture channels than PWM channels specified in the device tree, only a part of the capture channel is usable. Fix that by passing the bigger channel count to the pwm framework. This makes it possible that the .apply() callback is called with .hwpwm >= pwm_num_devs, catch that case and return an error code. Fixes: c97267ae831d ("pwm: sti: Add PWM capture callback") Link: https://lore.kernel.org/r/20240204212043.2951852-2-u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sti.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-sti.c b/drivers/pwm/pwm-sti.c index 6cf55cf34d39..69b1113c6b82 100644 --- a/drivers/pwm/pwm-sti.c +++ b/drivers/pwm/pwm-sti.c @@ -395,8 +395,17 @@ out: static int sti_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { + struct sti_pwm_chip *pc = to_sti_pwmchip(chip); + struct sti_pwm_compat_data *cdata = pc->cdata; + struct device *dev = pc->dev; int err; + if (pwm->hwpwm >= cdata->pwm_num_devs) { + dev_err(dev, "device %u is not valid for pwm mode\n", + pwm->hwpwm); + return -EINVAL; + } + if (state->polarity != PWM_POLARITY_NORMAL) return -EINVAL; @@ -646,7 +655,7 @@ static int sti_pwm_probe(struct platform_device *pdev) pc->chip.dev = dev; pc->chip.ops = &sti_pwm_ops; - pc->chip.npwm = pc->cdata->pwm_num_devs; + pc->chip.npwm = max(cdata->pwm_num_devs, cdata->cpt_num_devs); for (i = 0; i < cdata->cpt_num_devs; i++) { struct sti_cpt_ddata *ddata = &cdata->ddata[i]; From aab5c6f200238ac45001bec3d5494fff8438a8dc Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 5 Feb 2024 11:08:11 +0100 Subject: [PATCH 0119/1038] gpio: set device type for GPIO chips It's useful to have the device type information for those sub-devices that are actually GPIO chips registered with GPIOLIB. While at it: use the device type struct to setup the release callback which is the preferred way to use the device API. Signed-off-by: Bartosz Golaszewski Reviewed-by: Greg Kroah-Hartman --- drivers/gpio/gpiolib.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 24d046268a01..3cad49363a72 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -663,6 +663,11 @@ static void gpiodev_release(struct device *dev) kfree(gdev); } +static const struct device_type gpio_dev_type = { + .name = "gpio_chip", + .release = gpiodev_release, +}; + #ifdef CONFIG_GPIO_CDEV #define gcdev_register(gdev, devt) gpiolib_cdev_register((gdev), (devt)) #define gcdev_unregister(gdev) gpiolib_cdev_unregister((gdev)) @@ -680,6 +685,8 @@ static int gpiochip_setup_dev(struct gpio_device *gdev) struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev); int ret; + device_initialize(&gdev->dev); + /* * If fwnode doesn't belong to another device, it's safe to clear its * initialized flag. @@ -691,9 +698,6 @@ static int gpiochip_setup_dev(struct gpio_device *gdev) if (ret) return ret; - /* From this point, the .release() function cleans up gpio_device */ - gdev->dev.release = gpiodev_release; - ret = gpiochip_sysfs_register(gdev); if (ret) goto err_remove_device; @@ -825,6 +829,8 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev = kzalloc(sizeof(*gdev), GFP_KERNEL); if (!gdev) return -ENOMEM; + + gdev->dev.type = &gpio_dev_type; gdev->dev.bus = &gpio_bus_type; gdev->dev.parent = gc->parent; gdev->chip = gc; @@ -851,7 +857,6 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_free_ida; - device_initialize(&gdev->dev); if (gc->parent && gc->parent->driver) gdev->owner = gc->parent->driver->owner; else if (gc->owner) From 227ab73b89d66e3064b3c2bcb5fe382b1815763d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:15 +0100 Subject: [PATCH 0120/1038] fpga: ice40-spi: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Xu Yilun Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/6a3ddedc955cf55dddbc68ccf5ff5e46af5e8b6f.1707324793.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/fpga/ice40-spi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/fpga/ice40-spi.c b/drivers/fpga/ice40-spi.c index 7cbb3558b844..c0028ae4c5b7 100644 --- a/drivers/fpga/ice40-spi.c +++ b/drivers/fpga/ice40-spi.c @@ -66,7 +66,7 @@ static int ice40_fpga_ops_write_init(struct fpga_manager *mgr, } /* Lock the bus, assert CRESET_B and SS_B and delay >200ns */ - spi_bus_lock(dev->master); + spi_bus_lock(dev->controller); gpiod_set_value(priv->reset, 1); @@ -94,7 +94,7 @@ static int ice40_fpga_ops_write_init(struct fpga_manager *mgr, ret = spi_sync_locked(dev, &message); fail: - spi_bus_unlock(dev->master); + spi_bus_unlock(dev->controller); return ret; } From 167b78446706bb4d19f7dd93ca320aed25ae1bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:16 +0100 Subject: [PATCH 0121/1038] ieee802154: ca8210: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Reviewed-by: Simon Horman Acked-by: Stefan Schmidt Acked-by: Miquel Raynal Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/dec96b1d3cf43bb442128957b27d02007ffeb68d.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/net/ieee802154/ca8210.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index 4ec0dab38872..f732c150462b 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -2956,7 +2956,7 @@ static int ca8210_test_interface_init(struct ca8210_priv *priv) node_name, sizeof(node_name), "ca8210@%d_%d", - priv->spi->master->bus_num, + priv->spi->controller->bus_num, spi_get_chipselect(priv->spi, 0) ); From 2780e7b716a605781dbee753ef4983d775a65427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:17 +0100 Subject: [PATCH 0122/1038] iio: adc: ad_sigma_delta: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Acked-by: Nuno Sa Link: https://lore.kernel.org/r/de866d09c80c89816df62602ba4ba8ba30e9971f.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/iio/adc/ad_sigma_delta.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c index 7e2192870743..55442eddf57c 100644 --- a/drivers/iio/adc/ad_sigma_delta.c +++ b/drivers/iio/adc/ad_sigma_delta.c @@ -212,7 +212,7 @@ int ad_sd_calibrate(struct ad_sigma_delta *sigma_delta, if (ret) return ret; - spi_bus_lock(sigma_delta->spi->master); + spi_bus_lock(sigma_delta->spi->controller); sigma_delta->bus_locked = true; sigma_delta->keep_cs_asserted = true; reinit_completion(&sigma_delta->completion); @@ -235,7 +235,7 @@ out: sigma_delta->keep_cs_asserted = false; ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); sigma_delta->bus_locked = false; - spi_bus_unlock(sigma_delta->spi->master); + spi_bus_unlock(sigma_delta->spi->controller); return ret; } @@ -287,7 +287,7 @@ int ad_sigma_delta_single_conversion(struct iio_dev *indio_dev, ad_sigma_delta_set_channel(sigma_delta, chan->address); - spi_bus_lock(sigma_delta->spi->master); + spi_bus_lock(sigma_delta->spi->controller); sigma_delta->bus_locked = true; sigma_delta->keep_cs_asserted = true; reinit_completion(&sigma_delta->completion); @@ -322,7 +322,7 @@ out: sigma_delta->keep_cs_asserted = false; ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); sigma_delta->bus_locked = false; - spi_bus_unlock(sigma_delta->spi->master); + spi_bus_unlock(sigma_delta->spi->controller); iio_device_release_direct_mode(indio_dev); if (ret) @@ -387,7 +387,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) sigma_delta->samples_buf = samples_buf; - spi_bus_lock(sigma_delta->spi->master); + spi_bus_lock(sigma_delta->spi->controller); sigma_delta->bus_locked = true; sigma_delta->keep_cs_asserted = true; @@ -401,7 +401,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) return 0; err_unlock: - spi_bus_unlock(sigma_delta->spi->master); + spi_bus_unlock(sigma_delta->spi->controller); return ret; } @@ -426,7 +426,7 @@ static int ad_sd_buffer_postdisable(struct iio_dev *indio_dev) ad_sigma_delta_disable_all(sigma_delta); sigma_delta->bus_locked = false; - return spi_bus_unlock(sigma_delta->spi->master); + return spi_bus_unlock(sigma_delta->spi->controller); } static irqreturn_t ad_sd_trigger_handler(int irq, void *p) From a78acec53b8524593afeed7258a442adc3450818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:18 +0100 Subject: [PATCH 0123/1038] Input: pxspad - follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Dmitry Torokhov Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/820fa151077dc192391d546aec35328680803f9f.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/input/joystick/psxpad-spi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/input/joystick/psxpad-spi.c b/drivers/input/joystick/psxpad-spi.c index de734a927b4d..c47fc5f34bd0 100644 --- a/drivers/input/joystick/psxpad-spi.c +++ b/drivers/input/joystick/psxpad-spi.c @@ -342,8 +342,8 @@ static int psxpad_spi_probe(struct spi_device *spi) spi->mode = SPI_MODE_3; spi->bits_per_word = 8; /* (PlayStation 1/2 joypad might be possible works 250kHz/500kHz) */ - spi->master->min_speed_hz = 125000; - spi->master->max_speed_hz = 125000; + spi->controller->min_speed_hz = 125000; + spi->controller->max_speed_hz = 125000; spi_setup(spi); /* pad settings */ From 1245633c61baf159fcc1303d7f0855f49831b9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:19 +0100 Subject: [PATCH 0124/1038] Input: synaptics-rmi4 - follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Dmitry Torokhov Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/b88413456c970a97319fa7e505768bf82966541b.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/input/rmi4/rmi_spi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/input/rmi4/rmi_spi.c b/drivers/input/rmi4/rmi_spi.c index 07c866f42296..9d92129aa432 100644 --- a/drivers/input/rmi4/rmi_spi.c +++ b/drivers/input/rmi4/rmi_spi.c @@ -375,7 +375,7 @@ static int rmi_spi_probe(struct spi_device *spi) struct rmi_device_platform_data *spi_pdata = spi->dev.platform_data; int error; - if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) + if (spi->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) return -EINVAL; rmi_spi = devm_kzalloc(&spi->dev, sizeof(struct rmi_spi_xport), From 2c2f93fbfba7186cc081e23120f169eac3b5b62a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:20 +0100 Subject: [PATCH 0125/1038] media: mgb4: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Reviewed-by: Martin Tůma Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/ccad7ad4176265bf90af92add18a0242652f5a84.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/media/pci/mgb4/mgb4_core.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/media/pci/mgb4/mgb4_core.c b/drivers/media/pci/mgb4/mgb4_core.c index 5bfb8a06202e..9bcf10a77fd3 100644 --- a/drivers/media/pci/mgb4/mgb4_core.c +++ b/drivers/media/pci/mgb4/mgb4_core.c @@ -144,7 +144,7 @@ static int match_spi_adap(struct device *dev, void *data) return to_spi_device(dev) ? 1 : 0; } -static struct spi_master *get_spi_adap(struct platform_device *pdev) +static struct spi_controller *get_spi_adap(struct platform_device *pdev) { struct device *dev; @@ -152,7 +152,7 @@ static struct spi_master *get_spi_adap(struct platform_device *pdev) dev = device_find_child(&pdev->dev, NULL, match_spi_adap); mutex_unlock(&pdev->dev.mutex); - return dev ? container_of(dev, struct spi_master, dev) : NULL; + return dev ? container_of(dev, struct spi_controller, dev) : NULL; } static int init_spi(struct mgb4_dev *mgbdev, u32 devid) @@ -179,7 +179,7 @@ static int init_spi(struct mgb4_dev *mgbdev, u32 devid) }; struct pci_dev *pdev = mgbdev->pdev; struct device *dev = &pdev->dev; - struct spi_master *master; + struct spi_controller *ctlr; struct spi_device *spi_dev; u32 irq; int rv, id; @@ -207,8 +207,8 @@ static int init_spi(struct mgb4_dev *mgbdev, u32 devid) return PTR_ERR(mgbdev->spi_pdev); } - master = get_spi_adap(mgbdev->spi_pdev); - if (!master) { + ctlr = get_spi_adap(mgbdev->spi_pdev); + if (!ctlr) { dev_err(dev, "failed to get SPI adapter\n"); rv = -EINVAL; goto err_pdev; @@ -242,8 +242,8 @@ static int init_spi(struct mgb4_dev *mgbdev, u32 devid) spi_info.platform_data = &mgbdev->flash_data; - spi_dev = spi_new_device(master, &spi_info); - put_device(&master->dev); + spi_dev = spi_new_device(ctlr, &spi_info); + put_device(&ctlr->dev); if (!spi_dev) { dev_err(dev, "failed to create MTD device\n"); rv = -EINVAL; From cfa13a64bd631d8f04a1c385923706fcef9a63ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:21 +0100 Subject: [PATCH 0126/1038] media: netup_unidvb: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Mauro Carvalho Chehab Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/c37fceef8c631c5b49538f3f143573b605fc2c24.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- .../media/pci/netup_unidvb/netup_unidvb_spi.c | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c b/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c index 526042d8afae..e90aa1c1584c 100644 --- a/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c +++ b/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c @@ -35,7 +35,7 @@ struct netup_spi_regs { struct netup_spi { struct device *dev; - struct spi_master *master; + struct spi_controller *ctlr; struct netup_spi_regs __iomem *regs; u8 __iomem *mmio; spinlock_t lock; @@ -78,7 +78,7 @@ irqreturn_t netup_spi_interrupt(struct netup_spi *spi) reg = readw(&spi->regs->control_stat); if (!(reg & NETUP_SPI_CTRL_IRQ)) { spin_unlock_irqrestore(&spi->lock, flags); - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): not mine interrupt\n", __func__); return IRQ_NONE; } @@ -88,15 +88,15 @@ irqreturn_t netup_spi_interrupt(struct netup_spi *spi) spi->state = SPI_STATE_DONE; wake_up(&spi->waitq); spin_unlock_irqrestore(&spi->lock, flags); - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): SPI interrupt handled\n", __func__); return IRQ_HANDLED; } -static int netup_spi_transfer(struct spi_master *master, +static int netup_spi_transfer(struct spi_controller *ctlr, struct spi_message *msg) { - struct netup_spi *spi = spi_master_get_devdata(master); + struct netup_spi *spi = spi_controller_get_devdata(ctlr); struct spi_transfer *t; int result = 0; u32 tr_size; @@ -131,7 +131,7 @@ static int netup_spi_transfer(struct spi_master *master, NETUP_SPI_CTRL_START | (frag_last ? NETUP_SPI_CTRL_LAST_CS : 0), &spi->regs->control_stat); - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): control_stat 0x%04x\n", __func__, readw(&spi->regs->control_stat)); wait_event_timeout(spi->waitq, @@ -144,11 +144,11 @@ static int netup_spi_transfer(struct spi_master *master, } } else { if (spi->state == SPI_STATE_START) { - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): transfer timeout\n", __func__); } else { - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): invalid state %d\n", __func__, spi->state); } @@ -161,7 +161,7 @@ static int netup_spi_transfer(struct spi_master *master, } done: msg->status = result; - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return result; } @@ -172,30 +172,30 @@ static int netup_spi_setup(struct spi_device *spi) int netup_spi_init(struct netup_unidvb_dev *ndev) { - struct spi_master *master; + struct spi_controller *ctlr; struct netup_spi *nspi; - master = devm_spi_alloc_master(&ndev->pci_dev->dev, - sizeof(struct netup_spi)); - if (!master) { + ctlr = devm_spi_alloc_master(&ndev->pci_dev->dev, + sizeof(struct netup_spi)); + if (!ctlr) { dev_err(&ndev->pci_dev->dev, "%s(): unable to alloc SPI master\n", __func__); return -EINVAL; } - nspi = spi_master_get_devdata(master); - master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LSB_FIRST; - master->bus_num = -1; - master->num_chipselect = 1; - master->transfer_one_message = netup_spi_transfer; - master->setup = netup_spi_setup; + nspi = spi_controller_get_devdata(ctlr); + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LSB_FIRST; + ctlr->bus_num = -1; + ctlr->num_chipselect = 1; + ctlr->transfer_one_message = netup_spi_transfer; + ctlr->setup = netup_spi_setup; spin_lock_init(&nspi->lock); init_waitqueue_head(&nspi->waitq); - nspi->master = master; + nspi->ctlr = ctlr; nspi->regs = (struct netup_spi_regs __iomem *)(ndev->bmmio0 + 0x4000); writew(2, &nspi->regs->clock_divider); writew(NETUP_UNIDVB_IRQ_SPI, ndev->bmmio0 + REG_IMASK_SET); ndev->spi = nspi; - if (spi_register_master(master)) { + if (spi_register_controller(ctlr)) { ndev->spi = NULL; dev_err(&ndev->pci_dev->dev, "%s(): unable to register SPI bus\n", __func__); @@ -207,8 +207,8 @@ int netup_spi_init(struct netup_unidvb_dev *ndev) ndev->pci_bus, ndev->pci_slot, ndev->pci_func); - if (!spi_new_device(master, &netup_spi_board)) { - spi_unregister_master(master); + if (!spi_new_device(ctlr, &netup_spi_board)) { + spi_unregister_controller(ctlr); ndev->spi = NULL; dev_err(&ndev->pci_dev->dev, "%s(): unable to create SPI device\n", __func__); @@ -227,7 +227,7 @@ void netup_spi_release(struct netup_unidvb_dev *ndev) if (!spi) return; - spi_unregister_master(spi->master); + spi_unregister_controller(spi->ctlr); spin_lock_irqsave(&spi->lock, flags); reg = readw(&spi->regs->control_stat); writew(reg | NETUP_SPI_CTRL_IRQ, &spi->regs->control_stat); From dd868ae646d5770f80f90dc056d06eb2e6d39c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:22 +0100 Subject: [PATCH 0127/1038] media: usb/msi2500: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/d4dc8acc3c4105b9093e97f6337a81d11e6d30ae.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/media/usb/msi2500/msi2500.c | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c index 9759996ee6a4..5138486abfa0 100644 --- a/drivers/media/usb/msi2500/msi2500.c +++ b/drivers/media/usb/msi2500/msi2500.c @@ -107,7 +107,7 @@ struct msi2500_dev { struct video_device vdev; struct v4l2_device v4l2_dev; struct v4l2_subdev *v4l2_subdev; - struct spi_master *master; + struct spi_controller *ctlr; /* videobuf2 queue and queued buffers list */ struct vb2_queue vb_queue; @@ -574,7 +574,7 @@ static void msi2500_disconnect(struct usb_interface *intf) dev->udev = NULL; v4l2_device_disconnect(&dev->v4l2_dev); video_unregister_device(&dev->vdev); - spi_unregister_master(dev->master); + spi_unregister_controller(dev->ctlr); mutex_unlock(&dev->v4l2_lock); mutex_unlock(&dev->vb_queue_lock); @@ -1136,10 +1136,10 @@ static void msi2500_video_release(struct v4l2_device *v) kfree(dev); } -static int msi2500_transfer_one_message(struct spi_master *master, +static int msi2500_transfer_one_message(struct spi_controller *ctlr, struct spi_message *m) { - struct msi2500_dev *dev = spi_master_get_devdata(master); + struct msi2500_dev *dev = spi_controller_get_devdata(ctlr); struct spi_transfer *t; int ret = 0; u32 data; @@ -1154,7 +1154,7 @@ static int msi2500_transfer_one_message(struct spi_master *master, } m->status = ret; - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return ret; } @@ -1163,7 +1163,7 @@ static int msi2500_probe(struct usb_interface *intf, { struct msi2500_dev *dev; struct v4l2_subdev *sd; - struct spi_master *master; + struct spi_controller *ctlr; int ret; static struct spi_board_info board_info = { .modalias = "msi001", @@ -1220,30 +1220,30 @@ static int msi2500_probe(struct usb_interface *intf, } /* SPI master adapter */ - master = spi_alloc_master(dev->dev, 0); - if (master == NULL) { + ctlr = spi_alloc_master(dev->dev, 0); + if (ctlr == NULL) { ret = -ENOMEM; goto err_unregister_v4l2_dev; } - dev->master = master; - master->bus_num = -1; - master->num_chipselect = 1; - master->transfer_one_message = msi2500_transfer_one_message; - spi_master_set_devdata(master, dev); - ret = spi_register_master(master); + dev->ctlr = ctlr; + ctlr->bus_num = -1; + ctlr->num_chipselect = 1; + ctlr->transfer_one_message = msi2500_transfer_one_message; + spi_controller_set_devdata(ctlr, dev); + ret = spi_register_controller(ctlr); if (ret) { - spi_master_put(master); + spi_controller_put(ctlr); goto err_unregister_v4l2_dev; } /* load v4l2 subdevice */ - sd = v4l2_spi_new_subdev(&dev->v4l2_dev, master, &board_info); + sd = v4l2_spi_new_subdev(&dev->v4l2_dev, ctlr, &board_info); dev->v4l2_subdev = sd; if (sd == NULL) { dev_err(dev->dev, "cannot get v4l2 subdevice\n"); ret = -ENODEV; - goto err_unregister_master; + goto err_unregister_controller; } /* Register controls */ @@ -1276,8 +1276,8 @@ static int msi2500_probe(struct usb_interface *intf, return 0; err_free_controls: v4l2_ctrl_handler_free(&dev->hdl); -err_unregister_master: - spi_unregister_master(dev->master); +err_unregister_controller: + spi_unregister_controller(dev->ctlr); err_unregister_v4l2_dev: v4l2_device_unregister(&dev->v4l2_dev); err_free_mem: From d920b3a672b7f79cd13b341234aebd49233f836c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:23 +0100 Subject: [PATCH 0128/1038] media: v4l2-subdev: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into v4l2_spi_new_subdev(). Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/b6e3a53ce75dfb8a21a5958a381070944dfa0875.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/media/v4l2-core/v4l2-spi.c | 4 ++-- include/media/v4l2-common.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/media/v4l2-core/v4l2-spi.c b/drivers/media/v4l2-core/v4l2-spi.c index eadecdff7349..a7092c3930d6 100644 --- a/drivers/media/v4l2-core/v4l2-spi.c +++ b/drivers/media/v4l2-core/v4l2-spi.c @@ -34,7 +34,7 @@ void v4l2_spi_subdev_init(struct v4l2_subdev *sd, struct spi_device *spi, EXPORT_SYMBOL_GPL(v4l2_spi_subdev_init); struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, - struct spi_master *master, + struct spi_controller *ctlr, struct spi_board_info *info) { struct v4l2_subdev *sd = NULL; @@ -45,7 +45,7 @@ struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, if (info->modalias[0]) request_module(info->modalias); - spi = spi_new_device(master, info); + spi = spi_new_device(ctlr, info); if (!spi || !spi->dev.driver) goto error; diff --git a/include/media/v4l2-common.h b/include/media/v4l2-common.h index acf5be24a5ca..739b0f0fc1a0 100644 --- a/include/media/v4l2-common.h +++ b/include/media/v4l2-common.h @@ -278,13 +278,13 @@ static inline void v4l2_i2c_subdev_unregister(struct v4l2_subdev *sd) * * * @v4l2_dev: pointer to &struct v4l2_device. - * @master: pointer to struct spi_master. + * @ctlr: pointer to struct spi_controller. * @info: pointer to struct spi_board_info. * * returns a &struct v4l2_subdev pointer. */ struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, - struct spi_master *master, struct spi_board_info *info); + struct spi_controller *ctlr, struct spi_board_info *info); /** * v4l2_spi_subdev_init - Initialize a v4l2_subdev with data from an @@ -308,7 +308,7 @@ void v4l2_spi_subdev_unregister(struct v4l2_subdev *sd); static inline struct v4l2_subdev * v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, - struct spi_master *master, struct spi_board_info *info) + struct spi_controller *ctlr, struct spi_board_info *info) { return NULL; } From 26dcf09ee5d9ceba2c627ae3ba174a229f25638f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:24 +0100 Subject: [PATCH 0129/1038] misc: gehc-achc: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Greg Kroah-Hartman Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/3c93bf41d2399d06b5a379a76c8f6e877f3560b7.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/misc/gehc-achc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/misc/gehc-achc.c b/drivers/misc/gehc-achc.c index 4c9c5394da6f..b8fca4d393c6 100644 --- a/drivers/misc/gehc-achc.c +++ b/drivers/misc/gehc-achc.c @@ -65,7 +65,7 @@ static int ezport_start_programming(struct spi_device *spi, struct gpio_desc *re struct spi_transfer release_cs = { }; int ret; - spi_bus_lock(spi->master); + spi_bus_lock(spi->controller); /* assert chip select */ spi_message_init(&msg); @@ -85,16 +85,16 @@ static int ezport_start_programming(struct spi_device *spi, struct gpio_desc *re ret = spi_sync_locked(spi, &msg); fail: - spi_bus_unlock(spi->master); + spi_bus_unlock(spi->controller); return ret; } static void ezport_stop_programming(struct spi_device *spi, struct gpio_desc *reset) { /* reset without asserted chip select to return into normal mode */ - spi_bus_lock(spi->master); + spi_bus_lock(spi->controller); ezport_reset(reset); - spi_bus_unlock(spi->master); + spi_bus_unlock(spi->controller); } static int ezport_get_status_register(struct spi_device *spi) From b0a6776e53403aa380411f2a43cdefb9f00ff50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:25 +0100 Subject: [PATCH 0130/1038] mmc: mmc_spi: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Ulf Hansson Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/a9c95759cf77a19c160d1d497c76ac5770c67684.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/mmc/host/mmc_spi.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c index b8dda8160c4e..bf35761f783a 100644 --- a/drivers/mmc/host/mmc_spi.c +++ b/drivers/mmc/host/mmc_spi.c @@ -935,7 +935,7 @@ static void mmc_spi_request(struct mmc_host *mmc, struct mmc_request *mrq) #endif /* request exclusive bus access */ - spi_bus_lock(host->spi->master); + spi_bus_lock(host->spi->controller); crc_recover: /* issue command; then optionally data and stop */ @@ -967,7 +967,7 @@ crc_recover: } /* release the bus */ - spi_bus_unlock(host->spi->master); + spi_bus_unlock(host->spi->controller); mmc_request_done(host->mmc, mrq); } @@ -1157,7 +1157,7 @@ static int mmc_spi_probe(struct spi_device *spi) /* We rely on full duplex transfers, mostly to reduce * per-transfer overheads (by making fewer transfers). */ - if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) + if (spi->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) return -EINVAL; /* MMC and SD specs only seem to care that sampling is on the From 44ee998db9eef84bf005c39486566a67cb018354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:26 +0100 Subject: [PATCH 0131/1038] mtd: dataflash: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Miquel Raynal Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/38bf50b391c117621e406fa8cd00c4daef78615c.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/mtd/devices/mtd_dataflash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mtd/devices/mtd_dataflash.c b/drivers/mtd/devices/mtd_dataflash.c index 0c1b93303618..ec52277e3dd5 100644 --- a/drivers/mtd/devices/mtd_dataflash.c +++ b/drivers/mtd/devices/mtd_dataflash.c @@ -638,7 +638,7 @@ static int add_dataflash_otp(struct spi_device *spi, char *name, int nr_pages, /* name must be usable with cmdlinepart */ sprintf(priv->name, "spi%d.%d-%s", - spi->master->bus_num, spi_get_chipselect(spi, 0), + spi->controller->bus_num, spi_get_chipselect(spi, 0), name); device = &priv->mtd; From 1cc711a72ae7fd44e90839f0c8d3226664de55a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:27 +0100 Subject: [PATCH 0132/1038] net: ks8851: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Reviewed-by: Simon Horman Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/4f26761c0c86072a084b2a20439da1d0a681b07b.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/net/ethernet/micrel/ks8851_spi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c index 54f2eac11a63..2f803377c9f9 100644 --- a/drivers/net/ethernet/micrel/ks8851_spi.c +++ b/drivers/net/ethernet/micrel/ks8851_spi.c @@ -156,7 +156,7 @@ static void ks8851_rdreg(struct ks8851_net *ks, unsigned int op, txb[0] = cpu_to_le16(op | KS_SPIOP_RD); - if (kss->spidev->master->flags & SPI_CONTROLLER_HALF_DUPLEX) { + if (kss->spidev->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) { msg = &kss->spi_msg2; xfer = kss->spi_xfer2; @@ -180,7 +180,7 @@ static void ks8851_rdreg(struct ks8851_net *ks, unsigned int op, ret = spi_sync(kss->spidev, msg); if (ret < 0) netdev_err(ks->netdev, "read: spi_sync() failed\n"); - else if (kss->spidev->master->flags & SPI_CONTROLLER_HALF_DUPLEX) + else if (kss->spidev->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) memcpy(rxb, trx, rxl); else memcpy(rxb, trx + 2, rxl); From 7969b98b80c0332f940c547f84650a20aab33841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:28 +0100 Subject: [PATCH 0133/1038] net: vertexcom: mse102x: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Reviewed-by: Simon Horman Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/feacee68c00f6f976c82864ba2f10a3f0c1b99f2.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/net/ethernet/vertexcom/mse102x.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/vertexcom/mse102x.c b/drivers/net/ethernet/vertexcom/mse102x.c index aeed2a093e34..edd8b59680e5 100644 --- a/drivers/net/ethernet/vertexcom/mse102x.c +++ b/drivers/net/ethernet/vertexcom/mse102x.c @@ -664,7 +664,7 @@ static int mse102x_probe_spi(struct spi_device *spi) spi->bits_per_word = 8; spi->mode |= SPI_MODE_3; /* enforce minimum speed to ensure device functionality */ - spi->master->min_speed_hz = MIN_FREQ_HZ; + spi->controller->min_speed_hz = MIN_FREQ_HZ; if (!spi->max_speed_hz) spi->max_speed_hz = MAX_FREQ_HZ; From 85ad0ec049a771c4910c8aebb2d0bd9ce9311fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:29 +0100 Subject: [PATCH 0134/1038] platform/chrome: cros_ec_spi: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Tzung-Bi Shih Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/cfbafd8a581a3ca37134cb84cef4370959012d9d.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/platform/chrome/cros_ec_spi.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 3e88cc92e819..86a3d32a7763 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -409,7 +409,7 @@ static int do_cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev, if (!rx_buf) return -ENOMEM; - spi_bus_lock(ec_spi->spi->master); + spi_bus_lock(ec_spi->spi->controller); /* * Leave a gap between CS assertion and clocking of data to allow the @@ -469,7 +469,7 @@ static int do_cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev, final_ret = terminate_request(ec_dev); - spi_bus_unlock(ec_spi->spi->master); + spi_bus_unlock(ec_spi->spi->controller); if (!ret) ret = final_ret; @@ -554,7 +554,7 @@ static int do_cros_ec_cmd_xfer_spi(struct cros_ec_device *ec_dev, if (!rx_buf) return -ENOMEM; - spi_bus_lock(ec_spi->spi->master); + spi_bus_lock(ec_spi->spi->controller); /* Transmit phase - send our message */ debug_packet(ec_dev->dev, "out", ec_dev->dout, len); @@ -590,7 +590,7 @@ static int do_cros_ec_cmd_xfer_spi(struct cros_ec_device *ec_dev, final_ret = terminate_request(ec_dev); - spi_bus_unlock(ec_spi->spi->master); + spi_bus_unlock(ec_spi->spi->controller); if (!ret) ret = final_ret; From 2259233110d90059187c5ba75537eb93eba8417b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:30 +0100 Subject: [PATCH 0135/1038] spi: bitbang: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into the SPI bitbang controller drivers. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/f7f949feb803acb8bea75798f41371a13287f4e8.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-ath79.c | 4 +-- drivers/spi/spi-au1550.c | 2 +- drivers/spi/spi-bitbang.c | 64 ++++++++++++++++----------------- drivers/spi/spi-butterfly.c | 6 ++-- drivers/spi/spi-davinci.c | 6 ++-- drivers/spi/spi-gpio.c | 2 +- drivers/spi/spi-lm70llp.c | 6 ++-- drivers/spi/spi-oc-tiny.c | 6 ++-- drivers/spi/spi-omap-uwire.c | 4 +-- drivers/spi/spi-sh-sci.c | 10 +++--- drivers/spi/spi-xilinx.c | 4 +-- drivers/spi/spi-xtensa-xtfpga.c | 2 +- include/linux/spi/spi_bitbang.h | 2 +- 13 files changed, 59 insertions(+), 59 deletions(-) diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c index b7ada981464a..d78762d4db98 100644 --- a/drivers/spi/spi-ath79.c +++ b/drivers/spi/spi-ath79.c @@ -189,7 +189,7 @@ static int ath79_spi_probe(struct platform_device *pdev) host->num_chipselect = 3; host->mem_ops = &ath79_mem_ops; - sp->bitbang.master = host; + sp->bitbang.ctlr = host; sp->bitbang.chipselect = ath79_spi_chipselect; sp->bitbang.txrx_word[SPI_MODE_0] = ath79_spi_txrx_mode0; sp->bitbang.flags = SPI_CS_HIGH; @@ -237,7 +237,7 @@ static void ath79_spi_remove(struct platform_device *pdev) spi_bitbang_stop(&sp->bitbang); ath79_spi_disable(sp); - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); } static void ath79_spi_shutdown(struct platform_device *pdev) diff --git a/drivers/spi/spi-au1550.c b/drivers/spi/spi-au1550.c index 1011b1a8f241..825d2f1cdff8 100644 --- a/drivers/spi/spi-au1550.c +++ b/drivers/spi/spi-au1550.c @@ -800,7 +800,7 @@ static int au1550_spi_probe(struct platform_device *pdev) init_completion(&hw->host_done); - hw->bitbang.master = hw->host; + hw->bitbang.ctlr = hw->host; hw->bitbang.setup_transfer = au1550_spi_setupxfer; hw->bitbang.chipselect = au1550_spi_chipsel; hw->bitbang.txrx_bufs = au1550_spi_txrx_bufs; diff --git a/drivers/spi/spi-bitbang.c b/drivers/spi/spi-bitbang.c index ecd44016c197..a0e2204fc039 100644 --- a/drivers/spi/spi-bitbang.c +++ b/drivers/spi/spi-bitbang.c @@ -187,7 +187,7 @@ int spi_bitbang_setup(struct spi_device *spi) bool initial_setup = false; int retval; - bitbang = spi_master_get_devdata(spi->master); + bitbang = spi_controller_get_devdata(spi->controller); if (!cs) { cs = kzalloc(sizeof(*cs), GFP_KERNEL); @@ -236,7 +236,7 @@ static int spi_bitbang_bufs(struct spi_device *spi, struct spi_transfer *t) unsigned nsecs = cs->nsecs; struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi->master); + bitbang = spi_controller_get_devdata(spi->controller); if (bitbang->set_line_direction) { int err; @@ -268,11 +268,11 @@ static int spi_bitbang_bufs(struct spi_device *spi, struct spi_transfer *t) * transfer-at-a-time ones to leverage dma or fifo hardware. */ -static int spi_bitbang_prepare_hardware(struct spi_master *spi) +static int spi_bitbang_prepare_hardware(struct spi_controller *spi) { struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi); + bitbang = spi_controller_get_devdata(spi); mutex_lock(&bitbang->lock); bitbang->busy = 1; @@ -281,11 +281,11 @@ static int spi_bitbang_prepare_hardware(struct spi_master *spi) return 0; } -static int spi_bitbang_transfer_one(struct spi_master *master, +static int spi_bitbang_transfer_one(struct spi_controller *ctlr, struct spi_device *spi, struct spi_transfer *transfer) { - struct spi_bitbang *bitbang = spi_master_get_devdata(master); + struct spi_bitbang *bitbang = spi_controller_get_devdata(ctlr); int status = 0; if (bitbang->setup_transfer) { @@ -303,16 +303,16 @@ static int spi_bitbang_transfer_one(struct spi_master *master, status = -EREMOTEIO; out: - spi_finalize_current_transfer(master); + spi_finalize_current_transfer(ctlr); return status; } -static int spi_bitbang_unprepare_hardware(struct spi_master *spi) +static int spi_bitbang_unprepare_hardware(struct spi_controller *spi) { struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi); + bitbang = spi_controller_get_devdata(spi); mutex_lock(&bitbang->lock); bitbang->busy = 0; @@ -323,7 +323,7 @@ static int spi_bitbang_unprepare_hardware(struct spi_master *spi) static void spi_bitbang_set_cs(struct spi_device *spi, bool enable) { - struct spi_bitbang *bitbang = spi_master_get_devdata(spi->master); + struct spi_bitbang *bitbang = spi_controller_get_devdata(spi->controller); /* SPI core provides CS high / low, but bitbang driver * expects CS active @@ -341,10 +341,10 @@ static void spi_bitbang_set_cs(struct spi_device *spi, bool enable) int spi_bitbang_init(struct spi_bitbang *bitbang) { - struct spi_master *master = bitbang->master; + struct spi_controller *ctlr = bitbang->ctlr; bool custom_cs; - if (!master) + if (!ctlr) return -EINVAL; /* * We only need the chipselect callback if we are actually using it. @@ -352,39 +352,39 @@ int spi_bitbang_init(struct spi_bitbang *bitbang) * SPI_CONTROLLER_GPIO_SS flag is set, we always need to call the * driver-specific chipselect routine. */ - custom_cs = (!master->use_gpio_descriptors || - (master->flags & SPI_CONTROLLER_GPIO_SS)); + custom_cs = (!ctlr->use_gpio_descriptors || + (ctlr->flags & SPI_CONTROLLER_GPIO_SS)); if (custom_cs && !bitbang->chipselect) return -EINVAL; mutex_init(&bitbang->lock); - if (!master->mode_bits) - master->mode_bits = SPI_CPOL | SPI_CPHA | bitbang->flags; + if (!ctlr->mode_bits) + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | bitbang->flags; - if (master->transfer || master->transfer_one_message) + if (ctlr->transfer || ctlr->transfer_one_message) return -EINVAL; - master->prepare_transfer_hardware = spi_bitbang_prepare_hardware; - master->unprepare_transfer_hardware = spi_bitbang_unprepare_hardware; - master->transfer_one = spi_bitbang_transfer_one; + ctlr->prepare_transfer_hardware = spi_bitbang_prepare_hardware; + ctlr->unprepare_transfer_hardware = spi_bitbang_unprepare_hardware; + ctlr->transfer_one = spi_bitbang_transfer_one; /* * When using GPIO descriptors, the ->set_cs() callback doesn't even * get called unless SPI_CONTROLLER_GPIO_SS is set. */ if (custom_cs) - master->set_cs = spi_bitbang_set_cs; + ctlr->set_cs = spi_bitbang_set_cs; if (!bitbang->txrx_bufs) { bitbang->use_dma = 0; bitbang->txrx_bufs = spi_bitbang_bufs; - if (!master->setup) { + if (!ctlr->setup) { if (!bitbang->setup_transfer) bitbang->setup_transfer = spi_bitbang_setup_transfer; - master->setup = spi_bitbang_setup; - master->cleanup = spi_bitbang_cleanup; + ctlr->setup = spi_bitbang_setup; + ctlr->cleanup = spi_bitbang_cleanup; } } @@ -411,18 +411,18 @@ EXPORT_SYMBOL_GPL(spi_bitbang_init); * master methods. Those methods are the defaults if the bitbang->txrx_bufs * routine isn't initialized. * - * This routine registers the spi_master, which will process requests in a + * This routine registers the spi_controller, which will process requests in a * dedicated task, keeping IRQs unblocked most of the time. To stop * processing those requests, call spi_bitbang_stop(). * - * On success, this routine will take a reference to master. The caller is - * responsible for calling spi_bitbang_stop() to decrement the reference and - * spi_master_put() as counterpart of spi_alloc_master() to prevent a memory + * On success, this routine will take a reference to the controller. The caller + * is responsible for calling spi_bitbang_stop() to decrement the reference and + * spi_controller_put() as counterpart of spi_alloc_master() to prevent a memory * leak. */ int spi_bitbang_start(struct spi_bitbang *bitbang) { - struct spi_master *master = bitbang->master; + struct spi_controller *ctlr = bitbang->ctlr; int ret; ret = spi_bitbang_init(bitbang); @@ -432,9 +432,9 @@ int spi_bitbang_start(struct spi_bitbang *bitbang) /* driver may get busy before register() returns, especially * if someone registered boardinfo for devices */ - ret = spi_register_master(spi_master_get(master)); + ret = spi_register_controller(spi_controller_get(ctlr)); if (ret) - spi_master_put(master); + spi_controller_put(ctlr); return ret; } @@ -445,7 +445,7 @@ EXPORT_SYMBOL_GPL(spi_bitbang_start); */ void spi_bitbang_stop(struct spi_bitbang *bitbang) { - spi_unregister_master(bitbang->master); + spi_unregister_controller(bitbang->ctlr); } EXPORT_SYMBOL_GPL(spi_bitbang_stop); diff --git a/drivers/spi/spi-butterfly.c b/drivers/spi/spi-butterfly.c index 289b4454242a..1d267e6c22a4 100644 --- a/drivers/spi/spi-butterfly.c +++ b/drivers/spi/spi-butterfly.c @@ -205,7 +205,7 @@ static void butterfly_attach(struct parport *p) host->bus_num = 42; host->num_chipselect = 2; - pp->bitbang.master = host; + pp->bitbang.ctlr = host; pp->bitbang.chipselect = butterfly_chipselect; pp->bitbang.txrx_word[SPI_MODE_0] = butterfly_txrx_word_mode0; @@ -263,7 +263,7 @@ static void butterfly_attach(struct parport *p) pp->info[0].platform_data = &flash; pp->info[0].chip_select = 1; pp->info[0].controller_data = pp; - pp->dataflash = spi_new_device(pp->bitbang.master, &pp->info[0]); + pp->dataflash = spi_new_device(pp->bitbang.ctlr, &pp->info[0]); if (pp->dataflash) pr_debug("%s: dataflash at %s\n", p->name, dev_name(&pp->dataflash->dev)); @@ -308,7 +308,7 @@ static void butterfly_detach(struct parport *p) parport_release(pp->pd); parport_unregister_device(pp->pd); - spi_controller_put(pp->bitbang.master); + spi_controller_put(pp->bitbang.ctlr); } static struct parport_driver butterfly_driver = { diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index 5688be245c68..be3998104bfb 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -459,7 +459,7 @@ static bool davinci_spi_can_dma(struct spi_controller *host, static int davinci_spi_check_error(struct davinci_spi *dspi, int int_status) { - struct device *sdev = dspi->bitbang.master->dev.parent; + struct device *sdev = dspi->bitbang.ctlr->dev.parent; if (int_status & SPIFLG_TIMEOUT_MASK) { dev_err(sdev, "SPI Time-out Error\n"); @@ -742,7 +742,7 @@ static irqreturn_t davinci_spi_irq(s32 irq, void *data) static int davinci_spi_request_dma(struct davinci_spi *dspi) { - struct device *sdev = dspi->bitbang.master->dev.parent; + struct device *sdev = dspi->bitbang.ctlr->dev.parent; dspi->dma_rx = dma_request_chan(sdev, "rx"); if (IS_ERR(dspi->dma_rx)) @@ -913,7 +913,7 @@ static int davinci_spi_probe(struct platform_device *pdev) if (ret) goto free_host; - dspi->bitbang.master = host; + dspi->bitbang.ctlr = host; dspi->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(dspi->clk)) { diff --git a/drivers/spi/spi-gpio.c b/drivers/spi/spi-gpio.c index d8db4564b406..909cce109bba 100644 --- a/drivers/spi/spi-gpio.c +++ b/drivers/spi/spi-gpio.c @@ -427,7 +427,7 @@ static int spi_gpio_probe(struct platform_device *pdev) host->cleanup = spi_gpio_cleanup; bb = &spi_gpio->bitbang; - bb->master = host; + bb->ctlr = host; /* * There is some additional business, apart from driving the CS GPIO * line, that we need to do on selection. This makes the local diff --git a/drivers/spi/spi-lm70llp.c b/drivers/spi/spi-lm70llp.c index e6a65b5c8c31..f982bdebd028 100644 --- a/drivers/spi/spi-lm70llp.c +++ b/drivers/spi/spi-lm70llp.c @@ -212,7 +212,7 @@ static void spi_lm70llp_attach(struct parport *p) /* * SPI and bitbang hookup. */ - pp->bitbang.master = host; + pp->bitbang.ctlr = host; pp->bitbang.chipselect = lm70_chipselect; pp->bitbang.txrx_word[SPI_MODE_0] = lm70_txrx; pp->bitbang.flags = SPI_3WIRE; @@ -264,7 +264,7 @@ static void spi_lm70llp_attach(struct parport *p) * the board info's (void *)controller_data. */ pp->info.controller_data = pp; - pp->spidev_lm70 = spi_new_device(pp->bitbang.master, &pp->info); + pp->spidev_lm70 = spi_new_device(pp->bitbang.ctlr, &pp->info); if (pp->spidev_lm70) dev_dbg(&pp->spidev_lm70->dev, "spidev_lm70 at %s\n", dev_name(&pp->spidev_lm70->dev)); @@ -309,7 +309,7 @@ static void spi_lm70llp_detach(struct parport *p) parport_release(pp->pd); parport_unregister_device(pp->pd); - spi_controller_put(pp->bitbang.master); + spi_controller_put(pp->bitbang.ctlr); lm70llp = NULL; } diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c index cf7c111088a6..6ea38f5e7d64 100644 --- a/drivers/spi/spi-oc-tiny.c +++ b/drivers/spi/spi-oc-tiny.c @@ -194,7 +194,7 @@ static int tiny_spi_of_probe(struct platform_device *pdev) if (!np) return 0; - hw->bitbang.master->dev.of_node = pdev->dev.of_node; + hw->bitbang.ctlr->dev.of_node = pdev->dev.of_node; if (!of_property_read_u32(np, "clock-frequency", &val)) hw->freq = val; if (!of_property_read_u32(np, "baud-width", &val)) @@ -229,7 +229,7 @@ static int tiny_spi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, hw); /* setup the state for the bitbang driver */ - hw->bitbang.master = host; + hw->bitbang.ctlr = host; hw->bitbang.setup_transfer = tiny_spi_setup_transfer; hw->bitbang.txrx_bufs = tiny_spi_txrx_bufs; @@ -274,7 +274,7 @@ exit: static void tiny_spi_remove(struct platform_device *pdev) { struct tiny_spi *hw = platform_get_drvdata(pdev); - struct spi_controller *host = hw->bitbang.master; + struct spi_controller *host = hw->bitbang.ctlr; spi_bitbang_stop(&hw->bitbang); spi_controller_put(host); diff --git a/drivers/spi/spi-omap-uwire.c b/drivers/spi/spi-omap-uwire.c index 2dd1c1bcf4bf..210a98d903fa 100644 --- a/drivers/spi/spi-omap-uwire.c +++ b/drivers/spi/spi-omap-uwire.c @@ -448,7 +448,7 @@ static void uwire_off(struct uwire_spi *uwire) { uwire_write_reg(UWIRE_SR3, 0); clk_disable_unprepare(uwire->ck); - spi_controller_put(uwire->bitbang.master); + spi_controller_put(uwire->bitbang.ctlr); } static int uwire_probe(struct platform_device *pdev) @@ -493,7 +493,7 @@ static int uwire_probe(struct platform_device *pdev) host->setup = uwire_setup; host->cleanup = uwire_cleanup; - uwire->bitbang.master = host; + uwire->bitbang.ctlr = host; uwire->bitbang.chipselect = uwire_chipselect; uwire->bitbang.setup_transfer = uwire_setup_transfer; uwire->bitbang.txrx_bufs = uwire_txrx; diff --git a/drivers/spi/spi-sh-sci.c b/drivers/spi/spi-sh-sci.c index 148d615d2f38..3d560b154ad3 100644 --- a/drivers/spi/spi-sh-sci.c +++ b/drivers/spi/spi-sh-sci.c @@ -136,9 +136,9 @@ static int sh_sci_spi_probe(struct platform_device *dev) } /* setup spi bitbang adaptor */ - sp->bitbang.master = host; - sp->bitbang.master->bus_num = sp->info->bus_num; - sp->bitbang.master->num_chipselect = sp->info->num_chipselect; + sp->bitbang.ctlr = host; + sp->bitbang.ctlr->bus_num = sp->info->bus_num; + sp->bitbang.ctlr->num_chipselect = sp->info->num_chipselect; sp->bitbang.chipselect = sh_sci_spi_chipselect; sp->bitbang.txrx_word[SPI_MODE_0] = sh_sci_spi_txrx_mode0; @@ -166,7 +166,7 @@ static int sh_sci_spi_probe(struct platform_device *dev) setbits(sp, PIN_INIT, 0); iounmap(sp->membase); err1: - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); err0: return ret; } @@ -178,7 +178,7 @@ static void sh_sci_spi_remove(struct platform_device *dev) spi_bitbang_stop(&sp->bitbang); setbits(sp, PIN_INIT, 0); iounmap(sp->membase); - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); } static struct platform_driver sh_sci_spi_drv = { diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c index 12355957be97..7795328427a6 100644 --- a/drivers/spi/spi-xilinx.c +++ b/drivers/spi/spi-xilinx.c @@ -434,7 +434,7 @@ static int xilinx_spi_probe(struct platform_device *pdev) xspi = spi_controller_get_devdata(host); xspi->cs_inactive = 0xffffffff; - xspi->bitbang.master = host; + xspi->bitbang.ctlr = host; xspi->bitbang.chipselect = xilinx_spi_chipselect; xspi->bitbang.setup_transfer = xilinx_spi_setup_transfer; xspi->bitbang.txrx_bufs = xilinx_spi_txrx_bufs; @@ -516,7 +516,7 @@ static void xilinx_spi_remove(struct platform_device *pdev) /* Disable the global IPIF interrupt */ xspi->write_fn(0, regs_base + XIPIF_V123B_DGIER_OFFSET); - spi_controller_put(xspi->bitbang.master); + spi_controller_put(xspi->bitbang.ctlr); } /* work with hotplug and coldplug */ diff --git a/drivers/spi/spi-xtensa-xtfpga.c b/drivers/spi/spi-xtensa-xtfpga.c index 3c7721894376..3c2cda315397 100644 --- a/drivers/spi/spi-xtensa-xtfpga.c +++ b/drivers/spi/spi-xtensa-xtfpga.c @@ -93,7 +93,7 @@ static int xtfpga_spi_probe(struct platform_device *pdev) host->dev.of_node = pdev->dev.of_node; xspi = spi_controller_get_devdata(host); - xspi->bitbang.master = host; + xspi->bitbang.ctlr = host; xspi->bitbang.chipselect = xtfpga_spi_chipselect; xspi->bitbang.txrx_word[SPI_MODE_0] = xtfpga_spi_txrx_word; xspi->regs = devm_platform_ioremap_resource(pdev, 0); diff --git a/include/linux/spi/spi_bitbang.h b/include/linux/spi/spi_bitbang.h index 4444c2a992cb..b930eca2ef7b 100644 --- a/include/linux/spi/spi_bitbang.h +++ b/include/linux/spi/spi_bitbang.h @@ -10,7 +10,7 @@ struct spi_bitbang { u8 use_dma; u16 flags; /* extra spi->mode support */ - struct spi_master *master; + struct spi_controller *ctlr; /* setup_transfer() changes clock and/or wordsize to match settings * for this transfer; zeroes restore defaults from spi_device. From e71011dacc3413bed4118d2c42f10736ffcd762c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:31 +0100 Subject: [PATCH 0136/1038] spi: cadence-quadspi: Don't emit error message on allocation error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drivers are not supposed to emit an error message when a memory allocation fails, as the kernel is very loud in this case already anyhow. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/8da90785395664bf17d16428f46e84b95ef5cc3b.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-cadence-quadspi.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index f94e0d370d46..eb6cc861b00b 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1712,10 +1712,9 @@ static int cqspi_probe(struct platform_device *pdev) int irq; host = devm_spi_alloc_host(&pdev->dev, sizeof(*cqspi)); - if (!host) { - dev_err(&pdev->dev, "devm_spi_alloc_host failed\n"); + if (!host) return -ENOMEM; - } + host->mode_bits = SPI_RX_QUAD | SPI_RX_DUAL; host->mem_ops = &cqspi_mem_ops; host->mem_caps = &cqspi_mem_caps; From 28e59d8bf1ace0ddf05f989a48d6824d75731267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:32 +0100 Subject: [PATCH 0137/1038] spi: cadence-quadspi: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/d741960846b7f4896eeff91a4c8c7dfa8a6f8b7b.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-cadence-quadspi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index eb6cc861b00b..a5e2f7786b76 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1410,7 +1410,7 @@ static int cqspi_mem_process(struct spi_mem *mem, const struct spi_mem_op *op) static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) { int ret; - struct cqspi_st *cqspi = spi_master_get_devdata(mem->spi->master); + struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller); struct device *dev = &cqspi->pdev->dev; ret = pm_runtime_resume_and_get(dev); From 1747fbdedba8b6b3fd459895ed5d57e534549884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:33 +0100 Subject: [PATCH 0138/1038] spi: cavium: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/8aa122cffa6c8c18cc3367def872f81cd9471f2a.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-cavium.c | 6 +++--- drivers/spi/spi-cavium.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-cavium.c b/drivers/spi/spi-cavium.c index dfe224defd6e..26b8cd1c76e1 100644 --- a/drivers/spi/spi-cavium.c +++ b/drivers/spi/spi-cavium.c @@ -124,10 +124,10 @@ static int octeon_spi_do_transfer(struct octeon_spi *p, return xfer->len; } -int octeon_spi_transfer_one_message(struct spi_master *master, +int octeon_spi_transfer_one_message(struct spi_controller *ctlr, struct spi_message *msg) { - struct octeon_spi *p = spi_master_get_devdata(master); + struct octeon_spi *p = spi_controller_get_devdata(ctlr); unsigned int total_len = 0; int status = 0; struct spi_transfer *xfer; @@ -145,6 +145,6 @@ int octeon_spi_transfer_one_message(struct spi_master *master, err: msg->status = status; msg->actual_length = total_len; - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return status; } diff --git a/drivers/spi/spi-cavium.h b/drivers/spi/spi-cavium.h index 1f3ac463a20b..af53a0c31476 100644 --- a/drivers/spi/spi-cavium.h +++ b/drivers/spi/spi-cavium.h @@ -28,7 +28,7 @@ struct octeon_spi { #define OCTEON_SPI_TX(x) (x->regs.tx) #define OCTEON_SPI_DAT0(x) (x->regs.data) -int octeon_spi_transfer_one_message(struct spi_master *master, +int octeon_spi_transfer_one_message(struct spi_controller *ctlr, struct spi_message *msg); /* MPI register descriptions */ From 14cea92338a0776c1615994150e738ac0f5fbb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:34 +0100 Subject: [PATCH 0139/1038] spi: geni-qcom: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. This was already tried before in commit 8726bdcef62e ("spi: geni-qcom: switch to use modern name"), that's why this change is so small. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/895f9a42691dcea037b30dcce5e61be0b44e3a25.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-geni-qcom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index 15f84e68d4d2..37ef8c40b276 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -647,7 +647,7 @@ static void spi_geni_release_dma_chan(struct spi_geni_master *mas) static int spi_geni_init(struct spi_geni_master *mas) { - struct spi_master *spi = dev_get_drvdata(mas->dev); + struct spi_controller *spi = dev_get_drvdata(mas->dev); struct geni_se *se = &mas->se; unsigned int proto, major, minor, ver; u32 spi_tx_cfg, fifo_disable; From 2c2310c17fac13aa7e78756d7f3780c7891f9397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:35 +0100 Subject: [PATCH 0140/1038] spi: loopback-test: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/2f57c89d601d0875cd245a2e95c915b9bfcfc47d.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-loopback-test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c index bbf2015d8e5c..fee8893d2751 100644 --- a/drivers/spi/spi-loopback-test.c +++ b/drivers/spi/spi-loopback-test.c @@ -1031,8 +1031,8 @@ int spi_test_run_test(struct spi_device *spi, const struct spi_test *test, #define FOR_EACH_ALIGNMENT(var) \ for (var = 0; \ var < (test->iterate_##var ? \ - (spi->master->dma_alignment ? \ - spi->master->dma_alignment : \ + (spi->controller->dma_alignment ? \ + spi->controller->dma_alignment : \ test->iterate_##var) : \ 1); \ var++) From 8197b136bbbe64a7cab1020a4b067020e5977d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:36 +0100 Subject: [PATCH 0141/1038] spi: slave-mt27xx: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/184a5263af2ec01fd5ad9b8db667ae4d6cbec948.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-slave-mt27xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-slave-mt27xx.c b/drivers/spi/spi-slave-mt27xx.c index 6d6772974783..f1ddf4c099a3 100644 --- a/drivers/spi/spi-slave-mt27xx.c +++ b/drivers/spi/spi-slave-mt27xx.c @@ -297,7 +297,7 @@ static int mtk_spi_slave_transfer_one(struct spi_controller *ctlr, static int mtk_spi_slave_setup(struct spi_device *spi) { - struct mtk_spi_slave *mdata = spi_controller_get_devdata(spi->master); + struct mtk_spi_slave *mdata = spi_controller_get_devdata(spi->controller); u32 reg_val; reg_val = DMA_DONE_EN | DATA_DONE_EN | From d934cd6f0e5d0052772612db4b07df60cb9da387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:37 +0100 Subject: [PATCH 0142/1038] spi: spidev: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/15ebfdf58a6590962bdd51b49b2e85a69b3bf443.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spidev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 655f2c959cd4..95fb5f1c91c1 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -802,7 +802,7 @@ static int spidev_probe(struct spi_device *spi) spidev->devt = MKDEV(SPIDEV_MAJOR, minor); dev = device_create(&spidev_class, &spi->dev, spidev->devt, spidev, "spidev%d.%d", - spi->master->bus_num, spi_get_chipselect(spi, 0)); + spi->controller->bus_num, spi_get_chipselect(spi, 0)); status = PTR_ERR_OR_ZERO(dev); } else { dev_dbg(&spi->dev, "no minor number available!\n"); From bbd25d7260eeeaef89f7371cbadcd33dd7f7bff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:38 +0100 Subject: [PATCH 0143/1038] staging: fbtft: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Greg Kroah-Hartman Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/df79f22caf31d2c5ca2b15b33670323d1e1120c7.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/staging/fbtft/fbtft-core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c index 3626f429b002..68add4d598ae 100644 --- a/drivers/staging/fbtft/fbtft-core.c +++ b/drivers/staging/fbtft/fbtft-core.c @@ -794,7 +794,7 @@ int fbtft_register_framebuffer(struct fb_info *fb_info) if (par->txbuf.buf && par->txbuf.len >= 1024) sprintf(text1, ", %zu KiB buffer memory", par->txbuf.len >> 10); if (spi) - sprintf(text2, ", spi%d.%d at %d MHz", spi->master->bus_num, + sprintf(text2, ", spi%d.%d at %d MHz", spi->controller->bus_num, spi_get_chipselect(spi, 0), spi->max_speed_hz / 1000000); dev_info(fb_info->dev, "%s frame buffer, %dx%d, %d KiB video memory%s, fps=%lu%s\n", @@ -1215,7 +1215,7 @@ int fbtft_probe_common(struct fbtft_display *display, /* 9-bit SPI setup */ if (par->spi && display->buswidth == 9) { - if (par->spi->master->bits_per_word_mask & SPI_BPW_MASK(9)) { + if (par->spi->controller->bits_per_word_mask & SPI_BPW_MASK(9)) { par->spi->bits_per_word = 9; } else { dev_warn(&par->spi->dev, From ee3c668dda3d2783b0fff4091461356fe000e4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:39 +0100 Subject: [PATCH 0144/1038] staging: greybus: spi: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Viresh Kumar Acked-by: Greg Kroah-Hartman Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/f6f0568df29f9a51358e86326b917d81af8aba74.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/staging/greybus/spilib.c | 66 ++++++++++++++++---------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/drivers/staging/greybus/spilib.c b/drivers/staging/greybus/spilib.c index efb3bec58e15..34f10685139f 100644 --- a/drivers/staging/greybus/spilib.c +++ b/drivers/staging/greybus/spilib.c @@ -42,7 +42,7 @@ struct gb_spilib { #define XFER_TIMEOUT_TOLERANCE 200 -static struct spi_master *get_master_from_spi(struct gb_spilib *spi) +static struct spi_controller *get_controller_from_spi(struct gb_spilib *spi) { return gb_connection_get_data(spi->connection); } @@ -324,10 +324,10 @@ static void gb_spi_decode_response(struct gb_spilib *spi, } } -static int gb_spi_transfer_one_message(struct spi_master *master, +static int gb_spi_transfer_one_message(struct spi_controller *ctlr, struct spi_message *msg) { - struct gb_spilib *spi = spi_master_get_devdata(master); + struct gb_spilib *spi = spi_controller_get_devdata(ctlr); struct gb_connection *connection = spi->connection; struct gb_spi_transfer_response *response; struct gb_operation *operation; @@ -371,21 +371,21 @@ static int gb_spi_transfer_one_message(struct spi_master *master, out: msg->status = ret; clean_xfer_state(spi); - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return ret; } -static int gb_spi_prepare_transfer_hardware(struct spi_master *master) +static int gb_spi_prepare_transfer_hardware(struct spi_controller *ctlr) { - struct gb_spilib *spi = spi_master_get_devdata(master); + struct gb_spilib *spi = spi_controller_get_devdata(ctlr); return spi->ops->prepare_transfer_hardware(spi->parent); } -static int gb_spi_unprepare_transfer_hardware(struct spi_master *master) +static int gb_spi_unprepare_transfer_hardware(struct spi_controller *ctlr) { - struct gb_spilib *spi = spi_master_get_devdata(master); + struct gb_spilib *spi = spi_controller_get_devdata(ctlr); spi->ops->unprepare_transfer_hardware(spi->parent); @@ -440,7 +440,7 @@ static int gb_spi_get_master_config(struct gb_spilib *spi) static int gb_spi_setup_device(struct gb_spilib *spi, u8 cs) { - struct spi_master *master = get_master_from_spi(spi); + struct spi_controller *ctlr = get_controller_from_spi(spi); struct gb_spi_device_config_request request; struct gb_spi_device_config_response response; struct spi_board_info spi_board = { {0} }; @@ -471,11 +471,11 @@ static int gb_spi_setup_device(struct gb_spilib *spi, u8 cs) return -EINVAL; spi_board.mode = le16_to_cpu(response.mode); - spi_board.bus_num = master->bus_num; + spi_board.bus_num = ctlr->bus_num; spi_board.chip_select = cs; spi_board.max_speed_hz = le32_to_cpu(response.max_speed_hz); - spidev = spi_new_device(master, &spi_board); + spidev = spi_new_device(ctlr, &spi_board); if (!spidev) return -EINVAL; @@ -486,52 +486,52 @@ int gb_spilib_master_init(struct gb_connection *connection, struct device *dev, struct spilib_ops *ops) { struct gb_spilib *spi; - struct spi_master *master; + struct spi_controller *ctlr; int ret; u8 i; /* Allocate master with space for data */ - master = spi_alloc_master(dev, sizeof(*spi)); - if (!master) { + ctlr = spi_alloc_master(dev, sizeof(*spi)); + if (!ctlr) { dev_err(dev, "cannot alloc SPI master\n"); return -ENOMEM; } - spi = spi_master_get_devdata(master); + spi = spi_controller_get_devdata(ctlr); spi->connection = connection; - gb_connection_set_data(connection, master); + gb_connection_set_data(connection, ctlr); spi->parent = dev; spi->ops = ops; - /* get master configuration */ + /* get controller configuration */ ret = gb_spi_get_master_config(spi); if (ret) goto exit_spi_put; - master->bus_num = -1; /* Allow spi-core to allocate it dynamically */ - master->num_chipselect = spi->num_chipselect; - master->mode_bits = spi->mode; - master->flags = spi->flags; - master->bits_per_word_mask = spi->bits_per_word_mask; + ctlr->bus_num = -1; /* Allow spi-core to allocate it dynamically */ + ctlr->num_chipselect = spi->num_chipselect; + ctlr->mode_bits = spi->mode; + ctlr->flags = spi->flags; + ctlr->bits_per_word_mask = spi->bits_per_word_mask; /* Attach methods */ - master->cleanup = gb_spi_cleanup; - master->setup = gb_spi_setup; - master->transfer_one_message = gb_spi_transfer_one_message; + ctlr->cleanup = gb_spi_cleanup; + ctlr->setup = gb_spi_setup; + ctlr->transfer_one_message = gb_spi_transfer_one_message; if (ops && ops->prepare_transfer_hardware) { - master->prepare_transfer_hardware = + ctlr->prepare_transfer_hardware = gb_spi_prepare_transfer_hardware; } if (ops && ops->unprepare_transfer_hardware) { - master->unprepare_transfer_hardware = + ctlr->unprepare_transfer_hardware = gb_spi_unprepare_transfer_hardware; } - master->auto_runtime_pm = true; + ctlr->auto_runtime_pm = true; - ret = spi_register_master(master); + ret = spi_register_controller(ctlr); if (ret < 0) goto exit_spi_put; @@ -548,12 +548,12 @@ int gb_spilib_master_init(struct gb_connection *connection, struct device *dev, return 0; exit_spi_put: - spi_master_put(master); + spi_controller_put(ctlr); return ret; exit_spi_unregister: - spi_unregister_master(master); + spi_unregister_controller(ctlr); return ret; } @@ -561,9 +561,9 @@ EXPORT_SYMBOL_GPL(gb_spilib_master_init); void gb_spilib_master_exit(struct gb_connection *connection) { - struct spi_master *master = gb_connection_get_data(connection); + struct spi_controller *ctlr = gb_connection_get_data(connection); - spi_unregister_master(master); + spi_unregister_controller(ctlr); } EXPORT_SYMBOL_GPL(gb_spilib_master_exit); From b6af14eacc8814b0986e20507df423cebe9fd859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:40 +0100 Subject: [PATCH 0145/1038] tpm_tis_spi: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Reviewed-by: Jarkko Sakkinen Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/29000c8fc0612974242e3c23d14dd4771b92c71e.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/char/tpm/tpm_tis_spi_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/tpm/tpm_tis_spi_main.c b/drivers/char/tpm/tpm_tis_spi_main.c index c5c3197ee29f..c647de7b3709 100644 --- a/drivers/char/tpm/tpm_tis_spi_main.c +++ b/drivers/char/tpm/tpm_tis_spi_main.c @@ -146,7 +146,7 @@ static int tpm_tis_spi_transfer_full(struct tpm_tis_data *data, u32 addr, struct spi_transfer spi_xfer; u8 transfer_len; - spi_bus_lock(phy->spi_device->master); + spi_bus_lock(phy->spi_device->controller); while (len) { transfer_len = min_t(u16, len, MAX_SPI_FRAMESIZE); @@ -210,7 +210,7 @@ exit: spi_sync_locked(phy->spi_device, &m); } - spi_bus_unlock(phy->spi_device->master); + spi_bus_unlock(phy->spi_device->controller); return ret; } From 8c716f4a3d4fcbec976247e3443d36cbc24c0512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:41 +0100 Subject: [PATCH 0146/1038] usb: gadget: max3420_udc: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Reviewed-by: Herve Codina Acked-by: Greg Kroah-Hartman Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/5caf03b6f321a9870aabb9282f1f22211d052740.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/usb/gadget/udc/max3420_udc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/gadget/udc/max3420_udc.c b/drivers/usb/gadget/udc/max3420_udc.c index 89e8cf2a2a7d..7349ea774adf 100644 --- a/drivers/usb/gadget/udc/max3420_udc.c +++ b/drivers/usb/gadget/udc/max3420_udc.c @@ -1201,7 +1201,7 @@ static int max3420_probe(struct spi_device *spi) int err, irq; u8 reg[8]; - if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) { + if (spi->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) { dev_err(&spi->dev, "UDC needs full duplex to work\n"); return -EINVAL; } From b23031e730e72ec9067b7c38c25e776c5e27e116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:42 +0100 Subject: [PATCH 0147/1038] video: fbdev: mmp: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/136f59b6e272e5ff7ec210627c9c3ea27d066d51.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/video/fbdev/mmp/hw/mmp_spi.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/video/fbdev/mmp/hw/mmp_spi.c b/drivers/video/fbdev/mmp/hw/mmp_spi.c index 0f8f0312a7c4..cf23650d7f0b 100644 --- a/drivers/video/fbdev/mmp/hw/mmp_spi.c +++ b/drivers/video/fbdev/mmp/hw/mmp_spi.c @@ -32,7 +32,7 @@ static inline int lcd_spi_write(struct spi_device *spi, u32 data) int timeout = 100000, isr, ret = 0; u32 tmp; void __iomem *reg_base = (void __iomem *) - *(void **)spi_master_get_devdata(spi->master); + *(void **) spi_controller_get_devdata(spi->controller); /* clear ISR */ writel_relaxed(~SPI_IRQ_MASK, reg_base + SPU_IRQ_ISR); @@ -81,7 +81,7 @@ static inline int lcd_spi_write(struct spi_device *spi, u32 data) static int lcd_spi_setup(struct spi_device *spi) { void __iomem *reg_base = (void __iomem *) - *(void **)spi_master_get_devdata(spi->master); + *(void **) spi_controller_get_devdata(spi->controller); u32 tmp; tmp = CFG_SCLKCNT(16) | @@ -136,32 +136,32 @@ static int lcd_spi_one_transfer(struct spi_device *spi, struct spi_message *m) int lcd_spi_register(struct mmphw_ctrl *ctrl) { - struct spi_master *master; + struct spi_controller *ctlr; void **p_regbase; int err; - master = spi_alloc_master(ctrl->dev, sizeof(void *)); - if (!master) { + ctlr = spi_alloc_master(ctrl->dev, sizeof(void *)); + if (!ctlr) { dev_err(ctrl->dev, "unable to allocate SPI master\n"); return -ENOMEM; } - p_regbase = spi_master_get_devdata(master); + p_regbase = spi_controller_get_devdata(ctlr); *p_regbase = (void __force *)ctrl->reg_base; /* set bus num to 5 to avoid conflict with other spi hosts */ - master->bus_num = 5; - master->num_chipselect = 1; - master->setup = lcd_spi_setup; - master->transfer = lcd_spi_one_transfer; + ctlr->bus_num = 5; + ctlr->num_chipselect = 1; + ctlr->setup = lcd_spi_setup; + ctlr->transfer = lcd_spi_one_transfer; - err = spi_register_master(master); + err = spi_register_controller(ctlr); if (err < 0) { dev_err(ctrl->dev, "unable to register SPI master\n"); - spi_master_put(master); + spi_controller_put(ctlr); return err; } - dev_info(&master->dev, "registered\n"); + dev_info(&ctlr->dev, "registered\n"); return 0; } From 30060d57cee194d6b70283f2faf787e2fdc61b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:43 +0100 Subject: [PATCH 0148/1038] wifi: libertas: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Kalle Valo Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/971429eacddbad4c2f49b783ddd68746c1929ab3.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/net/wireless/marvell/libertas/if_spi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/marvell/libertas/if_spi.c b/drivers/net/wireless/marvell/libertas/if_spi.c index 8690b0114e23..b722a6587fd3 100644 --- a/drivers/net/wireless/marvell/libertas/if_spi.c +++ b/drivers/net/wireless/marvell/libertas/if_spi.c @@ -1052,7 +1052,7 @@ static int if_spi_init_card(struct if_spi_card *card) "attached to SPI bus_num %d, chip_select %d. " "spi->max_speed_hz=%d\n", card->card_id, card->card_rev, - card->spi->master->bus_num, + card->spi->controller->bus_num, spi_get_chipselect(card->spi, 0), card->spi->max_speed_hz); err = if_spi_prog_helper_firmware(card, helper); From 801185efa2402dce57828930e9684884fc8d62da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:44 +0100 Subject: [PATCH 0149/1038] spi: fsl-lib: Follow renaming of SPI "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. To not break all drivers compatibility macros were provided. To be able to remove these compatibility macros push the renaming into this driver. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/b3d724ca2b641f1d717a7ddb9810bdf27168dc35.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-fsl-lib.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/spi/spi-fsl-lib.c b/drivers/spi/spi-fsl-lib.c index 885757c29fbb..4fc2c56555b5 100644 --- a/drivers/spi/spi-fsl-lib.c +++ b/drivers/spi/spi-fsl-lib.c @@ -82,18 +82,18 @@ void mpc8xxx_spi_probe(struct device *dev, struct resource *mem, unsigned int irq) { struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); - struct spi_master *master; + struct spi_controller *ctlr; struct mpc8xxx_spi *mpc8xxx_spi; - master = dev_get_drvdata(dev); + ctlr = dev_get_drvdata(dev); /* the spi->mode bits understood by this driver: */ - master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LSB_FIRST | SPI_LOOP; - master->dev.of_node = dev->of_node; + ctlr->dev.of_node = dev->of_node; - mpc8xxx_spi = spi_master_get_devdata(master); + mpc8xxx_spi = spi_controller_get_devdata(ctlr); mpc8xxx_spi->dev = dev; mpc8xxx_spi->get_rx = mpc8xxx_spi_rx_buf_u8; mpc8xxx_spi->get_tx = mpc8xxx_spi_tx_buf_u8; @@ -104,8 +104,8 @@ void mpc8xxx_spi_probe(struct device *dev, struct resource *mem, mpc8xxx_spi->rx_shift = 0; mpc8xxx_spi->tx_shift = 0; - master->bus_num = pdata->bus_num; - master->num_chipselect = pdata->max_chipselect; + ctlr->bus_num = pdata->bus_num; + ctlr->num_chipselect = pdata->max_chipselect; init_completion(&mpc8xxx_spi->done); } From 620d269f29a569ba37419cc03cf1da2d55f6252a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:45 +0100 Subject: [PATCH 0150/1038] spi: Drop compat layer from renaming "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that all in-tree users followed the rename, the compat stuff can go away. This completes the renaming started with commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/ad1d949325b61a4682e8d6ecf9d05da751e6a99f.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- .../driver-api/driver-model/devres.rst | 2 +- drivers/spi/spi.c | 2 +- include/linux/spi/spi.h | 20 +------------------ 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index c5f99d834ec5..49c6fd82cd5d 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -462,7 +462,7 @@ SLAVE DMA ENGINE SPI devm_spi_alloc_master() devm_spi_alloc_slave() - devm_spi_register_master() + devm_spi_register_controller() WATCHDOG devm_watchdog_register_device() diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index f2170f4b5077..4b3e7a624de2 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -584,7 +584,7 @@ struct spi_device *spi_alloc_device(struct spi_controller *ctlr) return NULL; } - spi->master = spi->controller = ctlr; + spi->controller = ctlr; spi->dev.parent = &ctlr->dev; spi->dev.bus = &spi_bus_type; spi->dev.release = spidev_release; diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 600fbd5daf68..30ada46b51cd 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -131,7 +131,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * struct spi_device - Controller side proxy for an SPI slave device * @dev: Driver model representation of the device. * @controller: SPI controller used with the device. - * @master: Copy of controller, for backwards compatibility. * @max_speed_hz: Maximum clock rate to be used with this chip * (on this board); may be changed by the device's driver. * The spi_transfer.speed_hz can override this for each transfer. @@ -185,7 +184,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, struct spi_device { struct device dev; struct spi_controller *controller; - struct spi_controller *master; /* Compatibility layer */ u32 max_speed_hz; u8 chip_select[SPI_CS_CNT_MAX]; u8 bits_per_word; @@ -1298,7 +1296,7 @@ spi_max_transfer_size(struct spi_device *spi) */ static inline bool spi_is_bpw_supported(struct spi_device *spi, u32 bpw) { - u32 bpw_mask = spi->master->bits_per_word_mask; + u32 bpw_mask = spi->controller->bits_per_word_mask; if (bpw == 8 || (bpw <= 32 && bpw_mask & SPI_BPW_MASK(bpw))) return true; @@ -1670,20 +1668,4 @@ spi_transfer_is_last(struct spi_controller *ctlr, struct spi_transfer *xfer) return list_is_last(&xfer->transfer_list, &ctlr->cur_msg->transfers); } -/* Compatibility layer */ -#define spi_master spi_controller - -#define spi_master_get_devdata(_ctlr) spi_controller_get_devdata(_ctlr) -#define spi_master_set_devdata(_ctlr, _data) \ - spi_controller_set_devdata(_ctlr, _data) -#define spi_master_get(_ctlr) spi_controller_get(_ctlr) -#define spi_master_put(_ctlr) spi_controller_put(_ctlr) -#define spi_master_suspend(_ctlr) spi_controller_suspend(_ctlr) -#define spi_master_resume(_ctlr) spi_controller_resume(_ctlr) - -#define spi_register_master(_ctlr) spi_register_controller(_ctlr) -#define devm_spi_register_master(_dev, _ctlr) \ - devm_spi_register_controller(_dev, _ctlr) -#define spi_unregister_master(_ctlr) spi_unregister_controller(_ctlr) - #endif /* __LINUX_SPI_H */ From 76b31eb4c2da3ddb3195cc14f6aad24908adf524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 7 Feb 2024 19:40:46 +0100 Subject: [PATCH 0151/1038] Documentation: spi: Update documentation for renaming "master" to "controller" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. Adapt the documentation accordingly. Acked-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/3d643e22cacff12d3918ad5224baa1d01813d03b.1707324794.git.u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- Documentation/spi/spi-summary.rst | 74 +++++++++++++++---------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Documentation/spi/spi-summary.rst b/Documentation/spi/spi-summary.rst index 33f05901ccf3..e714472007f1 100644 --- a/Documentation/spi/spi-summary.rst +++ b/Documentation/spi/spi-summary.rst @@ -9,7 +9,7 @@ What is SPI? The "Serial Peripheral Interface" (SPI) is a synchronous four wire serial link used to connect microcontrollers to sensors, memory, and peripherals. It's a simple "de facto" standard, not complicated enough to acquire a -standardization body. SPI uses a master/slave configuration. +standardization body. SPI uses a host/target configuration. The three signal wires hold a clock (SCK, often on the order of 10 MHz), and parallel data lines with "Master Out, Slave In" (MOSI) or "Master In, @@ -19,11 +19,11 @@ commonly used. Each clock cycle shifts data out and data in; the clock doesn't cycle except when there is a data bit to shift. Not all data bits are used though; not every protocol uses those full duplex capabilities. -SPI masters use a fourth "chip select" line to activate a given SPI slave +SPI hosts use a fourth "chip select" line to activate a given SPI slave device, so those three signal wires may be connected to several chips in parallel. All SPI slaves support chipselects; they are usually active low signals, labeled nCSx for slave 'x' (e.g. nCS0). Some devices have -other signals, often including an interrupt to the master. +other signals, often including an interrupt to the host. Unlike serial busses like USB or SMBus, even low level protocols for SPI slave functions are usually not interoperable between vendors @@ -45,8 +45,8 @@ SPI slave functions are usually not interoperable between vendors In the same way, SPI slaves will only rarely support any kind of automatic discovery/enumeration protocol. The tree of slave devices accessible from -a given SPI master will normally be set up manually, with configuration -tables. +a given SPI host controller will normally be set up manually, with +configuration tables. SPI is only one of the names used by such four-wire protocols, and most controllers have no problem handling "MicroWire" (think of it as @@ -62,8 +62,8 @@ course they won't handle full duplex transfers. You may find such chips described as using "three wire" signaling: SCK, data, nCSx. (That data line is sometimes called MOMI or SISO.) -Microcontrollers often support both master and slave sides of the SPI -protocol. This document (and Linux) supports both the master and slave +Microcontrollers often support both host and target sides of the SPI +protocol. This document (and Linux) supports both the host and target sides of SPI interactions. @@ -118,7 +118,7 @@ starting low (CPOL=0) and data stabilized for sampling during the trailing clock edge (CPHA=1), that's SPI mode 1. Note that the clock mode is relevant as soon as the chipselect goes -active. So the master must set the clock to inactive before selecting +active. So the host must set the clock to inactive before selecting a slave, and the slave can tell the chosen polarity by sampling the clock level when its select line goes active. That's why many devices support for example both modes 0 and 3: they don't care about polarity, @@ -179,7 +179,7 @@ shows up in sysfs in several locations:: /sys/bus/spi/drivers/D ... driver for one or more spi*.* devices /sys/class/spi_master/spiB ... symlink to a logical node which could hold - class related state for the SPI master controller managing bus "B". + class related state for the SPI host controller managing bus "B". All spiB.* devices share one physical SPI bus segment, with SCLK, MOSI, and MISO. @@ -316,7 +316,7 @@ sharing a bus with a device that interprets chipselect "backwards" is not possible until the infrastructure knows how to deselect it. Then your board initialization code would register that table with the SPI -infrastructure, so that it's available later when the SPI master controller +infrastructure, so that it's available later when the SPI host controller driver is registered:: spi_register_board_info(spi_board_info, ARRAY_SIZE(spi_board_info)); @@ -474,34 +474,34 @@ How do I write an "SPI Master Controller Driver"? An SPI controller will probably be registered on the platform_bus; write a driver to bind to the device, whichever bus is involved. -The main task of this type of driver is to provide an "spi_master". -Use spi_alloc_master() to allocate the master, and spi_master_get_devdata() -to get the driver-private data allocated for that device. +The main task of this type of driver is to provide an "spi_controller". +Use spi_alloc_host() to allocate the host controller, and +spi_controller_get_devdata() to get the driver-private data allocated for that +device. :: - struct spi_master *master; + struct spi_controller *ctlr; struct CONTROLLER *c; - master = spi_alloc_master(dev, sizeof *c); - if (!master) + ctlr = spi_alloc_host(dev, sizeof *c); + if (!ctlr) return -ENODEV; - c = spi_master_get_devdata(master); + c = spi_controller_get_devdata(ctlr); -The driver will initialize the fields of that spi_master, including the -bus number (maybe the same as the platform device ID) and three methods -used to interact with the SPI core and SPI protocol drivers. It will -also initialize its own internal state. (See below about bus numbering -and those methods.) +The driver will initialize the fields of that spi_controller, including the bus +number (maybe the same as the platform device ID) and three methods used to +interact with the SPI core and SPI protocol drivers. It will also initialize +its own internal state. (See below about bus numbering and those methods.) -After you initialize the spi_master, then use spi_register_master() to +After you initialize the spi_controller, then use spi_register_controller() to publish it to the rest of the system. At that time, device nodes for the controller and any predeclared spi devices will be made available, and the driver model core will take care of binding them to drivers. -If you need to remove your SPI controller driver, spi_unregister_master() -will reverse the effect of spi_register_master(). +If you need to remove your SPI controller driver, spi_unregister_controller() +will reverse the effect of spi_register_controller(). Bus Numbering @@ -519,10 +519,10 @@ then be replaced by a dynamically assigned number. You'd then need to treat this as a non-static configuration (see above). -SPI Master Methods -^^^^^^^^^^^^^^^^^^ +SPI Host Controller Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``master->setup(struct spi_device *spi)`` +``ctlr->setup(struct spi_device *spi)`` This sets up the device clock rate, SPI mode, and word sizes. Drivers may change the defaults provided by board_info, and then call spi_setup(spi) to invoke this routine. It may sleep. @@ -534,34 +534,34 @@ SPI Master Methods .. note:: BUG ALERT: for some reason the first version of - many spi_master drivers seems to get this wrong. + many spi_controller drivers seems to get this wrong. When you code setup(), ASSUME that the controller is actively processing transfers for another device. -``master->cleanup(struct spi_device *spi)`` +``ctlr->cleanup(struct spi_device *spi)`` Your controller driver may use spi_device.controller_state to hold state it dynamically associates with that device. If you do that, be sure to provide the cleanup() method to free that state. -``master->prepare_transfer_hardware(struct spi_master *master)`` +``ctlr->prepare_transfer_hardware(struct spi_controller *ctlr)`` This will be called by the queue mechanism to signal to the driver that a message is coming in soon, so the subsystem requests the driver to prepare the transfer hardware by issuing this call. This may sleep. -``master->unprepare_transfer_hardware(struct spi_master *master)`` +``ctlr->unprepare_transfer_hardware(struct spi_controller *ctlr)`` This will be called by the queue mechanism to signal to the driver that there are no more messages pending in the queue and it may relax the hardware (e.g. by power management calls). This may sleep. -``master->transfer_one_message(struct spi_master *master, struct spi_message *mesg)`` +``ctlr->transfer_one_message(struct spi_controller *ctlr, struct spi_message *mesg)`` The subsystem calls the driver to transfer a single message while queuing transfers that arrive in the meantime. When the driver is finished with this message, it must call spi_finalize_current_message() so the subsystem can issue the next message. This may sleep. -``master->transfer_one(struct spi_master *master, struct spi_device *spi, struct spi_transfer *transfer)`` +``ctrl->transfer_one(struct spi_controller *ctlr, struct spi_device *spi, struct spi_transfer *transfer)`` The subsystem calls the driver to transfer a single transfer while queuing transfers that arrive in the meantime. When the driver is finished with this transfer, it must call @@ -576,15 +576,15 @@ SPI Master Methods * 0: transfer is finished * 1: transfer is still in progress -``master->set_cs_timing(struct spi_device *spi, u8 setup_clk_cycles, u8 hold_clk_cycles, u8 inactive_clk_cycles)`` - This method allows SPI client drivers to request SPI master controller +``ctrl->set_cs_timing(struct spi_device *spi, u8 setup_clk_cycles, u8 hold_clk_cycles, u8 inactive_clk_cycles)`` + This method allows SPI client drivers to request SPI host controller for configuring device specific CS setup, hold and inactive timing requirements. Deprecated Methods ^^^^^^^^^^^^^^^^^^ -``master->transfer(struct spi_device *spi, struct spi_message *message)`` +``ctrl->transfer(struct spi_device *spi, struct spi_message *message)`` This must not sleep. Its responsibility is to arrange that the transfer happens and its complete() callback is issued. The two will normally happen later, after other transfers complete, and From 5d389afc1f8fea707225bc2ee64e00d85dab623a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 1 Feb 2024 16:11:15 -0600 Subject: [PATCH 0152/1038] ACPI: video: Handle fetching EDID that is longer than 256 bytes The ACPI specification allows for an EDID to be up to 512 bytes but the _DDC EDID fetching code will only try up to 256 bytes. Modify the code to instead start at 512 bytes and work it's way down instead. As _DDC is now called up to 4 times on a machine debugging messages are noisier than necessary. Decrease from info to debug. Link: https://uefi.org/htmlspecs/ACPI_Spec_6_4_html/Apx_B_Video_Extensions/output-device-specific-methods.html#ddc-return-the-edid-for-this-device Signed-off-by: Mario Limonciello [ rjw: Type mismatch fix, minor whitespace adjustment ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_video.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 4afdda9db019..1fda30388297 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -612,7 +612,7 @@ acpi_video_device_lcd_get_level_current(struct acpi_video_device *device, static int acpi_video_device_EDID(struct acpi_video_device *device, - union acpi_object **edid, ssize_t length) + union acpi_object **edid, int length) { int status; struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; @@ -625,13 +625,11 @@ acpi_video_device_EDID(struct acpi_video_device *device, if (!device) return -ENODEV; - if (length == 128) - arg0.integer.value = 1; - else if (length == 256) - arg0.integer.value = 2; - else + if (!length || (length % 128)) return -EINVAL; + arg0.integer.value = length / 128; + status = acpi_evaluate_object(device->dev->handle, "_DDC", &args, &buffer); if (ACPI_FAILURE(status)) return -ENODEV; @@ -641,7 +639,8 @@ acpi_video_device_EDID(struct acpi_video_device *device, if (obj && obj->type == ACPI_TYPE_BUFFER) *edid = obj; else { - acpi_handle_info(device->dev->handle, "Invalid _DDC data\n"); + acpi_handle_debug(device->dev->handle, + "Invalid _DDC data for length %d\n", length); status = -EFAULT; kfree(obj); } @@ -1447,7 +1446,6 @@ int acpi_video_get_edid(struct acpi_device *device, int type, int device_id, for (i = 0; i < video->attached_count; i++) { video_device = video->attached_array[i].bind_info; - length = 256; if (!video_device) continue; @@ -1478,18 +1476,14 @@ int acpi_video_get_edid(struct acpi_device *device, int type, int device_id, continue; } - status = acpi_video_device_EDID(video_device, &buffer, length); - - if (ACPI_FAILURE(status) || !buffer || - buffer->type != ACPI_TYPE_BUFFER) { - length = 128; + for (length = 512; length > 0; length -= 128) { status = acpi_video_device_EDID(video_device, &buffer, length); - if (ACPI_FAILURE(status) || !buffer || - buffer->type != ACPI_TYPE_BUFFER) { - continue; - } + if (ACPI_SUCCESS(status)) + break; } + if (!length) + continue; *edid = buffer->buffer.pointer; return length; From 2e171a57c312cb732d633c5ccc51551588b7a855 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 7 Feb 2024 20:09:15 +0100 Subject: [PATCH 0153/1038] iwlwifi: mvm: Drop unused fw_trips_index[] from iwl_mvm_thermal_device The fw_trips_index[] array in struct iwl_mvm_thermal_device is only populated, but never read, so drop it. Note that the iwl_mvm_send_temp_report_ths_cmd() code populating fw_trips_index[] is questionable, because it accesses a trips table of a thermal zone directly, which is not guaranteed to work in the future. No functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Miri Korenblit --- drivers/net/wireless/intel/iwlwifi/mvm/mvm.h | 2 -- drivers/net/wireless/intel/iwlwifi/mvm/tt.c | 13 +------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index 40627961b834..a618824f0b65 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -539,12 +539,10 @@ struct iwl_mvm_tt_mgmt { /** *struct iwl_mvm_thermal_device - thermal zone related data * @temp_trips: temperature thresholds for report - * @fw_trips_index: keep indexes to original array - temp_trips * @tzone: thermal zone device data */ struct iwl_mvm_thermal_device { struct thermal_trip trips[IWL_MAX_DTS_TRIPS]; - u8 fw_trips_index[IWL_MAX_DTS_TRIPS]; struct thermal_zone_device *tzone; }; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index dee9c367dcd3..a2f921a1f5bd 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -562,7 +562,7 @@ int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) struct temp_report_ths_cmd cmd = {0}; int ret; #ifdef CONFIG_THERMAL - int i, j, idx = 0; + int i, idx = 0; lockdep_assert_held(&mvm->mutex); @@ -588,17 +588,6 @@ int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) /*sort cmd array*/ sort(cmd.thresholds, idx, sizeof(s16), compare_temps, NULL); - /* we should save the indexes of trips because we sort - * and compress the orginal array - */ - for (i = 0; i < idx; i++) { - for (j = 0; j < IWL_MAX_DTS_TRIPS; j++) { - if ((int)(le16_to_cpu(cmd.thresholds[i]) * 1000) == - mvm->tz_device.trips[j].temperature) - mvm->tz_device.fw_trips_index[i] = j; - } - } - send: #endif ret = iwl_mvm_send_cmd_pdu(mvm, WIDE_ID(PHY_OPS_GROUP, From 61d88437546f5f40b6411aac17c55d0298c7a19d Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 7 Feb 2024 20:10:24 +0100 Subject: [PATCH 0154/1038] iwlwifi: mvm: Populate trip table before registering thermal zone The trip table in iwl_mvm_thermal_zone_register() is populated after passing it to thermal_zone_device_register_with_trips(), so it may be accessed (for instance, via sysfs) before it is ready. To prevent that from happening, modify the function to populate the trip table before calling thermal_zone_device_register_with_trips(). Also make the code use THERMAL_TEMP_INVALID as the "invalid temperature" value which is also meaningful for the core. Signed-off-by: Rafael J. Wysocki Acked-by: Miri Korenblit --- drivers/net/wireless/intel/iwlwifi/mvm/tt.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index a2f921a1f5bd..4fcbf02b415e 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -575,7 +575,7 @@ int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) /* compress trips to cmd array, remove uninitialized values*/ for (i = 0; i < IWL_MAX_DTS_TRIPS; i++) { - if (mvm->tz_device.trips[i].temperature != INT_MIN) { + if (mvm->tz_device.trips[i].temperature != THERMAL_TEMP_INVALID) { cmd.thresholds[idx++] = cpu_to_le16((s16)(mvm->tz_device.trips[i].temperature / 1000)); } @@ -675,6 +675,14 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) BUILD_BUG_ON(ARRAY_SIZE(name) >= THERMAL_NAME_LENGTH); sprintf(name, "iwlwifi_%u", atomic_inc_return(&counter) & 0xFF); + /* + * 0 is a valid temperature, + * so initialize the array with S16_MIN which invalid temperature + */ + for (i = 0 ; i < IWL_MAX_DTS_TRIPS; i++) { + mvm->tz_device.trips[i].temperature = THERMAL_TEMP_INVALID; + mvm->tz_device.trips[i].type = THERMAL_TRIP_PASSIVE; + } mvm->tz_device.tzone = thermal_zone_device_register_with_trips(name, mvm->tz_device.trips, IWL_MAX_DTS_TRIPS, @@ -693,15 +701,6 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) if (ret) { IWL_DEBUG_TEMP(mvm, "Failed to enable thermal zone\n"); thermal_zone_device_unregister(mvm->tz_device.tzone); - return; - } - - /* 0 is a valid temperature, - * so initialize the array with S16_MIN which invalid temperature - */ - for (i = 0 ; i < IWL_MAX_DTS_TRIPS; i++) { - mvm->tz_device.trips[i].temperature = INT_MIN; - mvm->tz_device.trips[i].type = THERMAL_TRIP_PASSIVE; } } From 85af3310df34b0931daba8732d5b2c64a81c3b8d Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 7 Feb 2024 20:12:38 +0100 Subject: [PATCH 0155/1038] iwlwifi: mvm: Use for_each_thermal_trip() for walking trip points The code walking trip points in iwl_mvm_send_temp_report_ths_cmd() reads the trip table passed to thermal_zone_device_register_with_trips() in order to get the current trip temperatures, but this is not guaranteed to work in the future, because the thermal zone will store trip points information internally. For this reason, make iwl_mvm_send_temp_report_ths_cmd() use for_each_thermal_trip() as appropriate for walking trip points in a given thermal zone. No intentional functional impact, but it is requisite for future thermal core improvements. Signed-off-by: Rafael J. Wysocki Acked-by: Miri Korenblit --- drivers/net/wireless/intel/iwlwifi/mvm/tt.c | 40 +++++++++++++-------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index 4fcbf02b415e..d2a00cbcbc42 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -555,6 +555,22 @@ static int compare_temps(const void *a, const void *b) return ((s16)le16_to_cpu(*(__le16 *)a) - (s16)le16_to_cpu(*(__le16 *)b)); } + +struct iwl_trip_walk_data { + __le16 *thresholds; + int count; +}; + +static int iwl_trip_temp_cb(struct thermal_trip *trip, void *arg) +{ + struct iwl_trip_walk_data *twd = arg; + + if (trip->temperature == THERMAL_TEMP_INVALID) + return 0; + + twd->thresholds[twd->count++] = cpu_to_le16((s16)(trip->temperature / 1000)); + return 0; +} #endif int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) @@ -562,31 +578,25 @@ int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) struct temp_report_ths_cmd cmd = {0}; int ret; #ifdef CONFIG_THERMAL - int i, idx = 0; + struct iwl_trip_walk_data twd = { .thresholds = cmd.thresholds, .count = 0 }; lockdep_assert_held(&mvm->mutex); if (!mvm->tz_device.tzone) goto send; - /* The driver holds array of temperature trips that are unsorted - * and uncompressed, the FW should get it compressed and sorted + /* + * The thermal core holds an array of temperature trips that are + * unsorted and uncompressed, the FW should get it compressed and + * sorted. */ /* compress trips to cmd array, remove uninitialized values*/ - for (i = 0; i < IWL_MAX_DTS_TRIPS; i++) { - if (mvm->tz_device.trips[i].temperature != THERMAL_TEMP_INVALID) { - cmd.thresholds[idx++] = - cpu_to_le16((s16)(mvm->tz_device.trips[i].temperature / 1000)); - } - } - cmd.num_temps = cpu_to_le32(idx); + for_each_thermal_trip(mvm->tz_device.tzone, iwl_trip_temp_cb, &twd); - if (!idx) - goto send; - - /*sort cmd array*/ - sort(cmd.thresholds, idx, sizeof(s16), compare_temps, NULL); + cmd.num_temps = cpu_to_le32(twd.count); + if (twd.count) + sort(cmd.thresholds, twd.count, sizeof(s16), compare_temps, NULL); send: #endif From 4274521fabee05375d10bea0e36a806ed4ab7b45 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:35 +0000 Subject: [PATCH 0156/1038] PM: EM: Add missing newline for the message log Fix missing newline for the string long in the error code path. Reviewed-by: Hongyan Xia Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 7b44f5b89fa1..8b9dd4a39f63 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -250,7 +250,7 @@ static void em_cpufreq_update_efficiencies(struct device *dev) policy = cpufreq_cpu_get(cpumask_first(em_span_cpus(pd))); if (!policy) { - dev_warn(dev, "EM: Access to CPUFreq policy failed"); + dev_warn(dev, "EM: Access to CPUFreq policy failed\n"); return; } From e7b1cc9a7ea6d7862baac0fd7b145573618727dd Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:36 +0000 Subject: [PATCH 0157/1038] PM: EM: Extend em_cpufreq_update_efficiencies() argument list In order to prepare the code for the modifiable EM perf_state table, make em_cpufreq_update_efficiencies() take a pointer to the EM table as its second argument and modify it to use that new argument instead of the 'table' member of dev->em_pd. No functional impact. Reviewed-by: Hongyan Xia Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 8b9dd4a39f63..8c373b151875 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -237,15 +237,15 @@ static int em_create_pd(struct device *dev, int nr_states, return 0; } -static void em_cpufreq_update_efficiencies(struct device *dev) +static void +em_cpufreq_update_efficiencies(struct device *dev, struct em_perf_state *table) { struct em_perf_domain *pd = dev->em_pd; - struct em_perf_state *table; struct cpufreq_policy *policy; int found = 0; int i; - if (!_is_cpu_device(dev) || !pd) + if (!_is_cpu_device(dev)) return; policy = cpufreq_cpu_get(cpumask_first(em_span_cpus(pd))); @@ -254,8 +254,6 @@ static void em_cpufreq_update_efficiencies(struct device *dev) return; } - table = pd->table; - for (i = 0; i < pd->nr_perf_states; i++) { if (!(table[i].flags & EM_PERF_STATE_INEFFICIENT)) continue; @@ -397,7 +395,7 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, dev->em_pd->flags |= flags; - em_cpufreq_update_efficiencies(dev); + em_cpufreq_update_efficiencies(dev, dev->em_pd->table); em_debug_create_pd(dev); dev_info(dev, "EM: created perf domain\n"); From 99907d6054f2d39a625004f9f4e3fe9297838a3c Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:37 +0000 Subject: [PATCH 0158/1038] PM: EM: Find first CPU active while updating OPP efficiency The Energy Model might be updated at runtime and the energy efficiency for each OPP may change. Thus, there is a need to update also the cpufreq framework and make it aligned to the new values. In order to do that, use a first active CPU from the Performance Domain. This is needed since the first CPU in the cpumask might be offline when we run this code path. Reviewed-by: Hongyan Xia Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 8c373b151875..0c3220ff54f7 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -243,12 +243,19 @@ em_cpufreq_update_efficiencies(struct device *dev, struct em_perf_state *table) struct em_perf_domain *pd = dev->em_pd; struct cpufreq_policy *policy; int found = 0; - int i; + int i, cpu; if (!_is_cpu_device(dev)) return; - policy = cpufreq_cpu_get(cpumask_first(em_span_cpus(pd))); + /* Try to get a CPU which is active and in this PD */ + cpu = cpumask_first_and(em_span_cpus(pd), cpu_active_mask); + if (cpu >= nr_cpu_ids) { + dev_warn(dev, "EM: No online CPU for CPUFreq policy\n"); + return; + } + + policy = cpufreq_cpu_get(cpu); if (!policy) { dev_warn(dev, "EM: Access to CPUFreq policy failed\n"); return; From a3c78778f50c4db6cc0bb6aa2986c0174b1267d0 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:38 +0000 Subject: [PATCH 0159/1038] PM: EM: Refactor em_pd_get_efficient_state() to be more flexible The Energy Model (EM) is going to support runtime modification. There are going to be 2 EM tables which store information. This patch aims to prepare the code to be generic and use one of the tables. The function will no longer get a pointer to 'struct em_perf_domain' (the EM) but instead a pointer to 'struct em_perf_state' (which is one of the EM's tables). Prepare em_pd_get_efficient_state() for the upcoming changes and make it possible to be re-used. Return an index for the best performance state for a given EM table. The function arguments that are introduced should allow to work on different performance state arrays. The caller of em_pd_get_efficient_state() should be able to use the index either on the default or the modifiable EM table. Reviewed-by: Daniel Lezcano Reviewed-by: Hongyan Xia Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 88d91e087471..1dcd1645dde7 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -175,33 +175,35 @@ void em_dev_unregister_perf_domain(struct device *dev); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM - * @pd : Performance domain for which we want an efficient frequency - * @freq : Frequency to map with the EM + * @table: List of performance states, in ascending order + * @nr_perf_states: Number of performance states + * @freq: Frequency to map with the EM + * @pd_flags: Performance Domain flags * * It is called from the scheduler code quite frequently and as a consequence * doesn't implement any check. * - * Return: An efficient performance state, high enough to meet @freq + * Return: An efficient performance state id, high enough to meet @freq * requirement. */ -static inline -struct em_perf_state *em_pd_get_efficient_state(struct em_perf_domain *pd, - unsigned long freq) +static inline int +em_pd_get_efficient_state(struct em_perf_state *table, int nr_perf_states, + unsigned long freq, unsigned long pd_flags) { struct em_perf_state *ps; int i; - for (i = 0; i < pd->nr_perf_states; i++) { - ps = &pd->table[i]; + for (i = 0; i < nr_perf_states; i++) { + ps = &table[i]; if (ps->frequency >= freq) { - if (pd->flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && + if (pd_flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && ps->flags & EM_PERF_STATE_INEFFICIENT) continue; - break; + return i; } } - return ps; + return nr_perf_states - 1; } /** @@ -226,7 +228,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, { unsigned long freq, ref_freq, scale_cpu; struct em_perf_state *ps; - int cpu; + int cpu, i; if (!sum_util) return 0; @@ -250,7 +252,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * Find the lowest performance state of the Energy Model above the * requested frequency. */ - ps = em_pd_get_efficient_state(pd, freq); + i = em_pd_get_efficient_state(pd->table, pd->nr_perf_states, freq, + pd->flags); + ps = &pd->table[i]; /* * The capacity of a CPU in the domain at the performance state (ps) From faf7075b79a259136e2b57ce52b48a7096270e8b Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:39 +0000 Subject: [PATCH 0160/1038] PM: EM: Introduce em_compute_costs() Move the EM costs computation code into a new dedicated function, em_compute_costs(), that can be reused in other places in the future. This change is not expected to alter the general functionality. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 72 ++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 0c3220ff54f7..5c47caaf270e 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -103,14 +103,52 @@ static void em_debug_create_pd(struct device *dev) {} static void em_debug_remove_pd(struct device *dev) {} #endif +static int em_compute_costs(struct device *dev, struct em_perf_state *table, + struct em_data_callback *cb, int nr_states, + unsigned long flags) +{ + unsigned long prev_cost = ULONG_MAX; + u64 fmax; + int i, ret; + + /* Compute the cost of each performance state. */ + fmax = (u64) table[nr_states - 1].frequency; + for (i = nr_states - 1; i >= 0; i--) { + unsigned long power_res, cost; + + if (flags & EM_PERF_DOMAIN_ARTIFICIAL) { + ret = cb->get_cost(dev, table[i].frequency, &cost); + if (ret || !cost || cost > EM_MAX_POWER) { + dev_err(dev, "EM: invalid cost %lu %d\n", + cost, ret); + return -EINVAL; + } + } else { + power_res = table[i].power; + cost = div64_u64(fmax * power_res, table[i].frequency); + } + + table[i].cost = cost; + + if (table[i].cost >= prev_cost) { + table[i].flags = EM_PERF_STATE_INEFFICIENT; + dev_dbg(dev, "EM: OPP:%lu is inefficient\n", + table[i].frequency); + } else { + prev_cost = table[i].cost; + } + } + + return 0; +} + static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, int nr_states, struct em_data_callback *cb, unsigned long flags) { - unsigned long power, freq, prev_freq = 0, prev_cost = ULONG_MAX; + unsigned long power, freq, prev_freq = 0; struct em_perf_state *table; int i, ret; - u64 fmax; table = kcalloc(nr_states, sizeof(*table), GFP_KERNEL); if (!table) @@ -154,33 +192,9 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, table[i].frequency = prev_freq = freq; } - /* Compute the cost of each performance state. */ - fmax = (u64) table[nr_states - 1].frequency; - for (i = nr_states - 1; i >= 0; i--) { - unsigned long power_res, cost; - - if (flags & EM_PERF_DOMAIN_ARTIFICIAL) { - ret = cb->get_cost(dev, table[i].frequency, &cost); - if (ret || !cost || cost > EM_MAX_POWER) { - dev_err(dev, "EM: invalid cost %lu %d\n", - cost, ret); - goto free_ps_table; - } - } else { - power_res = table[i].power; - cost = div64_u64(fmax * power_res, table[i].frequency); - } - - table[i].cost = cost; - - if (table[i].cost >= prev_cost) { - table[i].flags = EM_PERF_STATE_INEFFICIENT; - dev_dbg(dev, "EM: OPP:%lu is inefficient\n", - table[i].frequency); - } else { - prev_cost = table[i].cost; - } - } + ret = em_compute_costs(dev, table, cb, nr_states, flags); + if (ret) + goto free_ps_table; pd->table = table; pd->nr_perf_states = nr_states; From 818867224d41725dcf4abe890d8f24e5d6bd9c67 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:40 +0000 Subject: [PATCH 0161/1038] PM: EM: Check if the get_cost() callback is present in em_compute_costs() Subsequent changes will introduce a case in which 'cb->get_cost' may not be set in em_compute_costs(), so add a check to ensure that it is not NULL before attempting to dereference it. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 5c47caaf270e..21d761223255 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -116,7 +116,7 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, for (i = nr_states - 1; i >= 0; i--) { unsigned long power_res, cost; - if (flags & EM_PERF_DOMAIN_ARTIFICIAL) { + if ((flags & EM_PERF_DOMAIN_ARTIFICIAL) && cb->get_cost) { ret = cb->get_cost(dev, table[i].frequency, &cost); if (ret || !cost || cost > EM_MAX_POWER) { dev_err(dev, "EM: invalid cost %lu %d\n", From 8552d6820168d6508bd1f7cd49be248dcb74efb3 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:41 +0000 Subject: [PATCH 0162/1038] PM: EM: Split the allocation and initialization of the EM table Split the process of allocation and data initialization for the EM table. The upcoming changes for modifiable EM will use it. This change is not expected to alter the general functionality. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 55 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 21d761223255..7468fa92134b 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -142,18 +142,26 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return 0; } +static int em_allocate_perf_table(struct em_perf_domain *pd, + int nr_states) +{ + pd->table = kcalloc(nr_states, sizeof(struct em_perf_state), + GFP_KERNEL); + if (!pd->table) + return -ENOMEM; + + return 0; +} + static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, - int nr_states, struct em_data_callback *cb, + struct em_perf_state *table, + struct em_data_callback *cb, unsigned long flags) { unsigned long power, freq, prev_freq = 0; - struct em_perf_state *table; + int nr_states = pd->nr_perf_states; int i, ret; - table = kcalloc(nr_states, sizeof(*table), GFP_KERNEL); - if (!table) - return -ENOMEM; - /* Build the list of performance states for this performance domain */ for (i = 0, freq = 0; i < nr_states; i++, freq++) { /* @@ -165,7 +173,7 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, if (ret) { dev_err(dev, "EM: invalid perf. state: %d\n", ret); - goto free_ps_table; + return -EINVAL; } /* @@ -175,7 +183,7 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, if (freq <= prev_freq) { dev_err(dev, "EM: non-increasing freq: %lu\n", freq); - goto free_ps_table; + return -EINVAL; } /* @@ -185,7 +193,7 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, if (!power || power > EM_MAX_POWER) { dev_err(dev, "EM: invalid power: %lu\n", power); - goto free_ps_table; + return -EINVAL; } table[i].power = power; @@ -194,16 +202,9 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, ret = em_compute_costs(dev, table, cb, nr_states, flags); if (ret) - goto free_ps_table; - - pd->table = table; - pd->nr_perf_states = nr_states; + return -EINVAL; return 0; - -free_ps_table: - kfree(table); - return -EINVAL; } static int em_create_pd(struct device *dev, int nr_states, @@ -234,11 +235,15 @@ static int em_create_pd(struct device *dev, int nr_states, return -ENOMEM; } - ret = em_create_perf_table(dev, pd, nr_states, cb, flags); - if (ret) { - kfree(pd); - return ret; - } + pd->nr_perf_states = nr_states; + + ret = em_allocate_perf_table(pd, nr_states); + if (ret) + goto free_pd; + + ret = em_create_perf_table(dev, pd, pd->table, cb, flags); + if (ret) + goto free_pd_table; if (_is_cpu_device(dev)) for_each_cpu(cpu, cpus) { @@ -249,6 +254,12 @@ static int em_create_pd(struct device *dev, int nr_states, dev->em_pd = pd; return 0; + +free_pd_table: + kfree(pd->table); +free_pd: + kfree(pd); + return -EINVAL; } static void From ca0fc871f16f4bef746b5ba814b67afb59119700 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:42 +0000 Subject: [PATCH 0163/1038] PM: EM: Introduce runtime modifiable table The new runtime table can be populated with a new power data to better reflect the actual efficiency of the device e.g. CPU. The power can vary over time e.g. due to the SoC temperature change. Higher temperature can increase power values. For longer running scenarios, such as game or camera, when also other devices are used (e.g. GPU, ISP) the CPU power can change. The new EM framework is able to addresses this issue and change the EM data at runtime safely. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 12 ++++++++ kernel/power/energy_model.c | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 1dcd1645dde7..8ddf1d8a9581 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -36,9 +36,20 @@ struct em_perf_state { */ #define EM_PERF_STATE_INEFFICIENT BIT(0) +/** + * struct em_perf_table - Performance states table + * @rcu: RCU used for safe access and destruction + * @state: List of performance states, in ascending order + */ +struct em_perf_table { + struct rcu_head rcu; + struct em_perf_state state[]; +}; + /** * struct em_perf_domain - Performance domain * @table: List of performance states, in ascending order + * @em_table: Pointer to the runtime modifiable em_perf_table * @nr_perf_states: Number of performance states * @flags: See "em_perf_domain flags" * @cpus: Cpumask covering the CPUs of the domain. It's here @@ -54,6 +65,7 @@ struct em_perf_state { */ struct em_perf_domain { struct em_perf_state *table; + struct em_perf_table __rcu *em_table; int nr_perf_states; unsigned long flags; unsigned long cpus[]; diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 7468fa92134b..131ff1d0dc5b 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -23,6 +23,9 @@ */ static DEFINE_MUTEX(em_pd_mutex); +static void em_cpufreq_update_efficiencies(struct device *dev, + struct em_perf_state *table); + static bool _is_cpu_device(struct device *dev) { return (dev->bus == &cpu_subsys); @@ -103,6 +106,31 @@ static void em_debug_create_pd(struct device *dev) {} static void em_debug_remove_pd(struct device *dev) {} #endif +static void em_destroy_table_rcu(struct rcu_head *rp) +{ + struct em_perf_table __rcu *table; + + table = container_of(rp, struct em_perf_table, rcu); + kfree(table); +} + +static void em_free_table(struct em_perf_table __rcu *table) +{ + call_rcu(&table->rcu, em_destroy_table_rcu); +} + +static struct em_perf_table __rcu * +em_allocate_table(struct em_perf_domain *pd) +{ + struct em_perf_table __rcu *table; + int table_size; + + table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + + table = kzalloc(sizeof(*table) + table_size, GFP_KERNEL); + return table; +} + static int em_compute_costs(struct device *dev, struct em_perf_state *table, struct em_data_callback *cb, int nr_states, unsigned long flags) @@ -153,6 +181,24 @@ static int em_allocate_perf_table(struct em_perf_domain *pd, return 0; } +static int em_create_runtime_table(struct em_perf_domain *pd) +{ + struct em_perf_table __rcu *table; + int table_size; + + table = em_allocate_table(pd); + if (!table) + return -ENOMEM; + + /* Initialize runtime table with existing data */ + table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + memcpy(table->state, pd->table, table_size); + + rcu_assign_pointer(pd->em_table, table); + + return 0; +} + static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, struct em_perf_state *table, struct em_data_callback *cb, @@ -245,6 +291,10 @@ static int em_create_pd(struct device *dev, int nr_states, if (ret) goto free_pd_table; + ret = em_create_runtime_table(pd); + if (ret) + goto free_pd_table; + if (_is_cpu_device(dev)) for_each_cpu(cpu, cpus) { cpu_dev = get_cpu_device(cpu); @@ -461,6 +511,9 @@ void em_dev_unregister_perf_domain(struct device *dev) em_debug_remove_pd(dev); kfree(dev->em_pd->table); + + em_free_table(dev->em_pd->em_table); + kfree(dev->em_pd); dev->em_pd = NULL; mutex_unlock(&em_pd_mutex); From aa11a7ebfd5d698f541641922beede1cb474bf70 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:43 +0000 Subject: [PATCH 0164/1038] PM: EM: Use runtime modified EM for CPUs energy estimation in EAS The new Energy Model (EM) supports runtime modification of the performance state table to better model the power used by the SoC. Use this new feature to improve energy estimation and therefore task placement in Energy Aware Scheduler (EAS). Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 8ddf1d8a9581..5f842da3bb0c 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -239,9 +239,14 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, unsigned long allowed_cpu_cap) { unsigned long freq, ref_freq, scale_cpu; + struct em_perf_table *em_table; struct em_perf_state *ps; int cpu, i; +#ifdef CONFIG_SCHED_DEBUG + WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n"); +#endif + if (!sum_util) return 0; @@ -264,9 +269,10 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * Find the lowest performance state of the Energy Model above the * requested frequency. */ - i = em_pd_get_efficient_state(pd->table, pd->nr_perf_states, freq, - pd->flags); - ps = &pd->table[i]; + em_table = rcu_dereference(pd->em_table); + i = em_pd_get_efficient_state(em_table->state, pd->nr_perf_states, + freq, pd->flags); + ps = &em_table->state[i]; /* * The capacity of a CPU in the domain at the performance state (ps) From ffcf9bce7af02a21fb73738999de1e3d4fde5aca Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:44 +0000 Subject: [PATCH 0165/1038] PM: EM: Add functions for memory allocations for new EM tables The runtime modified EM table can be provided from drivers. Create mechanism which allows safely allocate and free the table for device drivers. The same table can be used by the EAS in task scheduler code paths, so make sure the memory is not freed when the device driver module is unloaded. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 11 +++++++++++ kernel/power/energy_model.c | 38 +++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 5f842da3bb0c..27911dc1887e 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -39,10 +40,12 @@ struct em_perf_state { /** * struct em_perf_table - Performance states table * @rcu: RCU used for safe access and destruction + * @kref: Reference counter to track the users * @state: List of performance states, in ascending order */ struct em_perf_table { struct rcu_head rcu; + struct kref kref; struct em_perf_state state[]; }; @@ -184,6 +187,8 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, bool microwatts); void em_dev_unregister_perf_domain(struct device *dev); +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); +void em_table_free(struct em_perf_table __rcu *table); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM @@ -365,6 +370,12 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd) { return 0; } +static inline +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) +{ + return NULL; +} +static inline void em_table_free(struct em_perf_table __rcu *table) {} #endif #endif diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 131ff1d0dc5b..16795743f969 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -114,13 +114,36 @@ static void em_destroy_table_rcu(struct rcu_head *rp) kfree(table); } -static void em_free_table(struct em_perf_table __rcu *table) +static void em_release_table_kref(struct kref *kref) { + struct em_perf_table __rcu *table; + + /* It was the last owner of this table so we can free */ + table = container_of(kref, struct em_perf_table, kref); + call_rcu(&table->rcu, em_destroy_table_rcu); } -static struct em_perf_table __rcu * -em_allocate_table(struct em_perf_domain *pd) +/** + * em_table_free() - Handles safe free of the EM table when needed + * @table : EM table which is going to be freed + * + * No return values. + */ +void em_table_free(struct em_perf_table __rcu *table) +{ + kref_put(&table->kref, em_release_table_kref); +} + +/** + * em_table_alloc() - Allocate a new EM table + * @pd : EM performance domain for which this must be done + * + * Allocate a new EM table and initialize its kref to indicate that it + * has a user. + * Returns allocated table or NULL. + */ +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) { struct em_perf_table __rcu *table; int table_size; @@ -128,6 +151,11 @@ em_allocate_table(struct em_perf_domain *pd) table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; table = kzalloc(sizeof(*table) + table_size, GFP_KERNEL); + if (!table) + return NULL; + + kref_init(&table->kref); + return table; } @@ -186,7 +214,7 @@ static int em_create_runtime_table(struct em_perf_domain *pd) struct em_perf_table __rcu *table; int table_size; - table = em_allocate_table(pd); + table = em_table_alloc(pd); if (!table) return -ENOMEM; @@ -512,7 +540,7 @@ void em_dev_unregister_perf_domain(struct device *dev) kfree(dev->em_pd->table); - em_free_table(dev->em_pd->em_table); + em_table_free(dev->em_pd->em_table); kfree(dev->em_pd); dev->em_pd = NULL; From 977230d5d50314f9920d3ee6348773d8babbfb58 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:45 +0000 Subject: [PATCH 0166/1038] PM: EM: Introduce em_dev_update_perf_domain() for EM updates Add API function em_dev_update_perf_domain() which allows the EM to be changed safely. Concurrent updaters are serialized with a mutex and the removal of memory that will not be used any more is carried out with the help of RCU. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 8 +++++++ kernel/power/energy_model.c | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 27911dc1887e..324a3a8e0a2d 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -183,6 +183,8 @@ struct em_data_callback { struct em_perf_domain *em_cpu_get(int cpu); struct em_perf_domain *em_pd_get(struct device *dev); +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table); int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, bool microwatts); @@ -376,6 +378,12 @@ struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) return NULL; } static inline void em_table_free(struct em_perf_table __rcu *table) {} +static inline +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table) +{ + return -EINVAL; +} #endif #endif diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 16795743f969..667619b70be7 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -209,6 +209,50 @@ static int em_allocate_perf_table(struct em_perf_domain *pd, return 0; } +/** + * em_dev_update_perf_domain() - Update runtime EM table for a device + * @dev : Device for which the EM is to be updated + * @new_table : The new EM table that is going to be used from now + * + * Update EM runtime modifiable table for the @dev using the provided @table. + * + * This function uses a mutex to serialize writers, so it must not be called + * from a non-sleeping context. + * + * Return 0 on success or an error code on failure. + */ +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table) +{ + struct em_perf_table __rcu *old_table; + struct em_perf_domain *pd; + + if (!dev) + return -EINVAL; + + /* Serialize update/unregister or concurrent updates */ + mutex_lock(&em_pd_mutex); + + if (!dev->em_pd) { + mutex_unlock(&em_pd_mutex); + return -EINVAL; + } + pd = dev->em_pd; + + kref_get(&new_table->kref); + + old_table = pd->em_table; + rcu_assign_pointer(pd->em_table, new_table); + + em_cpufreq_update_efficiencies(dev, new_table->state); + + em_table_free(old_table); + + mutex_unlock(&em_pd_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(em_dev_update_perf_domain); + static int em_create_runtime_table(struct em_perf_domain *pd) { struct em_perf_table __rcu *table; From ee1a19873ce1234a3c2e6f84af3624fc73bfbd9c Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:46 +0000 Subject: [PATCH 0167/1038] PM: EM: Add em_perf_state_from_pd() to get performance states table Introduce a wrapper to get the performance states table of the performance domain. The function should be called within the RCU read critical section. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 324a3a8e0a2d..158dad6ea313 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -338,6 +338,23 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd) return pd->nr_perf_states; } +/** + * em_perf_state_from_pd() - Get the performance states table of perf. + * domain + * @pd : performance domain for which this must be done + * + * To use this function the rcu_read_lock() should be hold. After the usage + * of the performance states table is finished, the rcu_read_unlock() should + * be called. + * + * Return: the pointer to performance states table of the performance domain + */ +static inline +struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) +{ + return rcu_dereference(pd->em_table)->state; +} + #else struct em_data_callback {}; #define EM_ADV_DATA_CB(_active_power_cb, _cost_cb) { } @@ -384,6 +401,11 @@ int em_dev_update_perf_domain(struct device *dev, { return -EINVAL; } +static inline +struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) +{ + return NULL; +} #endif #endif From 5a367f7b7014af86bd1ac0865a42db55187dbd3c Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:47 +0000 Subject: [PATCH 0168/1038] PM: EM: Add performance field to struct em_perf_state and optimize The performance doesn't scale linearly with the frequency. Also, it may be different in different workloads. Some CPUs are designed to be particularly good at some applications e.g. images or video processing and other CPUs in different. When those different types of CPUs are combined in one SoC they should be properly modeled to get max of the HW in Energy Aware Scheduler (EAS). The Energy Model (EM) provides the power vs. performance curves to the EAS, but assumes the CPUs capacity is fixed and scales linearly with the frequency. This patch allows to adjust the curve on the 'performance' axis as well. Code speed optimization: Removing map_util_freq() allows to avoid one division and one multiplication operations from the EAS hot code path. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 24 ++++++++++++------------ kernel/power/energy_model.c | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 158dad6ea313..ce24ea3fe41c 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -13,6 +13,7 @@ /** * struct em_perf_state - Performance state of a performance domain + * @performance: CPU performance (capacity) at a given frequency * @frequency: The frequency in KHz, for consistency with CPUFreq * @power: The power consumed at this level (by 1 CPU or by a registered * device). It can be a total power: static and dynamic. @@ -21,6 +22,7 @@ * @flags: see "em_perf_state flags" description below. */ struct em_perf_state { + unsigned long performance; unsigned long frequency; unsigned long power; unsigned long cost; @@ -196,25 +198,25 @@ void em_table_free(struct em_perf_table __rcu *table); * em_pd_get_efficient_state() - Get an efficient performance state from the EM * @table: List of performance states, in ascending order * @nr_perf_states: Number of performance states - * @freq: Frequency to map with the EM + * @max_util: Max utilization to map with the EM * @pd_flags: Performance Domain flags * * It is called from the scheduler code quite frequently and as a consequence * doesn't implement any check. * - * Return: An efficient performance state id, high enough to meet @freq + * Return: An efficient performance state id, high enough to meet @max_util * requirement. */ static inline int em_pd_get_efficient_state(struct em_perf_state *table, int nr_perf_states, - unsigned long freq, unsigned long pd_flags) + unsigned long max_util, unsigned long pd_flags) { struct em_perf_state *ps; int i; for (i = 0; i < nr_perf_states; i++) { ps = &table[i]; - if (ps->frequency >= freq) { + if (ps->performance >= max_util) { if (pd_flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && ps->flags & EM_PERF_STATE_INEFFICIENT) continue; @@ -245,9 +247,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, unsigned long max_util, unsigned long sum_util, unsigned long allowed_cpu_cap) { - unsigned long freq, ref_freq, scale_cpu; struct em_perf_table *em_table; struct em_perf_state *ps; + unsigned long scale_cpu; int cpu, i; #ifdef CONFIG_SCHED_DEBUG @@ -260,25 +262,23 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, /* * In order to predict the performance state, map the utilization of * the most utilized CPU of the performance domain to a requested - * frequency, like schedutil. Take also into account that the real - * frequency might be set lower (due to thermal capping). Thus, clamp + * performance, like schedutil. Take also into account that the real + * performance might be set lower (due to thermal capping). Thus, clamp * max utilization to the allowed CPU capacity before calculating - * effective frequency. + * effective performance. */ cpu = cpumask_first(to_cpumask(pd->cpus)); scale_cpu = arch_scale_cpu_capacity(cpu); - ref_freq = arch_scale_freq_ref(cpu); max_util = min(max_util, allowed_cpu_cap); - freq = map_util_freq(max_util, ref_freq, scale_cpu); /* * Find the lowest performance state of the Energy Model above the - * requested frequency. + * requested performance. */ em_table = rcu_dereference(pd->em_table); i = em_pd_get_efficient_state(em_table->state, pd->nr_perf_states, - freq, pd->flags); + max_util, pd->flags); ps = &em_table->state[i]; /* diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 667619b70be7..41418aa6daa6 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -46,6 +46,7 @@ static void em_debug_create_ps(struct em_perf_state *ps, struct dentry *pd) debugfs_create_ulong("frequency", 0444, d, &ps->frequency); debugfs_create_ulong("power", 0444, d, &ps->power); debugfs_create_ulong("cost", 0444, d, &ps->cost); + debugfs_create_ulong("performance", 0444, d, &ps->performance); debugfs_create_ulong("inefficient", 0444, d, &ps->flags); } @@ -159,6 +160,30 @@ struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) return table; } +static void em_init_performance(struct device *dev, struct em_perf_domain *pd, + struct em_perf_state *table, int nr_states) +{ + u64 fmax, max_cap; + int i, cpu; + + /* This is needed only for CPUs and EAS skip other devices */ + if (!_is_cpu_device(dev)) + return; + + cpu = cpumask_first(em_span_cpus(pd)); + + /* + * Calculate the performance value for each frequency with + * linear relationship. The final CPU capacity might not be ready at + * boot time, but the EM will be updated a bit later with correct one. + */ + fmax = (u64) table[nr_states - 1].frequency; + max_cap = (u64) arch_scale_cpu_capacity(cpu); + for (i = 0; i < nr_states; i++) + table[i].performance = div64_u64(max_cap * table[i].frequency, + fmax); +} + static int em_compute_costs(struct device *dev, struct em_perf_state *table, struct em_data_callback *cb, int nr_states, unsigned long flags) @@ -318,6 +343,8 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, table[i].frequency = prev_freq = freq; } + em_init_performance(dev, pd, table, nr_states); + ret = em_compute_costs(dev, table, cb, nr_states, flags); if (ret) return -EINVAL; From e3f1164fc9ee8430b3a51e400abfa1b67664f538 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:48 +0000 Subject: [PATCH 0169/1038] PM: EM: Support late CPUs booting and capacity adjustment The patch adds needed infrastructure to handle the late CPUs boot, which might change the previous CPUs capacity values. With this changes the new CPUs which try to register EM will trigger the needed re-calculations for other CPUs EMs. Thanks to that the em_per_state::performance values will be aligned with the CPU capacity information after all CPUs finish the boot and EM registrations. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 124 ++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 41418aa6daa6..b192b0ac8c6e 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -25,6 +25,9 @@ static DEFINE_MUTEX(em_pd_mutex); static void em_cpufreq_update_efficiencies(struct device *dev, struct em_perf_state *table); +static void em_check_capacity_update(void); +static void em_update_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(em_update_work, em_update_workfn); static bool _is_cpu_device(struct device *dev) { @@ -583,6 +586,10 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, unlock: mutex_unlock(&em_pd_mutex); + + if (_is_cpu_device(dev)) + em_check_capacity_update(); + return ret; } EXPORT_SYMBOL_GPL(em_dev_register_perf_domain); @@ -618,3 +625,120 @@ void em_dev_unregister_perf_domain(struct device *dev) mutex_unlock(&em_pd_mutex); } EXPORT_SYMBOL_GPL(em_dev_unregister_perf_domain); + +/* + * Adjustment of CPU performance values after boot, when all CPUs capacites + * are correctly calculated. + */ +static void em_adjust_new_capacity(struct device *dev, + struct em_perf_domain *pd, + u64 max_cap) +{ + struct em_perf_table __rcu *em_table; + struct em_perf_state *ps, *new_ps; + int ret, ps_size; + + em_table = em_table_alloc(pd); + if (!em_table) { + dev_warn(dev, "EM: allocation failed\n"); + return; + } + + new_ps = em_table->state; + + rcu_read_lock(); + ps = em_perf_state_from_pd(pd); + /* Initialize data based on old table */ + ps_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + memcpy(new_ps, ps, ps_size); + + rcu_read_unlock(); + + em_init_performance(dev, pd, new_ps, pd->nr_perf_states); + ret = em_compute_costs(dev, new_ps, NULL, pd->nr_perf_states, + pd->flags); + if (ret) { + dev_warn(dev, "EM: compute costs failed\n"); + return; + } + + ret = em_dev_update_perf_domain(dev, em_table); + if (ret) + dev_warn(dev, "EM: update failed %d\n", ret); + + /* + * This is one-time-update, so give up the ownership in this updater. + * The EM framework has incremented the usage counter and from now + * will keep the reference (then free the memory when needed). + */ + em_table_free(em_table); +} + +static void em_check_capacity_update(void) +{ + cpumask_var_t cpu_done_mask; + struct em_perf_state *table; + struct em_perf_domain *pd; + unsigned long cpu_capacity; + int cpu; + + if (!zalloc_cpumask_var(&cpu_done_mask, GFP_KERNEL)) { + pr_warn("no free memory\n"); + return; + } + + /* Check if CPUs capacity has changed than update EM */ + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy; + unsigned long em_max_perf; + struct device *dev; + int nr_states; + + if (cpumask_test_cpu(cpu, cpu_done_mask)) + continue; + + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_debug("Accessing cpu%d policy failed\n", cpu); + schedule_delayed_work(&em_update_work, + msecs_to_jiffies(1000)); + break; + } + cpufreq_cpu_put(policy); + + pd = em_cpu_get(cpu); + if (!pd || em_is_artificial(pd)) + continue; + + cpumask_or(cpu_done_mask, cpu_done_mask, + em_span_cpus(pd)); + + nr_states = pd->nr_perf_states; + cpu_capacity = arch_scale_cpu_capacity(cpu); + + rcu_read_lock(); + table = em_perf_state_from_pd(pd); + em_max_perf = table[pd->nr_perf_states - 1].performance; + rcu_read_unlock(); + + /* + * Check if the CPU capacity has been adjusted during boot + * and trigger the update for new performance values. + */ + if (em_max_perf == cpu_capacity) + continue; + + pr_debug("updating cpu%d cpu_cap=%lu old capacity=%lu\n", + cpu, cpu_capacity, em_max_perf); + + dev = get_cpu_device(cpu); + em_adjust_new_capacity(dev, pd, cpu_capacity); + } + + free_cpumask_var(cpu_done_mask); +} + +static void em_update_workfn(struct work_struct *work) +{ + em_check_capacity_update(); +} From 1b600da510735a0f92c8b4140a7e2cb037a6a6c3 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:49 +0000 Subject: [PATCH 0170/1038] PM: EM: Optimize em_cpu_energy() and remove division The Energy Model (EM) can be modified at runtime which brings new possibilities. The em_cpu_energy() is called by the Energy Aware Scheduler (EAS) in its hot path. The energy calculation uses power value for a given performance state (ps) and the CPU busy time as percentage for that given frequency. It is possible to avoid the division by 'scale_cpu' at runtime, because EM is updated whenever new max capacity CPU is set in the system. Use that feature and do the needed division during the calculation of the coefficient 'ps->cost'. That enhanced 'ps->cost' value can be then just multiplied simply by utilization: pd_nrg = ps->cost * \Sum cpu_util to get the needed energy for whole Performance Domain (PD). With this optimization and earlier removal of map_util_freq(), the em_cpu_energy() should run faster on the Big CPU by 1.43x and on the Little CPU by 1.69x (RockPi 4B board). Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 55 ++++++++++-------------------------- kernel/power/energy_model.c | 7 ++--- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index ce24ea3fe41c..aabfc26fcd31 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -115,27 +115,6 @@ struct em_perf_domain { #define EM_MAX_NUM_CPUS 16 #endif -/* - * To avoid an overflow on 32bit machines while calculating the energy - * use a different order in the operation. First divide by the 'cpu_scale' - * which would reduce big value stored in the 'cost' field, then multiply by - * the 'sum_util'. This would allow to handle existing platforms, which have - * e.g. power ~1.3 Watt at max freq, so the 'cost' value > 1mln micro-Watts. - * In such scenario, where there are 4 CPUs in the Perf. Domain the 'sum_util' - * could be 4096, then multiplication: 'cost' * 'sum_util' would overflow. - * This reordering of operations has some limitations, we lose small - * precision in the estimation (comparing to 64bit platform w/o reordering). - * - * We are safe on 64bit machine. - */ -#ifdef CONFIG_64BIT -#define em_estimate_energy(cost, sum_util, scale_cpu) \ - (((cost) * (sum_util)) / (scale_cpu)) -#else -#define em_estimate_energy(cost, sum_util, scale_cpu) \ - (((cost) / (scale_cpu)) * (sum_util)) -#endif - struct em_data_callback { /** * active_power() - Provide power at the next performance state of @@ -249,8 +228,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, { struct em_perf_table *em_table; struct em_perf_state *ps; - unsigned long scale_cpu; - int cpu, i; + int i; #ifdef CONFIG_SCHED_DEBUG WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n"); @@ -267,9 +245,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * max utilization to the allowed CPU capacity before calculating * effective performance. */ - cpu = cpumask_first(to_cpumask(pd->cpus)); - scale_cpu = arch_scale_cpu_capacity(cpu); - + max_util = map_util_perf(max_util); max_util = min(max_util, allowed_cpu_cap); /* @@ -282,12 +258,12 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, ps = &em_table->state[i]; /* - * The capacity of a CPU in the domain at the performance state (ps) - * can be computed as: + * The performance (capacity) of a CPU in the domain at the performance + * state (ps) can be computed as: * - * ps->freq * scale_cpu - * ps->cap = -------------------- (1) - * cpu_max_freq + * ps->freq * scale_cpu + * ps->performance = -------------------- (1) + * cpu_max_freq * * So, ignoring the costs of idle states (which are not available in * the EM), the energy consumed by this CPU at that performance state @@ -295,9 +271,10 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * * ps->power * cpu_util * cpu_nrg = -------------------- (2) - * ps->cap + * ps->performance * - * since 'cpu_util / ps->cap' represents its percentage of busy time. + * since 'cpu_util / ps->performance' represents its percentage of busy + * time. * * NOTE: Although the result of this computation actually is in * units of power, it can be manipulated as an energy value @@ -307,9 +284,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * By injecting (1) in (2), 'cpu_nrg' can be re-expressed as a product * of two terms: * - * ps->power * cpu_max_freq cpu_util - * cpu_nrg = ------------------------ * --------- (3) - * ps->freq scale_cpu + * ps->power * cpu_max_freq + * cpu_nrg = ------------------------ * cpu_util (3) + * ps->freq * scale_cpu * * The first term is static, and is stored in the em_perf_state struct * as 'ps->cost'. @@ -319,11 +296,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * total energy of the domain (which is the simple sum of the energy of * all of its CPUs) can be factorized as: * - * ps->cost * \Sum cpu_util - * pd_nrg = ------------------------ (4) - * scale_cpu + * pd_nrg = ps->cost * \Sum cpu_util (4) */ - return em_estimate_energy(ps->cost, sum_util, scale_cpu); + return ps->cost * sum_util; } /** diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index b192b0ac8c6e..a631d7d52c40 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -192,11 +192,9 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, unsigned long flags) { unsigned long prev_cost = ULONG_MAX; - u64 fmax; int i, ret; /* Compute the cost of each performance state. */ - fmax = (u64) table[nr_states - 1].frequency; for (i = nr_states - 1; i >= 0; i--) { unsigned long power_res, cost; @@ -208,8 +206,9 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return -EINVAL; } } else { - power_res = table[i].power; - cost = div64_u64(fmax * power_res, table[i].frequency); + /* increase resolution of 'cost' precision */ + power_res = table[i].power * 10; + cost = power_res / table[i].performance; } table[i].cost = cost; From e20b7a8172b5f6fea82d063c8f1f4df881701759 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:50 +0000 Subject: [PATCH 0171/1038] powercap/dtpm_cpu: Use new Energy Model interface to get table Energy Model framework support modifications at runtime of the power values. Use the new EM table API which is protected with RCU. Align the code so that this RCU read section is short. This change is not expected to alter the general functionality. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/powercap/dtpm_cpu.c | 39 +++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c index 9193c3b8edeb..ee0d1aa3e023 100644 --- a/drivers/powercap/dtpm_cpu.c +++ b/drivers/powercap/dtpm_cpu.c @@ -42,6 +42,7 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) { struct dtpm_cpu *dtpm_cpu = to_dtpm_cpu(dtpm); struct em_perf_domain *pd = em_cpu_get(dtpm_cpu->cpu); + struct em_perf_state *table; struct cpumask cpus; unsigned long freq; u64 power; @@ -50,20 +51,22 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus)); nr_cpus = cpumask_weight(&cpus); + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - power = pd->table[i].power * nr_cpus; + power = table[i].power * nr_cpus; if (power > power_limit) break; } - freq = pd->table[i - 1].frequency; + freq = table[i - 1].frequency; + power_limit = table[i - 1].power * nr_cpus; + rcu_read_unlock(); freq_qos_update_request(&dtpm_cpu->qos_req, freq); - power_limit = pd->table[i - 1].power * nr_cpus; - return power_limit; } @@ -87,9 +90,11 @@ static u64 scale_pd_power_uw(struct cpumask *pd_mask, u64 power) static u64 get_pd_power_uw(struct dtpm *dtpm) { struct dtpm_cpu *dtpm_cpu = to_dtpm_cpu(dtpm); + struct em_perf_state *table; struct em_perf_domain *pd; struct cpumask *pd_mask; unsigned long freq; + u64 power = 0; int i; pd = em_cpu_get(dtpm_cpu->cpu); @@ -98,33 +103,43 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) freq = cpufreq_quick_get(dtpm_cpu->cpu); + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - if (pd->table[i].frequency < freq) + if (table[i].frequency < freq) continue; - return scale_pd_power_uw(pd_mask, pd->table[i].power); + power = scale_pd_power_uw(pd_mask, table[i].power); + break; } + rcu_read_unlock(); - return 0; + return power; } static int update_pd_power_uw(struct dtpm *dtpm) { struct dtpm_cpu *dtpm_cpu = to_dtpm_cpu(dtpm); struct em_perf_domain *em = em_cpu_get(dtpm_cpu->cpu); + struct em_perf_state *table; struct cpumask cpus; int nr_cpus; cpumask_and(&cpus, cpu_online_mask, to_cpumask(em->cpus)); nr_cpus = cpumask_weight(&cpus); - dtpm->power_min = em->table[0].power; + rcu_read_lock(); + table = em_perf_state_from_pd(em); + + dtpm->power_min = table[0].power; dtpm->power_min *= nr_cpus; - dtpm->power_max = em->table[em->nr_perf_states - 1].power; + dtpm->power_max = table[em->nr_perf_states - 1].power; dtpm->power_max *= nr_cpus; + rcu_read_unlock(); + return 0; } @@ -180,6 +195,7 @@ static int __dtpm_cpu_setup(int cpu, struct dtpm *parent) { struct dtpm_cpu *dtpm_cpu; struct cpufreq_policy *policy; + struct em_perf_state *table; struct em_perf_domain *pd; char name[CPUFREQ_NAME_LEN]; int ret = -ENOMEM; @@ -216,9 +232,12 @@ static int __dtpm_cpu_setup(int cpu, struct dtpm *parent) if (ret) goto out_kfree_dtpm_cpu; + rcu_read_lock(); + table = em_perf_state_from_pd(pd); ret = freq_qos_add_request(&policy->constraints, &dtpm_cpu->qos_req, FREQ_QOS_MAX, - pd->table[pd->nr_perf_states - 1].frequency); + table[pd->nr_perf_states - 1].frequency); + rcu_read_unlock(); if (ret) goto out_dtpm_unregister; From 27d2c37e7dea41801378c473089b80385ca65491 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:51 +0000 Subject: [PATCH 0172/1038] powercap/dtpm_devfreq: Use new Energy Model interface to get table Energy Model framework support modifications at runtime of the power values. Use the new EM table API which is protected with RCU. Align the code so that this RCU read section is short. This change is not expected to alter the general functionality. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/powercap/dtpm_devfreq.c | 34 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/drivers/powercap/dtpm_devfreq.c b/drivers/powercap/dtpm_devfreq.c index 612c3b59dd5b..f40bce8176df 100644 --- a/drivers/powercap/dtpm_devfreq.c +++ b/drivers/powercap/dtpm_devfreq.c @@ -37,11 +37,16 @@ static int update_pd_power_uw(struct dtpm *dtpm) struct devfreq *devfreq = dtpm_devfreq->devfreq; struct device *dev = devfreq->dev.parent; struct em_perf_domain *pd = em_pd_get(dev); + struct em_perf_state *table; - dtpm->power_min = pd->table[0].power; + rcu_read_lock(); + table = em_perf_state_from_pd(pd); - dtpm->power_max = pd->table[pd->nr_perf_states - 1].power; + dtpm->power_min = table[0].power; + dtpm->power_max = table[pd->nr_perf_states - 1].power; + + rcu_read_unlock(); return 0; } @@ -51,20 +56,23 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) struct devfreq *devfreq = dtpm_devfreq->devfreq; struct device *dev = devfreq->dev.parent; struct em_perf_domain *pd = em_pd_get(dev); + struct em_perf_state *table; unsigned long freq; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - if (pd->table[i].power > power_limit) + if (table[i].power > power_limit) break; } - freq = pd->table[i - 1].frequency; + freq = table[i - 1].frequency; + power_limit = table[i - 1].power; + rcu_read_unlock(); dev_pm_qos_update_request(&dtpm_devfreq->qos_req, freq); - power_limit = pd->table[i - 1].power; - return power_limit; } @@ -89,8 +97,9 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) struct device *dev = devfreq->dev.parent; struct em_perf_domain *pd = em_pd_get(dev); struct devfreq_dev_status status; + struct em_perf_state *table; unsigned long freq; - u64 power; + u64 power = 0; int i; mutex_lock(&devfreq->lock); @@ -100,19 +109,22 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) freq = DIV_ROUND_UP(status.current_frequency, HZ_PER_KHZ); _normalize_load(&status); + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - if (pd->table[i].frequency < freq) + if (table[i].frequency < freq) continue; - power = pd->table[i].power; + power = table[i].power; power *= status.busy_time; power >>= 10; - return power; + break; } + rcu_read_unlock(); - return 0; + return power; } static void pd_release(struct dtpm *dtpm) From 207472b8ef715e8089424388c322e8b1633d5254 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:52 +0000 Subject: [PATCH 0173/1038] drivers/thermal/cpufreq_cooling: Use new Energy Model interface Energy Model framework support modifications at runtime of the power values. Use the new EM table which is protected with RCU. Align the code so that this RCU read section is short. This change is not expected to alter the general functionality. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/thermal/cpufreq_cooling.c | 45 +++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index e2cc7bd30862..9d1b1459700d 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -91,12 +91,16 @@ struct cpufreq_cooling_device { static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, unsigned int freq) { + struct em_perf_state *table; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { - if (freq > cpufreq_cdev->em->table[i].frequency) + if (freq > table[i].frequency) break; } + rcu_read_unlock(); return cpufreq_cdev->max_level - i - 1; } @@ -104,16 +108,20 @@ static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, u32 freq) { + struct em_perf_state *table; unsigned long power_mw; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { - if (freq > cpufreq_cdev->em->table[i].frequency) + if (freq > table[i].frequency) break; } - power_mw = cpufreq_cdev->em->table[i + 1].power; + power_mw = table[i + 1].power; power_mw /= MICROWATT_PER_MILLIWATT; + rcu_read_unlock(); return power_mw; } @@ -121,18 +129,24 @@ static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, u32 power) { + struct em_perf_state *table; unsigned long em_power_mw; + u32 freq; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); for (i = cpufreq_cdev->max_level; i > 0; i--) { /* Convert EM power to milli-Watts to make safe comparison */ - em_power_mw = cpufreq_cdev->em->table[i].power; + em_power_mw = table[i].power; em_power_mw /= MICROWATT_PER_MILLIWATT; if (power >= em_power_mw) break; } + freq = table[i].frequency; + rcu_read_unlock(); - return cpufreq_cdev->em->table[i].frequency; + return freq; } /** @@ -262,8 +276,9 @@ static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, static int cpufreq_state2power(struct thermal_cooling_device *cdev, unsigned long state, u32 *power) { - unsigned int freq, num_cpus, idx; struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; + unsigned int freq, num_cpus, idx; + struct em_perf_state *table; /* Request state should be less than max_level */ if (state > cpufreq_cdev->max_level) @@ -272,7 +287,12 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus); idx = cpufreq_cdev->max_level - state; - freq = cpufreq_cdev->em->table[idx].frequency; + + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); + freq = table[idx].frequency; + rcu_read_unlock(); + *power = cpu_freq_to_power(cpufreq_cdev, freq) * num_cpus; return 0; @@ -378,8 +398,17 @@ static unsigned int get_state_freq(struct cpufreq_cooling_device *cpufreq_cdev, #ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR /* Use the Energy Model table if available */ if (cpufreq_cdev->em) { + struct em_perf_state *table; + unsigned int freq; + idx = cpufreq_cdev->max_level - state; - return cpufreq_cdev->em->table[idx].frequency; + + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); + freq = table[idx].frequency; + rcu_read_unlock(); + + return freq; } #endif From 9f5fb518c3c022c5c50883c9006f6f2cd00d51d7 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:53 +0000 Subject: [PATCH 0174/1038] drivers/thermal/devfreq_cooling: Use new Energy Model interface Energy Model framework support modifications at runtime of the power values. Use the new EM table which is protected with RCU. Align the code so that this RCU read section is short. This change is not expected to alter the general functionality. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/thermal/devfreq_cooling.c | 51 +++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c index 262e62ab6cf2..50dec24e967a 100644 --- a/drivers/thermal/devfreq_cooling.c +++ b/drivers/thermal/devfreq_cooling.c @@ -87,6 +87,7 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, struct devfreq_cooling_device *dfc = cdev->devdata; struct devfreq *df = dfc->devfreq; struct device *dev = df->dev.parent; + struct em_perf_state *table; unsigned long freq; int perf_idx; @@ -100,7 +101,11 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, if (dfc->em_pd) { perf_idx = dfc->max_state - state; - freq = dfc->em_pd->table[perf_idx].frequency * 1000; + + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + freq = table[perf_idx].frequency * 1000; + rcu_read_unlock(); } else { freq = dfc->freq_table[state]; } @@ -123,14 +128,21 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, */ static int get_perf_idx(struct em_perf_domain *em_pd, unsigned long freq) { - int i; + struct em_perf_state *table; + int i, idx = -EINVAL; + rcu_read_lock(); + table = em_perf_state_from_pd(em_pd); for (i = 0; i < em_pd->nr_perf_states; i++) { - if (em_pd->table[i].frequency == freq) - return i; - } + if (table[i].frequency != freq) + continue; - return -EINVAL; + idx = i; + break; + } + rcu_read_unlock(); + + return idx; } static unsigned long get_voltage(struct devfreq *df, unsigned long freq) @@ -181,6 +193,7 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd struct devfreq_cooling_device *dfc = cdev->devdata; struct devfreq *df = dfc->devfreq; struct devfreq_dev_status status; + struct em_perf_state *table; unsigned long state; unsigned long freq; unsigned long voltage; @@ -204,7 +217,11 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd state = dfc->capped_state; /* Convert EM power into milli-Watts first */ - dfc->res_util = dfc->em_pd->table[state].power; + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + dfc->res_util = table[state].power; + rcu_read_unlock(); + dfc->res_util /= MICROWATT_PER_MILLIWATT; dfc->res_util *= SCALE_ERROR_MITIGATION; @@ -225,7 +242,11 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd _normalize_load(&status); /* Convert EM power into milli-Watts first */ - *power = dfc->em_pd->table[perf_idx].power; + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + *power = table[perf_idx].power; + rcu_read_unlock(); + *power /= MICROWATT_PER_MILLIWATT; /* Scale power for utilization */ *power *= status.busy_time; @@ -245,13 +266,19 @@ static int devfreq_cooling_state2power(struct thermal_cooling_device *cdev, unsigned long state, u32 *power) { struct devfreq_cooling_device *dfc = cdev->devdata; + struct em_perf_state *table; int perf_idx; if (state > dfc->max_state) return -EINVAL; perf_idx = dfc->max_state - state; - *power = dfc->em_pd->table[perf_idx].power; + + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + *power = table[perf_idx].power; + rcu_read_unlock(); + *power /= MICROWATT_PER_MILLIWATT; return 0; @@ -264,6 +291,7 @@ static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, struct devfreq *df = dfc->devfreq; struct devfreq_dev_status status; unsigned long freq, em_power_mw; + struct em_perf_state *table; s32 est_power; int i; @@ -288,13 +316,16 @@ static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, * Find the first cooling state that is within the power * budget. The EM power table is sorted ascending. */ + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); for (i = dfc->max_state; i > 0; i--) { /* Convert EM power to milli-Watts to make safe comparison */ - em_power_mw = dfc->em_pd->table[i].power; + em_power_mw = table[i].power; em_power_mw /= MICROWATT_PER_MILLIWATT; if (est_power >= em_power_mw) break; } + rcu_read_unlock(); *state = dfc->max_state - i; dfc->capped_state = *state; From 09417e673cbd578a1eaf8aba34a668119622d79c Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:54 +0000 Subject: [PATCH 0175/1038] PM: EM: Change debugfs configuration to use runtime EM table data Dump the runtime EM table values which can be modified in time. In order to do that allocate chunk of debug memory which can be later freed automatically thanks to devm_kcalloc(). This design can handle the fact that the EM table memory can change after EM update, so debug code cannot use the pointer from initialization phase. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 67 ++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index a631d7d52c40..548908e686ed 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -37,20 +37,65 @@ static bool _is_cpu_device(struct device *dev) #ifdef CONFIG_DEBUG_FS static struct dentry *rootdir; -static void em_debug_create_ps(struct em_perf_state *ps, struct dentry *pd) +struct em_dbg_info { + struct em_perf_domain *pd; + int ps_id; +}; + +#define DEFINE_EM_DBG_SHOW(name, fname) \ +static int em_debug_##fname##_show(struct seq_file *s, void *unused) \ +{ \ + struct em_dbg_info *em_dbg = s->private; \ + struct em_perf_state *table; \ + unsigned long val; \ + \ + rcu_read_lock(); \ + table = em_perf_state_from_pd(em_dbg->pd); \ + val = table[em_dbg->ps_id].name; \ + rcu_read_unlock(); \ + \ + seq_printf(s, "%lu\n", val); \ + return 0; \ +} \ +DEFINE_SHOW_ATTRIBUTE(em_debug_##fname) + +DEFINE_EM_DBG_SHOW(frequency, frequency); +DEFINE_EM_DBG_SHOW(power, power); +DEFINE_EM_DBG_SHOW(cost, cost); +DEFINE_EM_DBG_SHOW(performance, performance); +DEFINE_EM_DBG_SHOW(flags, inefficiency); + +static void em_debug_create_ps(struct em_perf_domain *em_pd, + struct em_dbg_info *em_dbg, int i, + struct dentry *pd) { + struct em_perf_state *table; + unsigned long freq; struct dentry *d; char name[24]; - snprintf(name, sizeof(name), "ps:%lu", ps->frequency); + em_dbg[i].pd = em_pd; + em_dbg[i].ps_id = i; + + rcu_read_lock(); + table = em_perf_state_from_pd(em_pd); + freq = table[i].frequency; + rcu_read_unlock(); + + snprintf(name, sizeof(name), "ps:%lu", freq); /* Create per-ps directory */ d = debugfs_create_dir(name, pd); - debugfs_create_ulong("frequency", 0444, d, &ps->frequency); - debugfs_create_ulong("power", 0444, d, &ps->power); - debugfs_create_ulong("cost", 0444, d, &ps->cost); - debugfs_create_ulong("performance", 0444, d, &ps->performance); - debugfs_create_ulong("inefficient", 0444, d, &ps->flags); + debugfs_create_file("frequency", 0444, d, &em_dbg[i], + &em_debug_frequency_fops); + debugfs_create_file("power", 0444, d, &em_dbg[i], + &em_debug_power_fops); + debugfs_create_file("cost", 0444, d, &em_dbg[i], + &em_debug_cost_fops); + debugfs_create_file("performance", 0444, d, &em_dbg[i], + &em_debug_performance_fops); + debugfs_create_file("inefficient", 0444, d, &em_dbg[i], + &em_debug_inefficiency_fops); } static int em_debug_cpus_show(struct seq_file *s, void *unused) @@ -73,6 +118,7 @@ DEFINE_SHOW_ATTRIBUTE(em_debug_flags); static void em_debug_create_pd(struct device *dev) { + struct em_dbg_info *em_dbg; struct dentry *d; int i; @@ -86,9 +132,14 @@ static void em_debug_create_pd(struct device *dev) debugfs_create_file("flags", 0444, d, dev->em_pd, &em_debug_flags_fops); + em_dbg = devm_kcalloc(dev, dev->em_pd->nr_perf_states, + sizeof(*em_dbg), GFP_KERNEL); + if (!em_dbg) + return; + /* Create a sub-directory for each performance state */ for (i = 0; i < dev->em_pd->nr_perf_states; i++) - em_debug_create_ps(&dev->em_pd->table[i], d); + em_debug_create_ps(dev->em_pd, em_dbg, i, d); } From 24e9fb635df2790eccb0e95ff65c6dee7a97fcb7 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:55 +0000 Subject: [PATCH 0176/1038] PM: EM: Remove old table Remove the old EM table which wasn't able to modify the data. Clean the unneeded function and refactor the code a bit. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 2 -- kernel/power/energy_model.c | 46 ++++++------------------------------ 2 files changed, 7 insertions(+), 41 deletions(-) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index aabfc26fcd31..92866a81abe4 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -53,7 +53,6 @@ struct em_perf_table { /** * struct em_perf_domain - Performance domain - * @table: List of performance states, in ascending order * @em_table: Pointer to the runtime modifiable em_perf_table * @nr_perf_states: Number of performance states * @flags: See "em_perf_domain flags" @@ -69,7 +68,6 @@ struct em_perf_table { * field is unused. */ struct em_perf_domain { - struct em_perf_state *table; struct em_perf_table __rcu *em_table; int nr_perf_states; unsigned long flags; diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 548908e686ed..57838d28af85 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -276,17 +276,6 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return 0; } -static int em_allocate_perf_table(struct em_perf_domain *pd, - int nr_states) -{ - pd->table = kcalloc(nr_states, sizeof(struct em_perf_state), - GFP_KERNEL); - if (!pd->table) - return -ENOMEM; - - return 0; -} - /** * em_dev_update_perf_domain() - Update runtime EM table for a device * @dev : Device for which the EM is to be updated @@ -331,24 +320,6 @@ int em_dev_update_perf_domain(struct device *dev, } EXPORT_SYMBOL_GPL(em_dev_update_perf_domain); -static int em_create_runtime_table(struct em_perf_domain *pd) -{ - struct em_perf_table __rcu *table; - int table_size; - - table = em_table_alloc(pd); - if (!table) - return -ENOMEM; - - /* Initialize runtime table with existing data */ - table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; - memcpy(table->state, pd->table, table_size); - - rcu_assign_pointer(pd->em_table, table); - - return 0; -} - static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, struct em_perf_state *table, struct em_data_callback *cb, @@ -409,6 +380,7 @@ static int em_create_pd(struct device *dev, int nr_states, struct em_data_callback *cb, cpumask_t *cpus, unsigned long flags) { + struct em_perf_table __rcu *em_table; struct em_perf_domain *pd; struct device *cpu_dev; int cpu, ret, num_cpus; @@ -435,17 +407,15 @@ static int em_create_pd(struct device *dev, int nr_states, pd->nr_perf_states = nr_states; - ret = em_allocate_perf_table(pd, nr_states); - if (ret) + em_table = em_table_alloc(pd); + if (!em_table) goto free_pd; - ret = em_create_perf_table(dev, pd, pd->table, cb, flags); + ret = em_create_perf_table(dev, pd, em_table->state, cb, flags); if (ret) goto free_pd_table; - ret = em_create_runtime_table(pd); - if (ret) - goto free_pd_table; + rcu_assign_pointer(pd->em_table, em_table); if (_is_cpu_device(dev)) for_each_cpu(cpu, cpus) { @@ -458,7 +428,7 @@ static int em_create_pd(struct device *dev, int nr_states, return 0; free_pd_table: - kfree(pd->table); + kfree(em_table); free_pd: kfree(pd); return -EINVAL; @@ -629,7 +599,7 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, dev->em_pd->flags |= flags; - em_cpufreq_update_efficiencies(dev, dev->em_pd->table); + em_cpufreq_update_efficiencies(dev, dev->em_pd->em_table->state); em_debug_create_pd(dev); dev_info(dev, "EM: created perf domain\n"); @@ -666,8 +636,6 @@ void em_dev_unregister_perf_domain(struct device *dev) mutex_lock(&em_pd_mutex); em_debug_remove_pd(dev); - kfree(dev->em_pd->table); - em_table_free(dev->em_pd->em_table); kfree(dev->em_pd); From 22ea02848c07d1cbd15a5f442138ca429866300d Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:56 +0000 Subject: [PATCH 0177/1038] PM: EM: Add em_dev_compute_costs() The device drivers can modify EM at runtime by providing a new EM table. The EM is used by the EAS and the em_perf_state::cost stores pre-calculated value to avoid overhead. This patch provides the API for device drivers to calculate the cost values properly (and not duplicate the same code). Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/energy_model.h | 8 ++++++++ kernel/power/energy_model.c | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 92866a81abe4..770755df852f 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -170,6 +170,8 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, void em_dev_unregister_perf_domain(struct device *dev); struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); void em_table_free(struct em_perf_table __rcu *table); +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM @@ -379,6 +381,12 @@ struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) { return NULL; } +static inline +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states) +{ + return -EINVAL; +} #endif #endif diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 57838d28af85..7101fa3fa0c0 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -276,6 +276,24 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, return 0; } +/** + * em_dev_compute_costs() - Calculate cost values for new runtime EM table + * @dev : Device for which the EM table is to be updated + * @table : The new EM table that is going to get the costs calculated + * + * Calculate the em_perf_state::cost values for new runtime EM table. The + * values are used for EAS during task placement. It also calculates and sets + * the efficiency flag for each performance state. When the function finish + * successfully the EM table is ready to be updated and used by EAS. + * + * Return 0 on success or a proper error in case of failure. + */ +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states) +{ + return em_compute_costs(dev, table, NULL, nr_states, 0); +} + /** * em_dev_update_perf_domain() - Update runtime EM table for a device * @dev : Device for which the EM is to be updated From eb1ad4d431674368bab4ef4e312f2e8c158294f9 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Thu, 8 Feb 2024 11:55:57 +0000 Subject: [PATCH 0178/1038] Documentation: EM: Update with runtime modification design Add a new description which covers the information about runtime EM. It contains the design decisions, describes models and how they reflect the reality. Remove description of the default EM. Add example driver code which modifies EM. Add API documentation for the new feature which allows to modify the EM in runtime. Reviewed-by: Dietmar Eggemann Tested-by: Dietmar Eggemann Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- Documentation/power/energy-model.rst | 183 ++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 4 deletions(-) diff --git a/Documentation/power/energy-model.rst b/Documentation/power/energy-model.rst index 13225965c9a4..ada4938c37e5 100644 --- a/Documentation/power/energy-model.rst +++ b/Documentation/power/energy-model.rst @@ -71,6 +71,31 @@ whose performance is scaled together. Performance domains generally have a required to have the same micro-architecture. CPUs in different performance domains can have different micro-architectures. +To better reflect power variation due to static power (leakage) the EM +supports runtime modifications of the power values. The mechanism relies on +RCU to free the modifiable EM perf_state table memory. Its user, the task +scheduler, also uses RCU to access this memory. The EM framework provides +API for allocating/freeing the new memory for the modifiable EM table. +The old memory is freed automatically using RCU callback mechanism when there +are no owners anymore for the given EM runtime table instance. This is tracked +using kref mechanism. The device driver which provided the new EM at runtime, +should call EM API to free it safely when it's no longer needed. The EM +framework will handle the clean-up when it's possible. + +The kernel code which want to modify the EM values is protected from concurrent +access using a mutex. Therefore, the device driver code must run in sleeping +context when it tries to modify the EM. + +With the runtime modifiable EM we switch from a 'single and during the entire +runtime static EM' (system property) design to a 'single EM which can be +changed during runtime according e.g. to the workload' (system and workload +property) design. + +It is possible also to modify the CPU performance values for each EM's +performance state. Thus, the full power and performance profile (which +is an exponential curve) can be changed according e.g. to the workload +or system property. + 2. Core APIs ------------ @@ -175,10 +200,82 @@ CPUfreq governor is in use in case of CPU device. Currently this calculation is not provided for other type of devices. More details about the above APIs can be found in ```` -or in Section 2.4 +or in Section 2.5 -2.4 Description details of this API +2.4 Runtime modifications +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Drivers willing to update the EM at runtime should use the following dedicated +function to allocate a new instance of the modified EM. The API is listed +below:: + + struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); + +This allows to allocate a structure which contains the new EM table with +also RCU and kref needed by the EM framework. The 'struct em_perf_table' +contains array 'struct em_perf_state state[]' which is a list of performance +states in ascending order. That list must be populated by the device driver +which wants to update the EM. The list of frequencies can be taken from +existing EM (created during boot). The content in the 'struct em_perf_state' +must be populated by the driver as well. + +This is the API which does the EM update, using RCU pointers swap:: + + int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table); + +Drivers must provide a pointer to the allocated and initialized new EM +'struct em_perf_table'. That new EM will be safely used inside the EM framework +and will be visible to other sub-systems in the kernel (thermal, powercap). +The main design goal for this API is to be fast and avoid extra calculations +or memory allocations at runtime. When pre-computed EMs are available in the +device driver, than it should be possible to simply re-use them with low +performance overhead. + +In order to free the EM, provided earlier by the driver (e.g. when the module +is unloaded), there is a need to call the API:: + + void em_table_free(struct em_perf_table __rcu *table); + +It will allow the EM framework to safely remove the memory, when there is +no other sub-system using it, e.g. EAS. + +To use the power values in other sub-systems (like thermal, powercap) there is +a need to call API which protects the reader and provide consistency of the EM +table data:: + + struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd); + +It returns the 'struct em_perf_state' pointer which is an array of performance +states in ascending order. +This function must be called in the RCU read lock section (after the +rcu_read_lock()). When the EM table is not needed anymore there is a need to +call rcu_real_unlock(). In this way the EM safely uses the RCU read section +and protects the users. It also allows the EM framework to manage the memory +and free it. More details how to use it can be found in Section 3.2 in the +example driver. + +There is dedicated API for device drivers to calculate em_perf_state::cost +values:: + + int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states); + +These 'cost' values from EM are used in EAS. The new EM table should be passed +together with the number of entries and device pointer. When the computation +of the cost values is done properly the return value from the function is 0. +The function takes care for right setting of inefficiency for each performance +state as well. It updates em_perf_state::flags accordingly. +Then such prepared new EM can be passed to the em_dev_update_perf_domain() +function, which will allow to use it. + +More details about the above APIs can be found in ```` +or in Section 3.2 with an example code showing simple implementation of the +updating mechanism in a device driver. + + +2.5 Description details of this API ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. kernel-doc:: include/linux/energy_model.h :internal: @@ -187,8 +284,11 @@ or in Section 2.4 :export: -3. Example driver ------------------ +3. Examples +----------- + +3.1 Example driver with EM registration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The CPUFreq framework supports dedicated callback for registering the EM for a given CPU(s) 'policy' object: cpufreq_driver::register_em(). @@ -242,3 +342,78 @@ EM framework:: 39 static struct cpufreq_driver foo_cpufreq_driver = { 40 .register_em = foo_cpufreq_register_em, 41 }; + + +3.2 Example driver with EM modification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section provides a simple example of a thermal driver modifying the EM. +The driver implements a foo_thermal_em_update() function. The driver is woken +up periodically to check the temperature and modify the EM data:: + + -> drivers/soc/example/example_em_mod.c + + 01 static void foo_get_new_em(struct foo_context *ctx) + 02 { + 03 struct em_perf_table __rcu *em_table; + 04 struct em_perf_state *table, *new_table; + 05 struct device *dev = ctx->dev; + 06 struct em_perf_domain *pd; + 07 unsigned long freq; + 08 int i, ret; + 09 + 10 pd = em_pd_get(dev); + 11 if (!pd) + 12 return; + 13 + 14 em_table = em_table_alloc(pd); + 15 if (!em_table) + 16 return; + 17 + 18 new_table = em_table->state; + 19 + 20 rcu_read_lock(); + 21 table = em_perf_state_from_pd(pd); + 22 for (i = 0; i < pd->nr_perf_states; i++) { + 23 freq = table[i].frequency; + 24 foo_get_power_perf_values(dev, freq, &new_table[i]); + 25 } + 26 rcu_read_unlock(); + 27 + 28 /* Calculate 'cost' values for EAS */ + 29 ret = em_dev_compute_costs(dev, table, pd->nr_perf_states); + 30 if (ret) { + 31 dev_warn(dev, "EM: compute costs failed %d\n", ret); + 32 em_free_table(em_table); + 33 return; + 34 } + 35 + 36 ret = em_dev_update_perf_domain(dev, em_table); + 37 if (ret) { + 38 dev_warn(dev, "EM: update failed %d\n", ret); + 39 em_free_table(em_table); + 40 return; + 41 } + 42 + 43 /* + 44 * Since it's one-time-update drop the usage counter. + 45 * The EM framework will later free the table when needed. + 46 */ + 47 em_table_free(em_table); + 48 } + 49 + 50 /* + 51 * Function called periodically to check the temperature and + 52 * update the EM if needed + 53 */ + 54 static void foo_thermal_em_update(struct foo_context *ctx) + 55 { + 56 struct device *dev = ctx->dev; + 57 int cpu; + 58 + 59 ctx->temperature = foo_get_temp(dev, ctx); + 60 if (ctx->temperature < FOO_EM_UPDATE_TEMP_THRESHOLD) + 61 return; + 62 + 63 foo_get_new_em(ctx); + 64 } From a77ce80f63f06d7ae933c332ed77c79136fa69b0 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:15 +0000 Subject: [PATCH 0179/1038] spi: s3c64xx: sort headers alphabetically Sorting headers alphabetically helps locating duplicates, and makes it easier to figure out where to insert new headers. Reviewed-by: Andi Shyti Reviewed-by: Peter Griffin Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-2-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 7f7eb8f742e4..8faa66bfa84e 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -3,19 +3,18 @@ // Copyright (c) 2009 Samsung Electronics Co., Ltd. // Jaswinder Singh -#include -#include -#include -#include #include +#include #include #include +#include +#include +#include +#include +#include #include #include #include -#include - -#include #define MAX_SPI_PORTS 12 #define S3C64XX_SPI_QUIRK_CS_AUTO (1 << 1) From 42a9ac378d918176e17592cbe79d8b0606f951e4 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:16 +0000 Subject: [PATCH 0180/1038] spi: s3c64xx: explicitly include The driver uses readl() but does not include . It is good practice to directly include all headers used, it avoids implicit dependencies and spurious breakage if someone rearranges headers and causes the implicit include to vanish. Include the missing header. Reviewed-by: Peter Griffin Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-3-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 8faa66bfa84e..9b8dd05360fa 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include From 4568fa574fcef3811a8140702979f076ef0f5bc0 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:17 +0000 Subject: [PATCH 0181/1038] spi: s3c64xx: explicitly include The driver uses GENMASK() but does not include . It is good practice to directly include all headers used, it avoids implicit dependencies and spurious breakage if someone rearranges headers and causes the implicit include to vanish. Include the missing header. Reviewed-by: Peter Griffin Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-4-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 9b8dd05360fa..5c9ac9d8dbd6 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -3,6 +3,7 @@ // Copyright (c) 2009 Samsung Electronics Co., Ltd. // Jaswinder Singh +#include #include #include #include From 7256d6bdd4fe0eac6d4bcd138c3d87f95f79c750 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:18 +0000 Subject: [PATCH 0182/1038] spi: s3c64xx: explicitly include The driver uses u32 and relies on an implicit inclusion of . It is good practice to directly include all headers used, it avoids implicit dependencies and spurious breakage if someone rearranges headers and causes the implicit include to vanish. Include the missing header. Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-5-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 5c9ac9d8dbd6..f6048345022d 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -17,6 +17,7 @@ #include #include #include +#include #define MAX_SPI_PORTS 12 #define S3C64XX_SPI_QUIRK_CS_AUTO (1 << 1) From a336d41bbea51e11e3e4f56bd3877a535c077129 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:19 +0000 Subject: [PATCH 0183/1038] spi: s3c64xx: avoid possible negative array index The platform id is used as an index into the fifo_lvl_mask array. Platforms can come with a negative device ID, PLATFORM_DEVID_NONE (-1), thus we risked a negative array index. Catch such cases and fail to probe. Reviewed-by: Sam Protsenko Reviewed-by: Peter Griffin Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-6-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index f6048345022d..87b7203f9b1f 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1189,6 +1189,9 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) "Failed to get alias id\n"); sdd->port_id = ret; } else { + if (pdev->id < 0) + return dev_err_probe(&pdev->dev, -EINVAL, + "Negative platform ID is not allowed\n"); sdd->port_id = pdev->id; } From 97b63f4707046b2ef99d077dd4d333c3acca06ae Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:20 +0000 Subject: [PATCH 0184/1038] spi: s3c64xx: fix typo, s/configuartion/configuration Fix typo, s/configuartion/configuration. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-7-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 87b7203f9b1f..6b93e8a0c637 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -180,7 +180,7 @@ struct s3c64xx_spi_port_config { * @cur_speed: Current clock speed * @rx_dma: Local receive DMA data (e.g. chan and direction) * @tx_dma: Local transmit DMA data (e.g. chan and direction) - * @port_conf: Local SPI port configuartion data + * @port_conf: Local SPI port configuration data * @port_id: Port identification number */ struct s3c64xx_spi_driver_data { From 271f18816b3ba2f75785660e427c16585b7302f2 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:21 +0000 Subject: [PATCH 0185/1038] spi: s3c64xx: remove unneeded (void *) casts in of_match_table of_device_id::data is an opaque pointer. No explicit cast is needed. Remove unneeded (void *) casts in of_match_table. Reviewed-by: Andi Shyti Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-8-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 6b93e8a0c637..3f8fc9d5a508 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1513,31 +1513,31 @@ static const struct platform_device_id s3c64xx_spi_driver_ids[] = { static const struct of_device_id s3c64xx_spi_dt_match[] = { { .compatible = "samsung,s3c2443-spi", - .data = (void *)&s3c2443_spi_port_config, + .data = &s3c2443_spi_port_config, }, { .compatible = "samsung,s3c6410-spi", - .data = (void *)&s3c6410_spi_port_config, + .data = &s3c6410_spi_port_config, }, { .compatible = "samsung,s5pv210-spi", - .data = (void *)&s5pv210_spi_port_config, + .data = &s5pv210_spi_port_config, }, { .compatible = "samsung,exynos4210-spi", - .data = (void *)&exynos4_spi_port_config, + .data = &exynos4_spi_port_config, }, { .compatible = "samsung,exynos7-spi", - .data = (void *)&exynos7_spi_port_config, + .data = &exynos7_spi_port_config, }, { .compatible = "samsung,exynos5433-spi", - .data = (void *)&exynos5433_spi_port_config, + .data = &exynos5433_spi_port_config, }, { .compatible = "samsung,exynos850-spi", - .data = (void *)&exynos850_spi_port_config, + .data = &exynos850_spi_port_config, }, { .compatible = "samsung,exynosautov9-spi", - .data = (void *)&exynosautov9_spi_port_config, + .data = &exynosautov9_spi_port_config, }, { .compatible = "tesla,fsd-spi", - .data = (void *)&fsd_spi_port_config, + .data = &fsd_spi_port_config, }, { }, }; From 9d47e411f4d636519a8d26587928d34cf52c0c1f Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:22 +0000 Subject: [PATCH 0186/1038] spi: s3c64xx: remove else after return Else case is not needed after a return, remove it. Reviewed-by: Andi Shyti Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-9-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 3f8fc9d5a508..fc0fe81ac95f 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -408,12 +408,10 @@ static bool s3c64xx_spi_can_dma(struct spi_controller *host, { struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); - if (sdd->rx_dma.ch && sdd->tx_dma.ch) { + if (sdd->rx_dma.ch && sdd->tx_dma.ch) return xfer->len > FIFO_DEPTH(sdd); - } else { - return false; - } + return false; } static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, From 5d7f4f4367079992c7a1bb1654ffea87ddc82be8 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:23 +0000 Subject: [PATCH 0187/1038] spi: s3c64xx: move common code outside if else Move common code outside if else to avoid code duplication. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-10-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index fc0fe81ac95f..8c76894d0eb1 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -292,20 +292,18 @@ static int prepare_dma(struct s3c64xx_spi_dma_data *dma, if (dma->direction == DMA_DEV_TO_MEM) { sdd = container_of((void *)dma, struct s3c64xx_spi_driver_data, rx_dma); - config.direction = dma->direction; config.src_addr = sdd->sfr_start + S3C64XX_SPI_RX_DATA; config.src_addr_width = sdd->cur_bpw / 8; config.src_maxburst = 1; - dmaengine_slave_config(dma->ch, &config); } else { sdd = container_of((void *)dma, struct s3c64xx_spi_driver_data, tx_dma); - config.direction = dma->direction; config.dst_addr = sdd->sfr_start + S3C64XX_SPI_TX_DATA; config.dst_addr_width = sdd->cur_bpw / 8; config.dst_maxburst = 1; - dmaengine_slave_config(dma->ch, &config); } + config.direction = dma->direction; + dmaengine_slave_config(dma->ch, &config); desc = dmaengine_prep_slave_sg(dma->ch, sgt->sgl, sgt->nents, dma->direction, DMA_PREP_INTERRUPT); From e9c49effde70fb4b10d0ad9c94b69fe6314fc608 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:24 +0000 Subject: [PATCH 0188/1038] spi: s3c64xx: check return code of dmaengine_slave_config() Check the return code of dmaengine_slave_config(). Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-11-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 8c76894d0eb1..5da82013e17b 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -303,7 +303,9 @@ static int prepare_dma(struct s3c64xx_spi_dma_data *dma, config.dst_maxburst = 1; } config.direction = dma->direction; - dmaengine_slave_config(dma->ch, &config); + ret = dmaengine_slave_config(dma->ch, &config); + if (ret) + return ret; desc = dmaengine_prep_slave_sg(dma->ch, sgt->sgl, sgt->nents, dma->direction, DMA_PREP_INTERRUPT); From 60dc8d342e933097eb82db5859edcac9077a6db5 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:25 +0000 Subject: [PATCH 0189/1038] spi: s3c64xx: propagate the dma_submit_error() error code DMA submit should just add the dma descriptor to a queue, without firing it. EIO is misleading and hides what happens in DMA. Propagate the dma_submit_error() error code, don't overwrite it. Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-12-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 5da82013e17b..6012c88cb2ec 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -322,7 +322,7 @@ static int prepare_dma(struct s3c64xx_spi_dma_data *dma, ret = dma_submit_error(dma->cookie); if (ret) { dev_err(&sdd->pdev->dev, "DMA submission failed"); - return -EIO; + return ret; } dma_async_issue_pending(dma->ch); From 4c6452050530b741daf108de0c02cd2299f8f5d1 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:26 +0000 Subject: [PATCH 0190/1038] spi: s3c64xx: rename prepare_dma() to s3c64xx_prepare_dma() Don't monopolize the name. Prepend the driver prefix to the function name. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-13-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 6012c88cb2ec..fbd500f26765 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -279,8 +279,8 @@ static void s3c64xx_spi_dmacb(void *data) spin_unlock_irqrestore(&sdd->lock, flags); } -static int prepare_dma(struct s3c64xx_spi_dma_data *dma, - struct sg_table *sgt) +static int s3c64xx_prepare_dma(struct s3c64xx_spi_dma_data *dma, + struct sg_table *sgt) { struct s3c64xx_spi_driver_data *sdd; struct dma_slave_config config; @@ -445,7 +445,7 @@ static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, chcfg |= S3C64XX_SPI_CH_TXCH_ON; if (dma_mode) { modecfg |= S3C64XX_SPI_MODE_TXDMA_ON; - ret = prepare_dma(&sdd->tx_dma, &xfer->tx_sg); + ret = s3c64xx_prepare_dma(&sdd->tx_dma, &xfer->tx_sg); } else { switch (sdd->cur_bpw) { case 32: @@ -477,7 +477,7 @@ static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, writel(((xfer->len * 8 / sdd->cur_bpw) & 0xffff) | S3C64XX_SPI_PACKET_CNT_EN, regs + S3C64XX_SPI_PACKET_CNT); - ret = prepare_dma(&sdd->rx_dma, &xfer->rx_sg); + ret = s3c64xx_prepare_dma(&sdd->rx_dma, &xfer->rx_sg); } } From 1a234accc93191a3a73eb4cc264abb6d79d63430 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:27 +0000 Subject: [PATCH 0191/1038] spi: s3c64xx: return ETIMEDOUT for wait_for_completion_timeout() ETIMEDOUT is more specific than EIO, use it for wait_for_completion_timeout(). Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-14-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index fbd500f26765..470610431ef8 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -527,7 +527,7 @@ static int s3c64xx_wait_for_dma(struct s3c64xx_spi_driver_data *sdd, /* * If the previous xfer was completed within timeout, then - * proceed further else return -EIO. + * proceed further else return -ETIMEDOUT. * DmaTx returns after simply writing data in the FIFO, * w/o waiting for real transmission on the bus to finish. * DmaRx returns only after Dma read data from FIFO which @@ -548,7 +548,7 @@ static int s3c64xx_wait_for_dma(struct s3c64xx_spi_driver_data *sdd, /* If timed out while checking rx/tx status return error */ if (!val) - return -EIO; + return -ETIMEDOUT; return 0; } @@ -578,7 +578,7 @@ static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd, if (use_irq) { val = msecs_to_jiffies(ms); if (!wait_for_completion_timeout(&sdd->xfer_completion, val)) - return -EIO; + return -ETIMEDOUT; } val = msecs_to_loops(ms); From 91a9b8e6b63eeb3634e736a4b65ae536c08155b2 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:28 +0000 Subject: [PATCH 0192/1038] spi: s3c64xx: drop blank line between declarations Drop the blank line and move the logical operation in the body of the function rather than in initialization list. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-15-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 470610431ef8..458ed0cc48e6 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1321,8 +1321,9 @@ static int s3c64xx_spi_suspend(struct device *dev) { struct spi_controller *host = dev_get_drvdata(dev); struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); + int ret; - int ret = spi_controller_suspend(host); + ret = spi_controller_suspend(host); if (ret) return ret; From f186d34071fb2a7db7249b09d5e1796b18b37d7d Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:29 +0000 Subject: [PATCH 0193/1038] spi: s3c64xx: downgrade dev_warn to dev_dbg for optional dt props "samsung,spi-src-clk" and "num-cs" are optional dt properties. Downgrade the message from warning to debug message. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-16-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 458ed0cc48e6..66260f7cd7ad 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1110,14 +1110,14 @@ static struct s3c64xx_spi_info *s3c64xx_spi_parse_dt(struct device *dev) return ERR_PTR(-ENOMEM); if (of_property_read_u32(dev->of_node, "samsung,spi-src-clk", &temp)) { - dev_warn(dev, "spi bus clock parent not specified, using clock at index 0 as parent\n"); + dev_dbg(dev, "spi bus clock parent not specified, using clock at index 0 as parent\n"); sci->src_clk_nr = 0; } else { sci->src_clk_nr = temp; } if (of_property_read_u32(dev->of_node, "num-cs", &temp)) { - dev_warn(dev, "number of chip select lines not specified, assuming 1 chip select line\n"); + dev_dbg(dev, "number of chip select lines not specified, assuming 1 chip select line\n"); sci->num_cs = 1; } else { sci->num_cs = temp; From eb8096c30ad07e6201830816e398b3ad603cc096 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:30 +0000 Subject: [PATCH 0194/1038] spi: s3c64xx: remove duplicated definition S3C64XX_SPI_TRAILCNT brings no benefit in terms of name over S3C64XX_SPI_MAX_TRAILCNT. Remove the duplicated definition. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-17-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 66260f7cd7ad..c1d028d1530e 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -116,8 +116,6 @@ #define S3C64XX_SPI_MAX_TRAILCNT 0x3ff #define S3C64XX_SPI_TRAILCNT_OFF 19 -#define S3C64XX_SPI_TRAILCNT S3C64XX_SPI_MAX_TRAILCNT - #define S3C64XX_SPI_POLLING_SIZE 32 #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) @@ -1093,7 +1091,7 @@ static void s3c64xx_spi_hwinit(struct s3c64xx_spi_driver_data *sdd) val = readl(regs + S3C64XX_SPI_MODE_CFG); val &= ~S3C64XX_SPI_MODE_4BURST; val &= ~(S3C64XX_SPI_MAX_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); - val |= (S3C64XX_SPI_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); + val |= (S3C64XX_SPI_MAX_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); writel(val, regs + S3C64XX_SPI_MODE_CFG); s3c64xx_flush_fifo(sdd); From acd6c7b1d2765fd30b7d7487aff50dc824db314e Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 12:04:31 +0000 Subject: [PATCH 0195/1038] spi: s3c64xx: drop a superfluous bitwise NOT operation val &= ~mask; val |= mask; is equivalent to: val |= mask; Drop the superfluous bitwise NOT operation. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207120431.2766269-18-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index c1d028d1530e..c15ca6a910dc 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1090,7 +1090,6 @@ static void s3c64xx_spi_hwinit(struct s3c64xx_spi_driver_data *sdd) val = readl(regs + S3C64XX_SPI_MODE_CFG); val &= ~S3C64XX_SPI_MODE_4BURST; - val &= ~(S3C64XX_SPI_MAX_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); val |= (S3C64XX_SPI_MAX_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); writel(val, regs + S3C64XX_SPI_MODE_CFG); From ff690e75d64b0ca119adbfc3bd0b444bc1d0a1c5 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 11:15:13 +0000 Subject: [PATCH 0196/1038] spi: dt-bindings: samsung: add google,gs101-spi compatible Add "google,gs101-spi" dedicated compatible for representing SPI of Google GS101 SoC. Reviewed-by: Sam Protsenko Reviewed-by: Krzysztof Kozlowski Reviewed-by: Peter Griffin Acked-by: Andi Shyti Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207111516.2563218-2-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- Documentation/devicetree/bindings/spi/samsung,spi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/spi/samsung,spi.yaml b/Documentation/devicetree/bindings/spi/samsung,spi.yaml index f71099852653..2f0a0835ecfb 100644 --- a/Documentation/devicetree/bindings/spi/samsung,spi.yaml +++ b/Documentation/devicetree/bindings/spi/samsung,spi.yaml @@ -17,6 +17,7 @@ properties: compatible: oneOf: - enum: + - google,gs101-spi - samsung,s3c2443-spi # for S3C2443, S3C2416 and S3C2450 - samsung,s3c6410-spi - samsung,s5pv210-spi # for S5PV210 and S5PC110 From 80d3204a3b1dbef570ed29d4d375e4d6922da82d Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 11:15:14 +0000 Subject: [PATCH 0197/1038] spi: s3c64xx: prepare for a different flavor of iowrite rep There are SoCs (gs101) that allow only 32 bit register accesses. As the requirement is rare enough, for those SoCs we'll open code in the driver some s3c64xx_iowrite{8,16}_32_rep() accessors. Prepare for such addition. Suggested-by: Sam Protsenko Signed-off-by: Tudor Ambarus Reviewed-by: Sam Protsenko Link: https://lore.kernel.org/r/20240207111516.2563218-3-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 7f7eb8f742e4..eb79c6e4f509 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -414,6 +414,26 @@ static bool s3c64xx_spi_can_dma(struct spi_controller *host, } +static void s3c64xx_iowrite_rep(const struct s3c64xx_spi_driver_data *sdd, + struct spi_transfer *xfer) +{ + void __iomem *addr = sdd->regs + S3C64XX_SPI_TX_DATA; + const void *buf = xfer->tx_buf; + unsigned int len = xfer->len; + + switch (sdd->cur_bpw) { + case 32: + iowrite32_rep(addr, buf, len / 4); + break; + case 16: + iowrite16_rep(addr, buf, len / 2); + break; + default: + iowrite8_rep(addr, buf, len); + break; + } +} + static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, struct spi_transfer *xfer, int dma_mode) { @@ -447,20 +467,7 @@ static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, modecfg |= S3C64XX_SPI_MODE_TXDMA_ON; ret = prepare_dma(&sdd->tx_dma, &xfer->tx_sg); } else { - switch (sdd->cur_bpw) { - case 32: - iowrite32_rep(regs + S3C64XX_SPI_TX_DATA, - xfer->tx_buf, xfer->len / 4); - break; - case 16: - iowrite16_rep(regs + S3C64XX_SPI_TX_DATA, - xfer->tx_buf, xfer->len / 2); - break; - default: - iowrite8_rep(regs + S3C64XX_SPI_TX_DATA, - xfer->tx_buf, xfer->len); - break; - } + s3c64xx_iowrite_rep(sdd, xfer); } } From b7bafb9f54fc4609ff84ecd633f918f6f973f471 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 11:15:15 +0000 Subject: [PATCH 0198/1038] spi: s3c64xx: add s3c64xx_iowrite{8,16}_32_rep accessors Allow SoCs that require 32 bits register accesses to write data in chunks of 8 or 16 bits. One SoC that requires 32 bit register accesses is the google gs101. The operation is rare, thus open code it in the driver rather than making it generic (through asm-generic/io.h). Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207111516.2563218-4-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index eb79c6e4f509..014fcc933c90 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -142,6 +142,7 @@ struct s3c64xx_spi_dma_data { * prescaler unit. * @clk_ioclk: True if clock is present on this device * @has_loopback: True if loopback mode can be supported + * @use_32bit_io: True if the SoC allows only 32-bit register accesses. * * The Samsung s3c64xx SPI controller are used on various Samsung SoC's but * differ in some aspects such as the size of the fifo and spi bus clock @@ -158,6 +159,7 @@ struct s3c64xx_spi_port_config { bool clk_from_cmu; bool clk_ioclk; bool has_loopback; + bool use_32bit_io; }; /** @@ -414,6 +416,30 @@ static bool s3c64xx_spi_can_dma(struct spi_controller *host, } +static void s3c64xx_iowrite8_32_rep(volatile void __iomem *addr, + const void *buffer, unsigned int count) +{ + if (count) { + const u8 *buf = buffer; + + do { + __raw_writel(*buf++, addr); + } while (--count); + } +} + +static void s3c64xx_iowrite16_32_rep(volatile void __iomem *addr, + const void *buffer, unsigned int count) +{ + if (count) { + const u16 *buf = buffer; + + do { + __raw_writel(*buf++, addr); + } while (--count); + } +} + static void s3c64xx_iowrite_rep(const struct s3c64xx_spi_driver_data *sdd, struct spi_transfer *xfer) { @@ -426,10 +452,16 @@ static void s3c64xx_iowrite_rep(const struct s3c64xx_spi_driver_data *sdd, iowrite32_rep(addr, buf, len / 4); break; case 16: - iowrite16_rep(addr, buf, len / 2); + if (sdd->port_conf->use_32bit_io) + s3c64xx_iowrite16_32_rep(addr, buf, len / 2); + else + iowrite16_rep(addr, buf, len / 2); break; default: - iowrite8_rep(addr, buf, len); + if (sdd->port_conf->use_32bit_io) + s3c64xx_iowrite8_32_rep(addr, buf, len); + else + iowrite8_rep(addr, buf, len); break; } } From e010c04958604edd7034913c5348f537852449bc Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Wed, 7 Feb 2024 11:15:16 +0000 Subject: [PATCH 0199/1038] spi: s3c64xx: add support for google,gs101-spi Add support for GS101 SPI. GS101 integrates 16 SPI nodes, all with 64 bytes FIFOs. GS101 allows just 32 bit register accesses, otherwise a Serror Interrupt is raised. Do the write reg accesses in 32 bits. Reviewed-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240207111516.2563218-5-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 014fcc933c90..7ab3f3c2e9aa 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -17,7 +17,7 @@ #include -#define MAX_SPI_PORTS 12 +#define MAX_SPI_PORTS 16 #define S3C64XX_SPI_QUIRK_CS_AUTO (1 << 1) #define AUTOSUSPEND_TIMEOUT 2000 @@ -1534,6 +1534,19 @@ static const struct s3c64xx_spi_port_config fsd_spi_port_config = { .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, }; +static const struct s3c64xx_spi_port_config gs101_spi_port_config = { + .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + .rx_lvl_offset = 15, + .tx_st_done = 25, + .clk_div = 4, + .high_speed = true, + .clk_from_cmu = true, + .has_loopback = true, + .use_32bit_io = true, + .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, +}; + static const struct platform_device_id s3c64xx_spi_driver_ids[] = { { .name = "s3c2443-spi", @@ -1546,6 +1559,9 @@ static const struct platform_device_id s3c64xx_spi_driver_ids[] = { }; static const struct of_device_id s3c64xx_spi_dt_match[] = { + { .compatible = "google,gs101-spi", + .data = &gs101_spi_port_config, + }, { .compatible = "samsung,s3c2443-spi", .data = (void *)&s3c2443_spi_port_config, }, From 3e7cfd6ad29a7483ce3addebb4fba87d6409c71c Mon Sep 17 00:00:00 2001 From: Thangaraj Samynathan Date: Wed, 7 Feb 2024 13:36:20 +0530 Subject: [PATCH 0200/1038] spi: mchp-pci1xxxx: Add support for DMA in SPI In PCI1xxxx C0, support for DMA in PCIe endpoint is added to enhance the SPI performance. With this support, the performance is improved from 6Mbps to 17Mbps with 20Mhz clock. - DMA Supports two Channels, 0 and 1 - SPI Instance 0 uses chan 0 and SPI Instance 1 uses chan 1 - DMA can be used only if SPI is mapped to PF0 in the multi function endpoint and the MSI interrupt is supported - MSI interrupt of one of the SPI instance is assigned to the DMA and both channels 0 and 1 share the same irq, the MSI address and MSI Data of the irq is obtained and stored in DMA registers to generate interrupt Signed-off-by: Thangaraj Samynathan Link: https://lore.kernel.org/r/20240207080621.30742-2-thangaraj.s@microchip.com Signed-off-by: Mark Brown --- drivers/spi/spi-pci1xxxx.c | 145 +++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/drivers/spi/spi-pci1xxxx.c b/drivers/spi/spi-pci1xxxx.c index 5b2d3e4e21b7..85a6068b244d 100644 --- a/drivers/spi/spi-pci1xxxx.c +++ b/drivers/spi/spi-pci1xxxx.c @@ -5,7 +5,12 @@ // Kumaravel Thiagarajan +#include +#include +#include #include +#include +#include #include #include #include @@ -32,8 +37,33 @@ #define SPI_MST_CTL_MODE_SEL (BIT(2)) #define SPI_MST_CTL_GO (BIT(0)) +#define SPI_SYSTEM_ADDR_BASE (0x2000) #define SPI_MST1_ADDR_BASE (0x800) +#define DEV_REV_REG (SPI_SYSTEM_ADDR_BASE + 0x00) +#define SPI_SYSLOCK_REG (SPI_SYSTEM_ADDR_BASE + 0xA0) +#define SPI_CONFIG_PERI_ENABLE_REG (SPI_SYSTEM_ADDR_BASE + 0x108) + +#define SPI_PERI_ENBLE_PF_MASK (GENMASK(17, 16)) +#define DEV_REV_MASK (GENMASK(7, 0)) + +#define SPI_SYSLOCK BIT(4) + +/* DMA Related Registers */ +#define SPI_DMA_ADDR_BASE (0x1000) +#define SPI_DMA_GLOBAL_WR_ENGINE_EN (SPI_DMA_ADDR_BASE + 0x0C) +#define SPI_DMA_GLOBAL_RD_ENGINE_EN (SPI_DMA_ADDR_BASE + 0x2C) +#define SPI_DMA_INTR_IMWR_WDONE_LOW (SPI_DMA_ADDR_BASE + 0x60) +#define SPI_DMA_INTR_IMWR_WDONE_HIGH (SPI_DMA_ADDR_BASE + 0x64) +#define SPI_DMA_INTR_IMWR_WABORT_LOW (SPI_DMA_ADDR_BASE + 0x68) +#define SPI_DMA_INTR_IMWR_WABORT_HIGH (SPI_DMA_ADDR_BASE + 0x6C) +#define SPI_DMA_INTR_WR_IMWR_DATA (SPI_DMA_ADDR_BASE + 0x70) +#define SPI_DMA_INTR_IMWR_RDONE_LOW (SPI_DMA_ADDR_BASE + 0xCC) +#define SPI_DMA_INTR_IMWR_RDONE_HIGH (SPI_DMA_ADDR_BASE + 0xD0) +#define SPI_DMA_INTR_IMWR_RABORT_LOW (SPI_DMA_ADDR_BASE + 0xD4) +#define SPI_DMA_INTR_IMWR_RABORT_HIGH (SPI_DMA_ADDR_BASE + 0xD8) +#define SPI_DMA_INTR_RD_IMWR_DATA (SPI_DMA_ADDR_BASE + 0xDC) + /* x refers to SPI Host Controller HW instance id in the below macros - 0 or 1 */ #define SPI_MST_CMD_BUF_OFFSET(x) (((x) * SPI_MST1_ADDR_BASE) + 0x00) @@ -50,6 +80,8 @@ #define SPI_MAX_DATA_LEN 320 #define PCI1XXXX_SPI_TIMEOUT (msecs_to_jiffies(100)) +#define SYSLOCK_RETRY_CNT (1000) +#define SPI_DMA_ENGINE_EN (0x1) #define SPI_INTR BIT(8) #define SPI_FORCE_CE BIT(4) @@ -76,7 +108,10 @@ struct pci1xxxx_spi_internal { struct pci1xxxx_spi { struct pci_dev *dev; u8 total_hw_instances; + u8 dev_rev; void __iomem *reg_base; + void __iomem *dma_offset_bar; + bool can_dma; struct pci1xxxx_spi_internal *spi_int[] __counted_by(total_hw_instances); }; @@ -106,6 +141,112 @@ static const struct pci_device_id pci1xxxx_spi_pci_id_table[] = { MODULE_DEVICE_TABLE(pci, pci1xxxx_spi_pci_id_table); +static int pci1xxxx_set_sys_lock(struct pci1xxxx_spi *par) +{ + writel(SPI_SYSLOCK, par->reg_base + SPI_SYSLOCK_REG); + return readl(par->reg_base + SPI_SYSLOCK_REG); +} + +static int pci1xxxx_acquire_sys_lock(struct pci1xxxx_spi *par) +{ + u32 regval; + + return readx_poll_timeout(pci1xxxx_set_sys_lock, par, regval, + (regval & SPI_SYSLOCK), 100, + SYSLOCK_RETRY_CNT * 100); +} + +static void pci1xxxx_release_sys_lock(struct pci1xxxx_spi *par) +{ + writel(0x0, par->reg_base + SPI_SYSLOCK_REG); +} + +static int pci1xxxx_check_spi_can_dma(struct pci1xxxx_spi *spi_bus, int irq) +{ + struct pci_dev *pdev = spi_bus->dev; + u32 pf_num; + u32 regval; + int ret; + + /* + * DEV REV Registers is a system register, HW Syslock bit + * should be acquired before accessing the register + */ + ret = pci1xxxx_acquire_sys_lock(spi_bus); + if (ret) { + dev_err(&pdev->dev, "Error failed to acquire syslock\n"); + return ret; + } + + regval = readl(spi_bus->reg_base + DEV_REV_REG); + spi_bus->dev_rev = regval & DEV_REV_MASK; + if (spi_bus->dev_rev >= 0xC0) { + regval = readl(spi_bus->reg_base + + SPI_CONFIG_PERI_ENABLE_REG); + pf_num = regval & SPI_PERI_ENBLE_PF_MASK; + } + + pci1xxxx_release_sys_lock(spi_bus); + + /* + * DMA is supported only from C0 and SPI can use DMA only if + * it is mapped to PF0 + */ + if (spi_bus->dev_rev < 0xC0 || pf_num) + return -EOPNOTSUPP; + + /* + * DMA Supported only with MSI Interrupts + * One of the SPI instance's MSI vector address and data + * is used for DMA Interrupt + */ + if (!irq_get_msi_desc(irq)) { + dev_warn(&pdev->dev, "Error MSI Interrupt not supported, will operate in PIO mode\n"); + return -EOPNOTSUPP; + } + + spi_bus->dma_offset_bar = pcim_iomap(pdev, 2, pci_resource_len(pdev, 2)); + if (!spi_bus->dma_offset_bar) { + dev_warn(&pdev->dev, "Error failed to map dma bar, will operate in PIO mode\n"); + return -EOPNOTSUPP; + } + + if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { + dev_warn(&pdev->dev, "Error failed to set DMA mask, will operate in PIO mode\n"); + pcim_iounmap(pdev, spi_bus->dma_offset_bar); + spi_bus->dma_offset_bar = NULL; + return -EOPNOTSUPP; + } + + return 0; +} + +static int pci1xxxx_spi_dma_init(struct pci1xxxx_spi *spi_bus, int irq) +{ + struct msi_msg msi; + int ret; + + ret = pci1xxxx_check_spi_can_dma(spi_bus, irq); + if (ret) + return ret; + + get_cached_msi_msg(irq, &msi); + writel(SPI_DMA_ENGINE_EN, spi_bus->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); + writel(SPI_DMA_ENGINE_EN, spi_bus->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WDONE_HIGH); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WABORT_HIGH); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RDONE_HIGH); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RABORT_HIGH); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WDONE_LOW); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WABORT_LOW); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RDONE_LOW); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RABORT_LOW); + writel(msi.data, spi_bus->dma_offset_bar + SPI_DMA_INTR_WR_IMWR_DATA); + writel(msi.data, spi_bus->dma_offset_bar + SPI_DMA_INTR_RD_IMWR_DATA); + spi_bus->can_dma = true; + return 0; +} + static void pci1xxxx_spi_set_cs(struct spi_device *spi, bool enable) { struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(spi->controller); @@ -324,6 +465,10 @@ static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id * goto error; } + ret = pci1xxxx_spi_dma_init(spi_bus, spi_sub_ptr->irq); + if (ret && ret != -EOPNOTSUPP) + return ret; + /* This register is only applicable for 1st instance */ regval = readl(spi_bus->reg_base + SPI_PCI_CTRL_REG_OFFSET(0)); if (!only_sec_inst) From 9538edeb72c989a4b90964ae4bba107eaf21a791 Mon Sep 17 00:00:00 2001 From: Thangaraj Samynathan Date: Wed, 7 Feb 2024 13:36:21 +0530 Subject: [PATCH 0201/1038] spi: mchp-pci1xxxx: DMA support for copying data to and from SPI Buf pci1xxxx_spi_transfer_with_dma adds DMA support to copy the data between host cpu buffer and SPI IO Buffer. On DMA Completion interrupt, the next SPI transaction is initiated in isr. Helper functions pci1xxxx_spi_setup, pci1xxxx_spi_setup_dma_from_io, pci1xxxx_spi_setup_dma_to_io and pci1xxxx_start_spi_xfer are added for setting up spi, setting up dma operations, and to start spi transfer respectively. In the existing implementation, codes are replaced with helper functions wherever applicable. Signed-off-by: Thangaraj Samynathan Link: https://lore.kernel.org/r/20240207080621.30742-3-thangaraj.s@microchip.com Signed-off-by: Mark Brown --- drivers/spi/spi-pci1xxxx.c | 369 ++++++++++++++++++++++++++++++++++--- 1 file changed, 340 insertions(+), 29 deletions(-) diff --git a/drivers/spi/spi-pci1xxxx.c b/drivers/spi/spi-pci1xxxx.c index 85a6068b244d..a99db6163aec 100644 --- a/drivers/spi/spi-pci1xxxx.c +++ b/drivers/spi/spi-pci1xxxx.c @@ -5,6 +5,7 @@ // Kumaravel Thiagarajan +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include @@ -37,6 +39,7 @@ #define SPI_MST_CTL_MODE_SEL (BIT(2)) #define SPI_MST_CTL_GO (BIT(0)) +#define SPI_PERI_ADDR_BASE (0x160000) #define SPI_SYSTEM_ADDR_BASE (0x2000) #define SPI_MST1_ADDR_BASE (0x800) @@ -48,22 +51,56 @@ #define DEV_REV_MASK (GENMASK(7, 0)) #define SPI_SYSLOCK BIT(4) +#define SPI0 (0) +#define SPI1 (1) /* DMA Related Registers */ #define SPI_DMA_ADDR_BASE (0x1000) #define SPI_DMA_GLOBAL_WR_ENGINE_EN (SPI_DMA_ADDR_BASE + 0x0C) +#define SPI_DMA_WR_DOORBELL_REG (SPI_DMA_ADDR_BASE + 0x10) #define SPI_DMA_GLOBAL_RD_ENGINE_EN (SPI_DMA_ADDR_BASE + 0x2C) +#define SPI_DMA_RD_DOORBELL_REG (SPI_DMA_ADDR_BASE + 0x30) +#define SPI_DMA_INTR_WR_STS (SPI_DMA_ADDR_BASE + 0x4C) +#define SPI_DMA_WR_INT_MASK (SPI_DMA_ADDR_BASE + 0x54) +#define SPI_DMA_INTR_WR_CLR (SPI_DMA_ADDR_BASE + 0x58) +#define SPI_DMA_ERR_WR_STS (SPI_DMA_ADDR_BASE + 0x5C) #define SPI_DMA_INTR_IMWR_WDONE_LOW (SPI_DMA_ADDR_BASE + 0x60) #define SPI_DMA_INTR_IMWR_WDONE_HIGH (SPI_DMA_ADDR_BASE + 0x64) #define SPI_DMA_INTR_IMWR_WABORT_LOW (SPI_DMA_ADDR_BASE + 0x68) #define SPI_DMA_INTR_IMWR_WABORT_HIGH (SPI_DMA_ADDR_BASE + 0x6C) #define SPI_DMA_INTR_WR_IMWR_DATA (SPI_DMA_ADDR_BASE + 0x70) +#define SPI_DMA_INTR_RD_STS (SPI_DMA_ADDR_BASE + 0xA0) +#define SPI_DMA_RD_INT_MASK (SPI_DMA_ADDR_BASE + 0xA8) +#define SPI_DMA_INTR_RD_CLR (SPI_DMA_ADDR_BASE + 0xAC) +#define SPI_DMA_ERR_RD_STS (SPI_DMA_ADDR_BASE + 0xB8) #define SPI_DMA_INTR_IMWR_RDONE_LOW (SPI_DMA_ADDR_BASE + 0xCC) #define SPI_DMA_INTR_IMWR_RDONE_HIGH (SPI_DMA_ADDR_BASE + 0xD0) #define SPI_DMA_INTR_IMWR_RABORT_LOW (SPI_DMA_ADDR_BASE + 0xD4) #define SPI_DMA_INTR_IMWR_RABORT_HIGH (SPI_DMA_ADDR_BASE + 0xD8) #define SPI_DMA_INTR_RD_IMWR_DATA (SPI_DMA_ADDR_BASE + 0xDC) +#define SPI_DMA_CH0_WR_BASE (SPI_DMA_ADDR_BASE + 0x200) +#define SPI_DMA_CH0_RD_BASE (SPI_DMA_ADDR_BASE + 0x300) +#define SPI_DMA_CH1_WR_BASE (SPI_DMA_ADDR_BASE + 0x400) +#define SPI_DMA_CH1_RD_BASE (SPI_DMA_ADDR_BASE + 0x500) + +#define SPI_DMA_CH_CTL1_OFFSET (0x00) +#define SPI_DMA_CH_XFER_LEN_OFFSET (0x08) +#define SPI_DMA_CH_SAR_LO_OFFSET (0x0C) +#define SPI_DMA_CH_SAR_HI_OFFSET (0x10) +#define SPI_DMA_CH_DAR_LO_OFFSET (0x14) +#define SPI_DMA_CH_DAR_HI_OFFSET (0x18) + +#define SPI_DMA_CH0_DONE_INT BIT(0) +#define SPI_DMA_CH1_DONE_INT BIT(1) +#define SPI_DMA_CH0_ABORT_INT BIT(16) +#define SPI_DMA_CH1_ABORT_INT BIT(17) +#define SPI_DMA_DONE_INT_MASK (SPI_DMA_CH0_DONE_INT | SPI_DMA_CH1_DONE_INT) +#define SPI_DMA_ABORT_INT_MASK (SPI_DMA_CH0_ABORT_INT | SPI_DMA_CH1_ABORT_INT) +#define DMA_CH_CONTROL_LIE BIT(3) +#define DMA_CH_CONTROL_RIE BIT(4) +#define DMA_INTR_EN (DMA_CH_CONTROL_RIE | DMA_CH_CONTROL_LIE) + /* x refers to SPI Host Controller HW instance id in the below macros - 0 or 1 */ #define SPI_MST_CMD_BUF_OFFSET(x) (((x) * SPI_MST1_ADDR_BASE) + 0x00) @@ -82,6 +119,7 @@ #define PCI1XXXX_SPI_TIMEOUT (msecs_to_jiffies(100)) #define SYSLOCK_RETRY_CNT (1000) #define SPI_DMA_ENGINE_EN (0x1) +#define SPI_DMA_ENGINE_DIS (0x0) #define SPI_INTR BIT(8) #define SPI_FORCE_CE BIT(4) @@ -94,11 +132,21 @@ struct pci1xxxx_spi_internal { u8 hw_inst; - bool spi_xfer_in_progress; + u8 clkdiv; int irq; + int mode; + bool spi_xfer_in_progress; + void *rx_buf; + bool dma_aborted_rd; + u32 bytes_recvd; + u32 tx_sgl_len; + u32 rx_sgl_len; + struct scatterlist *tx_sgl, *rx_sgl; + bool dma_aborted_wr; struct completion spi_xfer_done; struct spi_controller *spi_host; struct pci1xxxx_spi *parent; + struct spi_transfer *xfer; struct { unsigned int dev_sel : 3; unsigned int msi_vector_sel : 1; @@ -111,6 +159,8 @@ struct pci1xxxx_spi { u8 dev_rev; void __iomem *reg_base; void __iomem *dma_offset_bar; + /* lock to safely access the DMA registers in isr */ + spinlock_t dma_reg_lock; bool can_dma; struct pci1xxxx_spi_internal *spi_int[] __counted_by(total_hw_instances); }; @@ -230,6 +280,7 @@ static int pci1xxxx_spi_dma_init(struct pci1xxxx_spi *spi_bus, int irq) if (ret) return ret; + spin_lock_init(&spi_bus->dma_reg_lock); get_cached_msi_msg(irq, &msi); writel(SPI_DMA_ENGINE_EN, spi_bus->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); writel(SPI_DMA_ENGINE_EN, spi_bus->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); @@ -243,6 +294,7 @@ static int pci1xxxx_spi_dma_init(struct pci1xxxx_spi *spi_bus, int irq) writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RABORT_LOW); writel(msi.data, spi_bus->dma_offset_bar + SPI_DMA_INTR_WR_IMWR_DATA); writel(msi.data, spi_bus->dma_offset_bar + SPI_DMA_INTR_RD_IMWR_DATA); + dma_set_max_seg_size(&spi_bus->dev->dev, PCI1XXXX_SPI_BUFFER_SIZE); spi_bus->can_dma = true; return 0; } @@ -287,12 +339,79 @@ static u8 pci1xxxx_get_clock_div(u32 hz) return val; } -static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, - struct spi_device *spi, struct spi_transfer *xfer) +static void pci1xxxx_spi_setup_dma_to_io(struct pci1xxxx_spi_internal *p, + dma_addr_t dma_addr, u32 len) +{ + void __iomem *base; + + if (!p->hw_inst) + base = p->parent->dma_offset_bar + SPI_DMA_CH0_RD_BASE; + else + base = p->parent->dma_offset_bar + SPI_DMA_CH1_RD_BASE; + + writel(DMA_INTR_EN, base + SPI_DMA_CH_CTL1_OFFSET); + writel(len, base + SPI_DMA_CH_XFER_LEN_OFFSET); + writel(lower_32_bits(dma_addr), base + SPI_DMA_CH_SAR_LO_OFFSET); + writel(upper_32_bits(dma_addr), base + SPI_DMA_CH_SAR_HI_OFFSET); + /* Updated SPI Command Registers */ + writel(lower_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_CMD_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_DAR_LO_OFFSET); + writel(upper_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_CMD_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_DAR_HI_OFFSET); +} + +static void pci1xxxx_spi_setup_dma_from_io(struct pci1xxxx_spi_internal *p, + dma_addr_t dma_addr, u32 len) +{ + void *base; + + if (!p->hw_inst) + base = p->parent->dma_offset_bar + SPI_DMA_CH0_WR_BASE; + else + base = p->parent->dma_offset_bar + SPI_DMA_CH1_WR_BASE; + + writel(DMA_INTR_EN, base + SPI_DMA_CH_CTL1_OFFSET); + writel(len, base + SPI_DMA_CH_XFER_LEN_OFFSET); + writel(lower_32_bits(dma_addr), base + SPI_DMA_CH_DAR_LO_OFFSET); + writel(upper_32_bits(dma_addr), base + SPI_DMA_CH_DAR_HI_OFFSET); + writel(lower_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_RSP_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_SAR_LO_OFFSET); + writel(upper_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_RSP_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_SAR_HI_OFFSET); +} + +static void pci1xxxx_spi_setup(struct pci1xxxx_spi *par, u8 hw_inst, u32 mode, + u8 clkdiv, u32 len) +{ + u32 regval; + + regval = readl(par->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); + regval &= ~(SPI_MST_CTL_MODE_SEL | SPI_MST_CTL_CMD_LEN_MASK | + SPI_MST_CTL_SPEED_MASK); + + if (mode == SPI_MODE_3) + regval |= SPI_MST_CTL_MODE_SEL; + + regval |= FIELD_PREP(SPI_MST_CTL_CMD_LEN_MASK, len); + regval |= FIELD_PREP(SPI_MST_CTL_SPEED_MASK, clkdiv); + writel(regval, par->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); +} + +static void pci1xxxx_start_spi_xfer(struct pci1xxxx_spi_internal *p, u8 hw_inst) +{ + u32 regval; + + regval = readl(p->parent->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); + regval |= SPI_MST_CTL_GO; + writel(regval, p->parent->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); +} + +static int pci1xxxx_spi_transfer_with_io(struct spi_controller *spi_ctlr, + struct spi_device *spi, struct spi_transfer *xfer) { struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(spi_ctlr); - int mode, len, loop_iter, transfer_len; struct pci1xxxx_spi *par = p->parent; + int len, loop_iter, transfer_len; unsigned long bytes_transfered; unsigned long bytes_recvd; unsigned long loop_count; @@ -302,7 +421,7 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, u8 clkdiv; p->spi_xfer_in_progress = true; - mode = spi->mode; + p->bytes_recvd = 0; clkdiv = pci1xxxx_get_clock_div(xfer->speed_hz); tx_buf = xfer->tx_buf; rx_buf = xfer->rx_buf; @@ -327,26 +446,8 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, memcpy_toio(par->reg_base + SPI_MST_CMD_BUF_OFFSET(p->hw_inst), &tx_buf[bytes_transfered], len); bytes_transfered += len; - regval = readl(par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); - regval &= ~(SPI_MST_CTL_MODE_SEL | SPI_MST_CTL_CMD_LEN_MASK | - SPI_MST_CTL_SPEED_MASK); - - if (mode == SPI_MODE_3) - regval |= SPI_MST_CTL_MODE_SEL; - else - regval &= ~SPI_MST_CTL_MODE_SEL; - - regval |= (clkdiv << 5); - regval &= ~SPI_MST_CTL_CMD_LEN_MASK; - regval |= (len << 8); - writel(regval, par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); - regval = readl(par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); - regval |= SPI_MST_CTL_GO; - writel(regval, par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); + pci1xxxx_spi_setup(par, p->hw_inst, spi->mode, clkdiv, len); + pci1xxxx_start_spi_xfer(p, p->hw_inst); /* Wait for DMA_TERM interrupt */ result = wait_for_completion_timeout(&p->spi_xfer_done, @@ -366,7 +467,113 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, return 0; } -static irqreturn_t pci1xxxx_spi_isr(int irq, void *dev) +static int pci1xxxx_spi_transfer_with_dma(struct spi_controller *spi_ctlr, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(spi_ctlr); + struct pci1xxxx_spi *par = p->parent; + dma_addr_t rx_dma_addr = 0; + dma_addr_t tx_dma_addr = 0; + int ret = 0; + u32 regval; + + p->spi_xfer_in_progress = true; + p->tx_sgl = xfer->tx_sg.sgl; + p->rx_sgl = xfer->rx_sg.sgl; + p->rx_buf = xfer->rx_buf; + regval = readl(par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + writel(regval, par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + + if (!xfer->tx_buf || !p->tx_sgl) { + ret = -EINVAL; + goto error; + } + p->xfer = xfer; + p->mode = spi->mode; + p->clkdiv = pci1xxxx_get_clock_div(xfer->speed_hz); + p->bytes_recvd = 0; + p->rx_buf = xfer->rx_buf; + regval = readl(par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + writel(regval, par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + + tx_dma_addr = sg_dma_address(p->tx_sgl); + rx_dma_addr = sg_dma_address(p->rx_sgl); + p->tx_sgl_len = sg_dma_len(p->tx_sgl); + p->rx_sgl_len = sg_dma_len(p->rx_sgl); + pci1xxxx_spi_setup(par, p->hw_inst, p->mode, p->clkdiv, p->tx_sgl_len); + pci1xxxx_spi_setup_dma_to_io(p, (tx_dma_addr), p->tx_sgl_len); + if (rx_dma_addr) + pci1xxxx_spi_setup_dma_from_io(p, rx_dma_addr, p->rx_sgl_len); + writel(p->hw_inst, par->dma_offset_bar + SPI_DMA_RD_DOORBELL_REG); + + reinit_completion(&p->spi_xfer_done); + /* Wait for DMA_TERM interrupt */ + ret = wait_for_completion_timeout(&p->spi_xfer_done, PCI1XXXX_SPI_TIMEOUT); + if (!ret) { + ret = -ETIMEDOUT; + if (p->dma_aborted_rd) { + writel(SPI_DMA_ENGINE_DIS, + par->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); + /* + * DMA ENGINE reset takes time if any TLP + * completeion in progress, should wait + * till DMA Engine reset is completed. + */ + ret = readl_poll_timeout(par->dma_offset_bar + + SPI_DMA_GLOBAL_RD_ENGINE_EN, regval, + (regval == 0x0), 0, USEC_PER_MSEC); + if (ret) { + ret = -ECANCELED; + goto error; + } + writel(SPI_DMA_ENGINE_EN, + par->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); + p->dma_aborted_rd = false; + ret = -ECANCELED; + } + if (p->dma_aborted_wr) { + writel(SPI_DMA_ENGINE_DIS, + par->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); + + /* + * DMA ENGINE reset takes time if any TLP + * completeion in progress, should wait + * till DMA Engine reset is completed. + */ + ret = readl_poll_timeout(par->dma_offset_bar + + SPI_DMA_GLOBAL_WR_ENGINE_EN, regval, + (regval == 0x0), 0, USEC_PER_MSEC); + if (ret) { + ret = -ECANCELED; + goto error; + } + + writel(SPI_DMA_ENGINE_EN, + par->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); + p->dma_aborted_wr = false; + ret = -ECANCELED; + } + goto error; + } + ret = 0; + +error: + p->spi_xfer_in_progress = false; + + return ret; +} + +static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, + struct spi_device *spi, struct spi_transfer *xfer) +{ + if (spi_ctlr->can_dma(spi_ctlr, spi, xfer) && spi_ctlr->cur_msg_mapped) + return pci1xxxx_spi_transfer_with_dma(spi_ctlr, spi, xfer); + else + return pci1xxxx_spi_transfer_with_io(spi_ctlr, spi, xfer); +} + +static irqreturn_t pci1xxxx_spi_isr_io(int irq, void *dev) { struct pci1xxxx_spi_internal *p = dev; irqreturn_t spi_int_fired = IRQ_NONE; @@ -376,15 +583,117 @@ static irqreturn_t pci1xxxx_spi_isr(int irq, void *dev) regval = readl(p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); if (regval & SPI_INTR) { /* Clear xfer_done */ - complete(&p->spi_xfer_done); + if (p->parent->can_dma && p->rx_buf) + writel(p->hw_inst, p->parent->dma_offset_bar + + SPI_DMA_WR_DOORBELL_REG); + else + complete(&p->parent->spi_int[p->hw_inst]->spi_xfer_done); spi_int_fired = IRQ_HANDLED; } - writel(regval, p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); - return spi_int_fired; } +static void pci1xxxx_spi_setup_next_dma_transfer(struct pci1xxxx_spi_internal *p) +{ + dma_addr_t tx_dma_addr = 0; + dma_addr_t rx_dma_addr = 0; + u32 prev_len; + + p->tx_sgl = sg_next(p->tx_sgl); + if (p->rx_sgl) + p->rx_sgl = sg_next(p->rx_sgl); + if (!p->tx_sgl) { + /* Clear xfer_done */ + complete(&p->spi_xfer_done); + } else { + tx_dma_addr = sg_dma_address(p->tx_sgl); + prev_len = p->tx_sgl_len; + p->tx_sgl_len = sg_dma_len(p->tx_sgl); + if (prev_len != p->tx_sgl_len) + pci1xxxx_spi_setup(p->parent, + p->hw_inst, p->mode, p->clkdiv, p->tx_sgl_len); + pci1xxxx_spi_setup_dma_to_io(p, tx_dma_addr, p->tx_sgl_len); + if (p->rx_sgl) { + rx_dma_addr = sg_dma_address(p->rx_sgl); + p->rx_sgl_len = sg_dma_len(p->rx_sgl); + pci1xxxx_spi_setup_dma_from_io(p, rx_dma_addr, p->rx_sgl_len); + } + writel(p->hw_inst, p->parent->dma_offset_bar + SPI_DMA_RD_DOORBELL_REG); + } +} + +static irqreturn_t pci1xxxx_spi_isr_dma(int irq, void *dev) +{ + struct pci1xxxx_spi_internal *p = dev; + irqreturn_t spi_int_fired = IRQ_NONE; + unsigned long flags; + u32 regval; + + spin_lock_irqsave(&p->parent->dma_reg_lock, flags); + /* Clear the DMA RD INT and start spi xfer*/ + regval = readl(p->parent->dma_offset_bar + SPI_DMA_INTR_RD_STS); + if (regval & SPI_DMA_DONE_INT_MASK) { + if (regval & SPI_DMA_CH0_DONE_INT) + pci1xxxx_start_spi_xfer(p, SPI0); + if (regval & SPI_DMA_CH1_DONE_INT) + pci1xxxx_start_spi_xfer(p, SPI1); + spi_int_fired = IRQ_HANDLED; + } + if (regval & SPI_DMA_ABORT_INT_MASK) { + p->dma_aborted_rd = true; + spi_int_fired = IRQ_HANDLED; + } + writel(regval, p->parent->dma_offset_bar + SPI_DMA_INTR_RD_CLR); + + /* Clear the DMA WR INT */ + regval = readl(p->parent->dma_offset_bar + SPI_DMA_INTR_WR_STS); + if (regval & SPI_DMA_DONE_INT_MASK) { + if (regval & SPI_DMA_CH0_DONE_INT) + pci1xxxx_spi_setup_next_dma_transfer(p->parent->spi_int[SPI0]); + + if (regval & SPI_DMA_CH1_DONE_INT) + pci1xxxx_spi_setup_next_dma_transfer(p->parent->spi_int[SPI1]); + + spi_int_fired = IRQ_HANDLED; + } + if (regval & SPI_DMA_ABORT_INT_MASK) { + p->dma_aborted_wr = true; + spi_int_fired = IRQ_HANDLED; + } + writel(regval, p->parent->dma_offset_bar + SPI_DMA_INTR_WR_CLR); + spin_unlock_irqrestore(&p->parent->dma_reg_lock, flags); + + /* Clear the SPI GO_BIT Interrupt */ + regval = readl(p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + if (regval & SPI_INTR) { + writel(p->hw_inst, p->parent->dma_offset_bar + SPI_DMA_WR_DOORBELL_REG); + spi_int_fired = IRQ_HANDLED; + } + writel(regval, p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + return spi_int_fired; +} + +static irqreturn_t pci1xxxx_spi_isr(int irq, void *dev) +{ + struct pci1xxxx_spi_internal *p = dev; + + if (p->spi_host->can_dma(p->spi_host, NULL, p->xfer)) + return pci1xxxx_spi_isr_dma(irq, dev); + else + return pci1xxxx_spi_isr_io(irq, dev); +} + +static bool pci1xxxx_spi_can_dma(struct spi_controller *host, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(host); + struct pci1xxxx_spi *par = p->parent; + + return par->can_dma; +} + static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { u8 hw_inst_cnt, iter, start, only_sec_inst; @@ -505,7 +814,9 @@ static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id * spi_host->num_chipselect = SPI_CHIP_SEL_COUNT; spi_host->mode_bits = SPI_MODE_0 | SPI_MODE_3 | SPI_RX_DUAL | SPI_TX_DUAL | SPI_LOOP; + spi_host->can_dma = pci1xxxx_spi_can_dma; spi_host->transfer_one = pci1xxxx_spi_transfer_one; + spi_host->set_cs = pci1xxxx_spi_set_cs; spi_host->bits_per_word_mask = SPI_BPW_MASK(8); spi_host->max_speed_hz = PCI1XXXX_SPI_MAX_CLOCK_HZ; From d1ff85fdf0b8f63a6e042ae7559c630f9b1c50e2 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:52 +0100 Subject: [PATCH 0202/1038] spi: pl022: Use typedef for dma_filter_fn Use existing typedef for dma_filter_fn to avoid duplicating type definition. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202154.630336-1-krzysztof.kozlowski@linaro.org Signed-off-by: Mark Brown --- include/linux/amba/pl022.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/amba/pl022.h b/include/linux/amba/pl022.h index 9bf58aac0df2..e08488df6d28 100644 --- a/include/linux/amba/pl022.h +++ b/include/linux/amba/pl022.h @@ -16,6 +16,7 @@ #ifndef _SSP_PL022_H #define _SSP_PL022_H +#include #include /** @@ -235,7 +236,7 @@ struct dma_chan; struct pl022_ssp_controller { u16 bus_id; u8 enable_dma:1; - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); + dma_filter_fn dma_filter; void *dma_rx_param; void *dma_tx_param; int autosuspend_delay; From c42d9bead493854507e1a180942ebe33c9180598 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:53 +0100 Subject: [PATCH 0203/1038] spi: pl022: Add missing dma_filter field kerneldoc Add kerneldoc for dma_filter field in struct pl022_ssp_controller. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202154.630336-2-krzysztof.kozlowski@linaro.org Signed-off-by: Mark Brown --- include/linux/amba/pl022.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/amba/pl022.h b/include/linux/amba/pl022.h index e08488df6d28..d7b07d0311e1 100644 --- a/include/linux/amba/pl022.h +++ b/include/linux/amba/pl022.h @@ -225,6 +225,7 @@ struct dma_chan; * struct pl022_ssp_master - device.platform_data for SPI controller devices. * @bus_id: identifier for this bus * @enable_dma: if true enables DMA driven transfers. + * @dma_filter: callback filter for dma_request_channel. * @dma_rx_param: parameter to locate an RX DMA channel. * @dma_tx_param: parameter to locate a TX DMA channel. * @autosuspend_delay: delay in ms following transfer completion before the From 3d4dd10b376e1b8b6d0409f7e7b752f9baa51c24 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:54 +0100 Subject: [PATCH 0204/1038] spi: pxa2xx: Use typedef for dma_filter_fn Use existing typedef for dma_filter_fn to avoid duplicating type definition. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202154.630336-3-krzysztof.kozlowski@linaro.org Signed-off-by: Mark Brown --- include/linux/spi/pxa2xx_spi.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/spi/pxa2xx_spi.h b/include/linux/spi/pxa2xx_spi.h index 0916cb9bcb0a..ca2cd4e30ead 100644 --- a/include/linux/spi/pxa2xx_spi.h +++ b/include/linux/spi/pxa2xx_spi.h @@ -5,6 +5,7 @@ #ifndef __LINUX_SPI_PXA2XX_SPI_H #define __LINUX_SPI_PXA2XX_SPI_H +#include #include #include @@ -22,7 +23,7 @@ struct pxa2xx_spi_controller { bool is_target; /* DMA engine specific config */ - bool (*dma_filter)(struct dma_chan *chan, void *param); + dma_filter_fn dma_filter; void *tx_param; void *rx_param; From f7131935238d00745638b826f8c31efc8d361435 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Tue, 6 Feb 2024 22:13:42 +0100 Subject: [PATCH 0205/1038] ata: ahci: move marking of external port earlier Move the marking of an external port earlier in the call chain. This is needed for further cleanups. No functional change intended. Tested-by: Damien Le Moal Tested-by: Jian-Hong Pan Reviewed-by: Damien Le Moal Reviewed-by: Mario Limonciello Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 14 ++++++++++++++ drivers/ata/libahci.c | 7 ------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index da2e74fce2d9..aa58ce615e79 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1642,6 +1642,18 @@ static int ahci_init_msi(struct pci_dev *pdev, unsigned int n_ports, return pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX); } +static void ahci_mark_external_port(struct ata_port *ap) +{ + struct ahci_host_priv *hpriv = ap->host->private_data; + void __iomem *port_mmio = ahci_port_base(ap); + u32 tmp; + + /* mark esata ports */ + tmp = readl(port_mmio + PORT_CMD); + if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS)) + ap->pflags |= ATA_PFLAG_EXTERNAL; +} + static void ahci_update_initial_lpm_policy(struct ata_port *ap, struct ahci_host_priv *hpriv) { @@ -1934,6 +1946,8 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (ap->flags & ATA_FLAG_EM) ap->em_message_type = hpriv->em_msg_type; + ahci_mark_external_port(ap); + ahci_update_initial_lpm_policy(ap, hpriv); /* disabled/not-implemented port */ diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index 1a63200ea437..fca376f03c9e 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -1280,10 +1280,8 @@ static void ahci_port_init(struct device *dev, struct ata_port *ap, int port_no, void __iomem *mmio, void __iomem *port_mmio) { - struct ahci_host_priv *hpriv = ap->host->private_data; const char *emsg = NULL; int rc; - u32 tmp; /* make sure port is not active */ rc = ahci_deinit_port(ap, &emsg); @@ -1291,11 +1289,6 @@ static void ahci_port_init(struct device *dev, struct ata_port *ap, dev_warn(dev, "%s (%d)\n", emsg, rc); ahci_port_clear_pending_irq(ap); - - /* mark esata ports */ - tmp = readl(port_mmio + PORT_CMD); - if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS)) - ap->pflags |= ATA_PFLAG_EXTERNAL; } void ahci_init_controller(struct ata_host *host) From 45b96d65ec68f625ad26ee16d2f556e29f715005 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Tue, 6 Feb 2024 22:13:43 +0100 Subject: [PATCH 0206/1038] ata: ahci: a hotplug capable port is an external port A hotplug capable port is an external port, so mark it as such. We even say this ourselves in libata-scsi.c: /* set scsi removable (RMB) bit per ata bit, or if the * AHCI port says it's external (Hotplug-capable, eSATA). */ This also matches the terminology used in AHCI 1.3.1 (the keyword to search for is "externally accessible"). Tested-by: Damien Le Moal Tested-by: Jian-Hong Pan Reviewed-by: Damien Le Moal Reviewed-by: Mario Limonciello Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index aa58ce615e79..4d3ec6d15ad1 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1648,9 +1648,10 @@ static void ahci_mark_external_port(struct ata_port *ap) void __iomem *port_mmio = ahci_port_base(ap); u32 tmp; - /* mark esata ports */ + /* mark external ports (hotplug-capable, eSATA) */ tmp = readl(port_mmio + PORT_CMD); - if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS)) + if (((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS)) || + (tmp & PORT_CMD_HPCP)) ap->pflags |= ATA_PFLAG_EXTERNAL; } From 04d5fb7fbc936f56034e788bed3c84460bf9b844 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Tue, 6 Feb 2024 22:13:44 +0100 Subject: [PATCH 0207/1038] ata: ahci: drop hpriv param from ahci_update_initial_lpm_policy() There is no need for ahci_update_initial_lpm_policy() to take hpriv as a parameter, it can easily be derived from the ata_port. Tested-by: Damien Le Moal Tested-by: Jian-Hong Pan Acked-by: Jian-Hong Pan Reviewed-by: Damien Le Moal Reviewed-by: Mario Limonciello Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 4d3ec6d15ad1..346a0f9ef246 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1655,9 +1655,9 @@ static void ahci_mark_external_port(struct ata_port *ap) ap->pflags |= ATA_PFLAG_EXTERNAL; } -static void ahci_update_initial_lpm_policy(struct ata_port *ap, - struct ahci_host_priv *hpriv) +static void ahci_update_initial_lpm_policy(struct ata_port *ap) { + struct ahci_host_priv *hpriv = ap->host->private_data; int policy = CONFIG_SATA_MOBILE_LPM_POLICY; @@ -1949,7 +1949,7 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) ahci_mark_external_port(ap); - ahci_update_initial_lpm_policy(ap, hpriv); + ahci_update_initial_lpm_policy(ap); /* disabled/not-implemented port */ if (!(hpriv->port_map & (1 << i))) From ae1f3db006b71b78f2cae848b3a1446b02818ca7 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Tue, 6 Feb 2024 22:13:45 +0100 Subject: [PATCH 0208/1038] ata: ahci: do not enable LPM on external ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AHCI 1.3.3, 7.3.1.1 Software Flow for Hot Plug Removal Detection states: "To reliably detect hot plug removals, software must disable interface power management. Software should perform the following initialization on a port after a device is attached: -Set PxSCTL.IPM to 3h to disable interface power management state transitions. -Set PxCMD.ALPE to ‘0’ to disable aggressive power management. -Ensure PxIE.PRCE is set to ‘1’ to enable interrupts on hot plug removals. -Disable device initiated interface power management by issuing the appropriate SET FEATURES command." Further, AHCI 1.3.3, 7.3 Native Hot Plug Support states: "The HBA shall set the PxSERR.DIAG.X bit to ‘1’ when a COMINIT is received from the device. Hot plug insertions are detected via the PxIS.PCS bit that directly reflects the PxSERR.DIAG.X bit. The HBA shall set the PxSERR.DIAG.N bit to ‘1’ when the HBA’s internal PhyRdy signal changes state. Hot plug removals are detected via the PxIS.PRCS bit that directly reflects the PxSERR.DIAG.N bit. Note that PxSERR.DIAG.N is also set to ‘1’ on insertions and during interface power management entry/exit." ahci_set_lpm() already disables the PxIS.PRCS interrupt if setting a LPM policy != ATA_LPM_MAX_POWER, so we cannot detect hot plug removals when LPM policy != ATA_LPM_MAX_POWER. We do have PxIS.PCS interrupt enabled even for LPM policy != ATA_LPM_MAX_POWER, so we should theoretically still be able to detect hot plug insertions even when LPM is enabled. However, in practise, for LPM policy ATA_LPM_MED_POWER_WITH_DIPM, ATA_LPM_MIN_POWER_WITH_PARTIAL, and ATA_LPM_MIN_POWER, if there is no link enabled, sata_link_scr_lpm() will set SControl.DET = 0x4, which will transition the port to the "P:Offline" state. The P:Offline mode is described in SATA Gold 3.5a: 4.1.1.103 Phy offline: "In this mode the host Phy is forced off and the host Phy does not recognize nor respond to COMINIT or COMWAKE. This mode is entered by setting the DET field of the SControl register to 0100b. This is a mechanism for the host to turn off its Phy." So in the P:Offline state the PHY does not recognize the unsolicited COMINIT which is sent on a hot plug insertion. While we could change sata_link_scr_lpm() to never power off an external port for LPM policy != ATA_LPM_MAX_POWER (in order be able to handle hot plug insertions), we still would not be able to handle hot plug removals. Thus, simply modify ahci_update_initial_lpm_policy() to not enable LPM if the port advertises itself as an external port, as this function is already being used to set/override the initial LPM policy. Tested-by: Damien Le Moal Tested-by: Jian-Hong Pan Reviewed-by: Damien Le Moal Reviewed-by: Mario Limonciello Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 346a0f9ef246..9d052ff2b86c 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1665,6 +1665,15 @@ static void ahci_update_initial_lpm_policy(struct ata_port *ap) if (!(hpriv->flags & AHCI_HFLAG_USE_LPM_POLICY)) return; + /* + * AHCI contains a known incompatibility between LPM and hot-plug + * removal events, see 7.3.1 Hot Plug Removal Detection and Power + * Management Interaction in AHCI 1.3.1. Therefore, do not enable + * LPM if the port advertises itself as an external port. + */ + if (ap->pflags & ATA_PFLAG_EXTERNAL) + return; + /* user modified policy via module param */ if (mobile_lpm_policy != -1) { policy = mobile_lpm_policy; From 7627a0edef548c4c4dea62df51cc26bfe5bbcab8 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 6 Feb 2024 22:13:46 +0100 Subject: [PATCH 0209/1038] ata: ahci: Drop low power policy board type The low power policy board type was introduced to allow systems to get into deep states reliably. Before it was introduced `min_power` was causing problems for a number of drives. New power policies `min_power_with_partial` and `med_power_with_dipm` have been introduced which provide a more stable baseline for systems. Tested-by: Damien Le Moal Tested-by: Jian-Hong Pan Acked-by: Jian-Hong Pan Acked-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Mario Limonciello Reviewed-by: Mika Westerberg Suggested-by: Christoph Hellwig Signed-off-by: Mario Limonciello [cassel: rebase patch and fix trivial conflicts] Signed-off-by: Niklas Cassel --- drivers/ata/Kconfig | 5 +- drivers/ata/ahci.c | 109 +++++++++++++++++++------------------------- drivers/ata/ahci.h | 9 ++-- 3 files changed, 53 insertions(+), 70 deletions(-) diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index 42b51c9812a0..928ec93c6b45 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -116,15 +116,14 @@ config SATA_AHCI If unsure, say N. config SATA_MOBILE_LPM_POLICY - int "Default SATA Link Power Management policy for low power chipsets" + int "Default SATA Link Power Management policy" range 0 4 default 0 depends on SATA_AHCI help Select the Default SATA Link Power Management (LPM) policy to use for chipsets / "South Bridges" supporting low-power modes. Such - chipsets are typically found on most laptops but desktops and - servers now also widely use chipsets supporting low power modes. + chipsets are ubiquitous across laptops, desktops and servers. The value set has the following meanings: 0 => Keep firmware settings diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 9d052ff2b86c..ae0a592e2185 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -50,7 +50,6 @@ enum board_ids { board_ahci, board_ahci_43bit_dma, board_ahci_ign_iferr, - board_ahci_low_power, board_ahci_no_debounce_delay, board_ahci_nomsi, board_ahci_noncq, @@ -143,13 +142,6 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_low_power] = { - AHCI_HFLAGS (AHCI_HFLAG_USE_LPM_POLICY), - .flags = AHCI_FLAG_COMMON, - .pio_mask = ATA_PIO4, - .udma_mask = ATA_UDMA6, - .port_ops = &ahci_ops, - }, [board_ahci_no_debounce_delay] = { .flags = AHCI_FLAG_COMMON, .link_flags = ATA_LFLAG_NO_DEBOUNCE_DELAY, @@ -283,13 +275,13 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */ { PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */ { PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2929), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292a), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292b), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292c), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292f), board_ahci_low_power }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */ { PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x294e), board_ahci_low_power }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */ { PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */ { PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */ { PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */ @@ -299,9 +291,9 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */ { PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */ { PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b29), board_ahci_low_power }, /* PCH M AHCI */ + { PCI_VDEVICE(INTEL, 0x3b29), board_ahci }, /* PCH M AHCI */ { PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_low_power }, /* PCH M RAID */ + { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci }, /* PCH M RAID */ { PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */ { PCI_VDEVICE(INTEL, 0x19b0), board_ahci_pcs7 }, /* DNV AHCI */ { PCI_VDEVICE(INTEL, 0x19b1), board_ahci_pcs7 }, /* DNV AHCI */ @@ -324,9 +316,9 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x19cE), board_ahci_pcs7 }, /* DNV AHCI */ { PCI_VDEVICE(INTEL, 0x19cF), board_ahci_pcs7 }, /* DNV AHCI */ { PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */ - { PCI_VDEVICE(INTEL, 0x1c03), board_ahci_low_power }, /* CPT M AHCI */ + { PCI_VDEVICE(INTEL, 0x1c03), board_ahci }, /* CPT M AHCI */ { PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1c05), board_ahci_low_power }, /* CPT M RAID */ + { PCI_VDEVICE(INTEL, 0x1c05), board_ahci }, /* CPT M RAID */ { PCI_VDEVICE(INTEL, 0x1c06), board_ahci }, /* CPT RAID */ { PCI_VDEVICE(INTEL, 0x1c07), board_ahci }, /* CPT RAID */ { PCI_VDEVICE(INTEL, 0x1d02), board_ahci }, /* PBG AHCI */ @@ -334,29 +326,29 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x1d06), board_ahci }, /* PBG RAID */ { PCI_VDEVICE(INTEL, 0x2323), board_ahci }, /* DH89xxCC AHCI */ { PCI_VDEVICE(INTEL, 0x1e02), board_ahci }, /* Panther Point AHCI */ - { PCI_VDEVICE(INTEL, 0x1e03), board_ahci_low_power }, /* Panther M AHCI */ + { PCI_VDEVICE(INTEL, 0x1e03), board_ahci }, /* Panther M AHCI */ { PCI_VDEVICE(INTEL, 0x1e04), board_ahci }, /* Panther Point RAID */ { PCI_VDEVICE(INTEL, 0x1e05), board_ahci }, /* Panther Point RAID */ { PCI_VDEVICE(INTEL, 0x1e06), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e07), board_ahci_low_power }, /* Panther M RAID */ + { PCI_VDEVICE(INTEL, 0x1e07), board_ahci }, /* Panther M RAID */ { PCI_VDEVICE(INTEL, 0x1e0e), board_ahci }, /* Panther Point RAID */ { PCI_VDEVICE(INTEL, 0x8c02), board_ahci }, /* Lynx Point AHCI */ - { PCI_VDEVICE(INTEL, 0x8c03), board_ahci_low_power }, /* Lynx M AHCI */ + { PCI_VDEVICE(INTEL, 0x8c03), board_ahci }, /* Lynx M AHCI */ { PCI_VDEVICE(INTEL, 0x8c04), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c05), board_ahci_low_power }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x8c05), board_ahci }, /* Lynx M RAID */ { PCI_VDEVICE(INTEL, 0x8c06), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c07), board_ahci_low_power }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x8c07), board_ahci }, /* Lynx M RAID */ { PCI_VDEVICE(INTEL, 0x8c0e), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_low_power }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x9c02), board_ahci_low_power }, /* Lynx LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c03), board_ahci_low_power }, /* Lynx LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c04), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c05), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c06), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c07), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9dd3), board_ahci_low_power }, /* Cannon Lake PCH-LP AHCI */ + { PCI_VDEVICE(INTEL, 0x8c0f), board_ahci }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x9c02), board_ahci }, /* Lynx LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c03), board_ahci }, /* Lynx LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c04), board_ahci }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c05), board_ahci }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c06), board_ahci }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c07), board_ahci }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c0e), board_ahci }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c0f), board_ahci }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9dd3), board_ahci }, /* Cannon Lake PCH-LP AHCI */ { PCI_VDEVICE(INTEL, 0x1f22), board_ahci }, /* Avoton AHCI */ { PCI_VDEVICE(INTEL, 0x1f23), board_ahci }, /* Avoton AHCI */ { PCI_VDEVICE(INTEL, 0x1f24), board_ahci }, /* Avoton RAID */ @@ -390,26 +382,26 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x8d66), board_ahci }, /* Wellsburg RAID */ { PCI_VDEVICE(INTEL, 0x8d6e), board_ahci }, /* Wellsburg RAID */ { PCI_VDEVICE(INTEL, 0x23a3), board_ahci }, /* Coleto Creek AHCI */ - { PCI_VDEVICE(INTEL, 0x9c83), board_ahci_low_power }, /* Wildcat LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c85), board_ahci_low_power }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c87), board_ahci_low_power }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_low_power }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c83), board_ahci }, /* Wildcat LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c85), board_ahci }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c87), board_ahci }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c8f), board_ahci }, /* Wildcat LP RAID */ { PCI_VDEVICE(INTEL, 0x8c82), board_ahci }, /* 9 Series AHCI */ - { PCI_VDEVICE(INTEL, 0x8c83), board_ahci_low_power }, /* 9 Series M AHCI */ + { PCI_VDEVICE(INTEL, 0x8c83), board_ahci }, /* 9 Series M AHCI */ { PCI_VDEVICE(INTEL, 0x8c84), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c85), board_ahci_low_power }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x8c85), board_ahci }, /* 9 Series M RAID */ { PCI_VDEVICE(INTEL, 0x8c86), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c87), board_ahci_low_power }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x8c87), board_ahci }, /* 9 Series M RAID */ { PCI_VDEVICE(INTEL, 0x8c8e), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_low_power }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x9d03), board_ahci_low_power }, /* Sunrise LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9d05), board_ahci_low_power }, /* Sunrise LP RAID */ - { PCI_VDEVICE(INTEL, 0x9d07), board_ahci_low_power }, /* Sunrise LP RAID */ + { PCI_VDEVICE(INTEL, 0x8c8f), board_ahci }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x9d03), board_ahci }, /* Sunrise LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9d05), board_ahci }, /* Sunrise LP RAID */ + { PCI_VDEVICE(INTEL, 0x9d07), board_ahci }, /* Sunrise LP RAID */ { PCI_VDEVICE(INTEL, 0xa102), board_ahci }, /* Sunrise Point-H AHCI */ - { PCI_VDEVICE(INTEL, 0xa103), board_ahci_low_power }, /* Sunrise M AHCI */ + { PCI_VDEVICE(INTEL, 0xa103), board_ahci }, /* Sunrise M AHCI */ { PCI_VDEVICE(INTEL, 0xa105), board_ahci }, /* Sunrise Point-H RAID */ { PCI_VDEVICE(INTEL, 0xa106), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa107), board_ahci_low_power }, /* Sunrise M RAID */ + { PCI_VDEVICE(INTEL, 0xa107), board_ahci }, /* Sunrise M RAID */ { PCI_VDEVICE(INTEL, 0xa10f), board_ahci }, /* Sunrise Point-H RAID */ { PCI_VDEVICE(INTEL, 0xa182), board_ahci }, /* Lewisburg AHCI*/ { PCI_VDEVICE(INTEL, 0xa186), board_ahci }, /* Lewisburg RAID*/ @@ -422,16 +414,16 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0xa356), board_ahci }, /* Cannon Lake PCH-H RAID */ { PCI_VDEVICE(INTEL, 0x06d7), board_ahci }, /* Comet Lake-H RAID */ { PCI_VDEVICE(INTEL, 0xa386), board_ahci }, /* Comet Lake PCH-V RAID */ - { PCI_VDEVICE(INTEL, 0x0f22), board_ahci_low_power }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x0f23), board_ahci_low_power }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x22a3), board_ahci_low_power }, /* Cherry Tr. AHCI */ - { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_low_power }, /* ApolloLake AHCI */ - { PCI_VDEVICE(INTEL, 0x34d3), board_ahci_low_power }, /* Ice Lake LP AHCI */ - { PCI_VDEVICE(INTEL, 0x02d3), board_ahci_low_power }, /* Comet Lake PCH-U AHCI */ - { PCI_VDEVICE(INTEL, 0x02d7), board_ahci_low_power }, /* Comet Lake PCH RAID */ + { PCI_VDEVICE(INTEL, 0x0f22), board_ahci }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x0f23), board_ahci }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x22a3), board_ahci }, /* Cherry Tr. AHCI */ + { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci }, /* ApolloLake AHCI */ + { PCI_VDEVICE(INTEL, 0x34d3), board_ahci }, /* Ice Lake LP AHCI */ + { PCI_VDEVICE(INTEL, 0x02d3), board_ahci }, /* Comet Lake PCH-U AHCI */ + { PCI_VDEVICE(INTEL, 0x02d7), board_ahci }, /* Comet Lake PCH RAID */ /* Elkhart Lake IDs 0x4b60 & 0x4b62 https://sata-io.org/product/8803 not tested yet */ - { PCI_VDEVICE(INTEL, 0x4b63), board_ahci_low_power }, /* Elkhart Lake AHCI */ - { PCI_VDEVICE(INTEL, 0x7ae2), board_ahci_low_power }, /* Alder Lake-P AHCI */ + { PCI_VDEVICE(INTEL, 0x4b63), board_ahci }, /* Elkhart Lake AHCI */ + { PCI_VDEVICE(INTEL, 0x7ae2), board_ahci }, /* Alder Lake-P AHCI */ /* JMicron 360/1/3/5/6, match class to avoid IDE function */ { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, @@ -459,7 +451,7 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(AMD, 0x7800), board_ahci }, /* AMD Hudson-2 */ { PCI_VDEVICE(AMD, 0x7801), board_ahci_no_debounce_delay }, /* AMD Hudson-2 (AHCI mode) */ { PCI_VDEVICE(AMD, 0x7900), board_ahci }, /* AMD CZ */ - { PCI_VDEVICE(AMD, 0x7901), board_ahci_low_power }, /* AMD Green Sardine */ + { PCI_VDEVICE(AMD, 0x7901), board_ahci }, /* AMD Green Sardine */ /* AMD is using RAID class only for ahci controllers */ { PCI_VENDOR_ID_AMD, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci }, @@ -1660,11 +1652,6 @@ static void ahci_update_initial_lpm_policy(struct ata_port *ap) struct ahci_host_priv *hpriv = ap->host->private_data; int policy = CONFIG_SATA_MOBILE_LPM_POLICY; - - /* Ignore processing for chipsets that don't use policy */ - if (!(hpriv->flags & AHCI_HFLAG_USE_LPM_POLICY)) - return; - /* * AHCI contains a known incompatibility between LPM and hot-plug * removal events, see 7.3.1 Hot Plug Removal Detection and Power diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index df8f8a1a3a34..4a0a602c6b16 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -241,13 +241,10 @@ enum { AHCI_HFLAG_YES_ALPM = BIT(23), /* force ALPM cap on */ AHCI_HFLAG_NO_WRITE_TO_RO = BIT(24), /* don't write to read only registers */ - AHCI_HFLAG_USE_LPM_POLICY = BIT(25), /* chipset that should use - SATA_MOBILE_LPM_POLICY - as default lpm_policy */ - AHCI_HFLAG_SUSPEND_PHYS = BIT(26), /* handle PHYs during + AHCI_HFLAG_SUSPEND_PHYS = BIT(25), /* handle PHYs during suspend/resume */ - AHCI_HFLAG_NO_SXS = BIT(28), /* SXS not supported */ - AHCI_HFLAG_43BIT_ONLY = BIT(29), /* 43bit DMA addr limit */ + AHCI_HFLAG_NO_SXS = BIT(26), /* SXS not supported */ + AHCI_HFLAG_43BIT_ONLY = BIT(27), /* 43bit DMA addr limit */ /* ap->flags bits */ From be4f5995875353efd5c438375b2a68453f137b6e Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Jan 2024 09:01:00 +0000 Subject: [PATCH 0210/1038] iommu/amd: Remove unused PPR_* macros Commit 5a0b11a180a ("iommu/amd: Remove iommu_v2 module") missed to remove PPR_* macros. Remove these macros as its not used anymore. No functional change intended. Signed-off-by: Vasant Hegde Reviewed-by: Suravee Suthikulpanit Reviewed-by: Alejandro Jimenez Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240118090105.5864-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 8b3601f285fd..2cef9796f09c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -77,10 +77,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu) } #endif -#define PPR_SUCCESS 0x0 -#define PPR_INVALID 0x1 -#define PPR_FAILURE 0xf - int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, int status, int tag); From a4086637664926b52f5ecd087b3957f935ce48fb Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Jan 2024 09:01:01 +0000 Subject: [PATCH 0211/1038] iommu/amd: Remove unused IOVA_* macro These macros are not used after commit ac6d704679d343 ("iommu/dma: Pass address limit rather than size to iommu_setup_dma_ops()"). No functional change intended. Signed-off-by: Vasant Hegde Cc: Lu Baolu Reviewed-by: Suravee Suthikulpanit Reviewed-by: Alejandro Jimenez Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240118090105.5864-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4283dd8191f0..7fe3c9d44906 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -45,10 +45,6 @@ #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) -/* IO virtual address start page frame number */ -#define IOVA_START_PFN (1) -#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) - /* Reserved IOVA ranges */ #define MSI_RANGE_START (0xfee00000) #define MSI_RANGE_END (0xfeefffff) From 2edf056f57f59cf35980786f1ad4f681b9a5bebf Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Jan 2024 09:01:02 +0000 Subject: [PATCH 0212/1038] iommu/amd: Remove unused APERTURE_* macros These macros are not used after commit 518d9b450387 ("iommu/amd: Remove special mapping code for dma_ops path"). No functional change intended. Signed-off-by: Vasant Hegde Reviewed-by: Suravee Suthikulpanit Reviewed-by: Alejandro Jimenez Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240118090105.5864-4-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 809d74faa1a5..fdf7807280bd 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -513,14 +513,6 @@ extern struct kmem_cache *amd_iommu_irq_cache; #define for_each_iommu_safe(iommu, next) \ list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) -#define APERTURE_RANGE_SHIFT 27 /* 128 MB */ -#define APERTURE_RANGE_SIZE (1ULL << APERTURE_RANGE_SHIFT) -#define APERTURE_RANGE_PAGES (APERTURE_RANGE_SIZE >> PAGE_SHIFT) -#define APERTURE_MAX_RANGES 32 /* allows 4GB of DMA address space */ -#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) -#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) - - struct amd_iommu; struct iommu_domain; struct irq_domain; From e7b3533c81386464dfdcb01193075f8a9557083a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 5 Feb 2024 15:32:39 +0000 Subject: [PATCH 0213/1038] iommu/iova: Tidy up iova_cache_get() failure Failure handling in iova_cache_get() is a little messy, and we'd like to add some more to it, so let's tidy up a bit first. By leaving the hotplug handler until last we can take advantage of kmem_cache_destroy() being NULL-safe to have a single cleanup label. We can also improve the error reporting, noting that kmem_cache_create() already screams if it fails, so that one is redundant. Signed-off-by: Robin Murphy Acked-by: David Rientjes Reviewed-by: Pasha Tatashin Reviewed-by: John Garry Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/ae4a3bda2d6a9b738221553c838d30473bd624e7.1707144953.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index d30e453d0fb4..cf95001d85c0 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -254,26 +254,20 @@ static void free_iova_mem(struct iova *iova) int iova_cache_get(void) { + int err = -ENOMEM; + mutex_lock(&iova_cache_mutex); if (!iova_cache_users) { - int ret; + iova_cache = kmem_cache_create("iommu_iova", sizeof(struct iova), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!iova_cache) + goto out_err; - ret = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", NULL, - iova_cpuhp_dead); - if (ret) { - mutex_unlock(&iova_cache_mutex); - pr_err("Couldn't register cpuhp handler\n"); - return ret; - } - - iova_cache = kmem_cache_create( - "iommu_iova", sizeof(struct iova), 0, - SLAB_HWCACHE_ALIGN, NULL); - if (!iova_cache) { - cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); - mutex_unlock(&iova_cache_mutex); - pr_err("Couldn't create iova cache\n"); - return -ENOMEM; + err = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", + NULL, iova_cpuhp_dead); + if (err) { + pr_err("IOVA: Couldn't register cpuhp handler: %pe\n", ERR_PTR(err)); + goto out_err; } } @@ -281,6 +275,11 @@ int iova_cache_get(void) mutex_unlock(&iova_cache_mutex); return 0; + +out_err: + kmem_cache_destroy(iova_cache); + mutex_unlock(&iova_cache_mutex); + return err; } EXPORT_SYMBOL_GPL(iova_cache_get); From 7f845d8b2eed0986a03a777d4956b52a57007974 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 5 Feb 2024 15:32:40 +0000 Subject: [PATCH 0214/1038] iommu/iova: Reorganise some code The iova_cache_{get,put}() calls really represent top-level lifecycle management for the whole IOVA library, so it's long been rather confusing to have them buried right in the middle of the allocator implementation details. Move them to a more expected position at the end of the file, where it will then also be easier to expand them. With this, we can also move the rcache hotplug handler (plus another stray function) into the rcache portion of the file. Signed-off-by: Robin Murphy Acked-by: David Rientjes Reviewed-by: Pasha Tatashin Reviewed-by: John Garry Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/d4753562f4faa0e6b3aeebcbf88fdb60cc22d715.1707144953.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 128 +++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index cf95001d85c0..b5de865ee50b 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -24,24 +24,8 @@ static bool iova_rcache_insert(struct iova_domain *iovad, static unsigned long iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn); -static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); - -unsigned long iova_rcache_range(void) -{ - return PAGE_SIZE << (IOVA_RANGE_CACHE_MAX_SIZE - 1); -} - -static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) -{ - struct iova_domain *iovad; - - iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); - - free_cpu_cached_iovas(cpu, iovad); - return 0; -} - +static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_global_cached_iovas(struct iova_domain *iovad); static struct iova *to_iova(struct rb_node *node) @@ -252,53 +236,6 @@ static void free_iova_mem(struct iova *iova) kmem_cache_free(iova_cache, iova); } -int iova_cache_get(void) -{ - int err = -ENOMEM; - - mutex_lock(&iova_cache_mutex); - if (!iova_cache_users) { - iova_cache = kmem_cache_create("iommu_iova", sizeof(struct iova), 0, - SLAB_HWCACHE_ALIGN, NULL); - if (!iova_cache) - goto out_err; - - err = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", - NULL, iova_cpuhp_dead); - if (err) { - pr_err("IOVA: Couldn't register cpuhp handler: %pe\n", ERR_PTR(err)); - goto out_err; - } - } - - iova_cache_users++; - mutex_unlock(&iova_cache_mutex); - - return 0; - -out_err: - kmem_cache_destroy(iova_cache); - mutex_unlock(&iova_cache_mutex); - return err; -} -EXPORT_SYMBOL_GPL(iova_cache_get); - -void iova_cache_put(void) -{ - mutex_lock(&iova_cache_mutex); - if (WARN_ON(!iova_cache_users)) { - mutex_unlock(&iova_cache_mutex); - return; - } - iova_cache_users--; - if (!iova_cache_users) { - cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); - kmem_cache_destroy(iova_cache); - } - mutex_unlock(&iova_cache_mutex); -} -EXPORT_SYMBOL_GPL(iova_cache_put); - /** * alloc_iova - allocates an iova * @iovad: - iova domain in question @@ -653,6 +590,11 @@ struct iova_rcache { struct delayed_work work; }; +unsigned long iova_rcache_range(void) +{ + return PAGE_SIZE << (IOVA_RANGE_CACHE_MAX_SIZE - 1); +} + static struct iova_magazine *iova_magazine_alloc(gfp_t flags) { struct iova_magazine *mag; @@ -989,5 +931,63 @@ static void free_global_cached_iovas(struct iova_domain *iovad) spin_unlock_irqrestore(&rcache->lock, flags); } } + +static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) +{ + struct iova_domain *iovad; + + iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); + + free_cpu_cached_iovas(cpu, iovad); + return 0; +} + +int iova_cache_get(void) +{ + int err = -ENOMEM; + + mutex_lock(&iova_cache_mutex); + if (!iova_cache_users) { + iova_cache = kmem_cache_create("iommu_iova", sizeof(struct iova), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!iova_cache) + goto out_err; + + err = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", + NULL, iova_cpuhp_dead); + if (err) { + pr_err("IOVA: Couldn't register cpuhp handler: %pe\n", ERR_PTR(err)); + goto out_err; + } + } + + iova_cache_users++; + mutex_unlock(&iova_cache_mutex); + + return 0; + +out_err: + kmem_cache_destroy(iova_cache); + mutex_unlock(&iova_cache_mutex); + return err; +} +EXPORT_SYMBOL_GPL(iova_cache_get); + +void iova_cache_put(void) +{ + mutex_lock(&iova_cache_mutex); + if (WARN_ON(!iova_cache_users)) { + mutex_unlock(&iova_cache_mutex); + return; + } + iova_cache_users--; + if (!iova_cache_users) { + cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); + kmem_cache_destroy(iova_cache); + } + mutex_unlock(&iova_cache_mutex); +} +EXPORT_SYMBOL_GPL(iova_cache_put); + MODULE_AUTHOR("Anil S Keshavamurthy "); MODULE_LICENSE("GPL"); From 84e6f56be9c68b59aca51544f7ee33706542d7bc Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Mon, 5 Feb 2024 15:32:41 +0000 Subject: [PATCH 0215/1038] iommu/iova: use named kmem_cache for iova magazines The magazine buffers can take gigabytes of kmem memory, dominating all other allocations. For observability purpose create named slab cache so the iova magazine memory overhead can be clearly observed. With this change: > slabtop -o | head Active / Total Objects (% used) : 869731 / 952904 (91.3%) Active / Total Slabs (% used) : 103411 / 103974 (99.5%) Active / Total Caches (% used) : 135 / 211 (64.0%) Active / Total Size (% used) : 395389.68K / 411430.20K (96.1%) Minimum / Average / Maximum Object : 0.02K / 0.43K / 8.00K OBJS ACTIVE USE OBJ SIZE SLABS OBJ/SLAB CACHE SIZE NAME 244412 244239 99% 1.00K 61103 4 244412K iommu_iova_magazine 91636 88343 96% 0.03K 739 124 2956K kmalloc-32 75744 74844 98% 0.12K 2367 32 9468K kernfs_node_cache On this machine it is now clear that magazine use 242M of kmem memory. Acked-by: David Rientjes Signed-off-by: Pasha Tatashin [ rm: adjust to rework of iova_cache_{get,put} ] Signed-off-by: Robin Murphy Reviewed-by: John Garry Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/dc5c51aaba50906a92b9ba1a5137ed462484a7be.1707144953.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index b5de865ee50b..d59d0ea2fd21 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -590,6 +590,8 @@ struct iova_rcache { struct delayed_work work; }; +static struct kmem_cache *iova_magazine_cache; + unsigned long iova_rcache_range(void) { return PAGE_SIZE << (IOVA_RANGE_CACHE_MAX_SIZE - 1); @@ -599,7 +601,7 @@ static struct iova_magazine *iova_magazine_alloc(gfp_t flags) { struct iova_magazine *mag; - mag = kmalloc(sizeof(*mag), flags); + mag = kmem_cache_alloc(iova_magazine_cache, flags); if (mag) mag->size = 0; @@ -608,7 +610,7 @@ static struct iova_magazine *iova_magazine_alloc(gfp_t flags) static void iova_magazine_free(struct iova_magazine *mag) { - kfree(mag); + kmem_cache_free(iova_magazine_cache, mag); } static void @@ -953,6 +955,12 @@ int iova_cache_get(void) if (!iova_cache) goto out_err; + iova_magazine_cache = kmem_cache_create("iommu_iova_magazine", + sizeof(struct iova_magazine), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!iova_magazine_cache) + goto out_err; + err = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", NULL, iova_cpuhp_dead); if (err) { @@ -968,6 +976,7 @@ int iova_cache_get(void) out_err: kmem_cache_destroy(iova_cache); + kmem_cache_destroy(iova_magazine_cache); mutex_unlock(&iova_cache_mutex); return err; } @@ -984,6 +993,7 @@ void iova_cache_put(void) if (!iova_cache_users) { cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); kmem_cache_destroy(iova_cache); + kmem_cache_destroy(iova_magazine_cache); } mutex_unlock(&iova_cache_mutex); } From f2d6677ad5772d4972b11dc3a4c3e555973b27d9 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 5 Feb 2024 16:43:27 +0000 Subject: [PATCH 0216/1038] iommu/ipmmu-vmsa: Minor cleanups Remove the of_match_ptr() which was supposed to have gone long ago, but managed to got lost in a fix-squashing mishap. On a similar theme, we may as well also modernise the PM ops to get rid of the clunky #ifdefs, and modernise the resource mapping to keep the checkpatch brigade happy. Link: https://lore.kernel.org/linux-iommu/Yxni3d6CdI3FZ5D+@8bytes.org/ Signed-off-by: Robin Murphy Reviewed-by: Yoshihiro Shimoda Link: https://lore.kernel.org/r/791877b0d310dc2ab7dc616d2786ab24252b9b8e.1707151207.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index ace1fc4bd34b..90d3f03242db 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -1005,7 +1005,6 @@ static const struct of_device_id ipmmu_of_ids[] = { static int ipmmu_probe(struct platform_device *pdev) { struct ipmmu_vmsa_device *mmu; - struct resource *res; int irq; int ret; @@ -1025,8 +1024,7 @@ static int ipmmu_probe(struct platform_device *pdev) return ret; /* Map I/O memory and request IRQ. */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mmu->base = devm_ioremap_resource(&pdev->dev, res); + mmu->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mmu->base)) return PTR_ERR(mmu->base); @@ -1123,7 +1121,6 @@ static void ipmmu_remove(struct platform_device *pdev) ipmmu_device_reset(mmu); } -#ifdef CONFIG_PM_SLEEP static int ipmmu_resume_noirq(struct device *dev) { struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); @@ -1153,18 +1150,14 @@ static int ipmmu_resume_noirq(struct device *dev) } static const struct dev_pm_ops ipmmu_pm = { - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) }; -#define DEV_PM_OPS &ipmmu_pm -#else -#define DEV_PM_OPS NULL -#endif /* CONFIG_PM_SLEEP */ static struct platform_driver ipmmu_driver = { .driver = { .name = "ipmmu-vmsa", - .of_match_table = of_match_ptr(ipmmu_of_ids), - .pm = DEV_PM_OPS, + .of_match_table = ipmmu_of_ids, + .pm = pm_sleep_ptr(&ipmmu_pm), }, .probe = ipmmu_probe, .remove_new = ipmmu_remove, From 773b05e7f407c377d64a9173ceed54489cc47c3a Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Jan 2024 09:01:03 +0000 Subject: [PATCH 0217/1038] iommu/amd: Remove duplicate function declarations from amd_iommu.h Perf counter related functions are defined in amd-iommu.h as well. Hence remove duplicate declarations. No functional change intended. Signed-off-by: Vasant Hegde Reviewed-by: Suravee Suthikulpanit Reviewed-by: Alejandro Jimenez Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240118090105.5864-5-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 2cef9796f09c..7464222ad8af 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -39,14 +39,6 @@ extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; bool amd_iommu_v2_supported(void); -struct amd_iommu *get_amd_iommu(unsigned int idx); -u8 amd_iommu_pc_get_max_banks(unsigned int idx); -bool amd_iommu_pc_supported(void); -u8 amd_iommu_pc_get_max_counters(unsigned int idx); -int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value); -int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value); /* Device capabilities */ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); From 2dc9506bfb135edb684eb4f0bf17ea847c18b2a0 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Jan 2024 09:01:04 +0000 Subject: [PATCH 0218/1038] iommu/amd: Remove redundant error check in amd_iommu_probe_device() iommu_init_device() is not returning -ENOTSUPP since commit 61289cbaf6c8 ("iommu/amd: Remove old alias handling code"). No functional change intended. Signed-off-by: Vasant Hegde Reviewed-by: Suravee Suthikulpanit Reviewed-by: Alejandro Jimenez Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240118090105.5864-6-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 7fe3c9d44906..2e50f116672f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1950,8 +1950,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) ret = iommu_init_device(iommu, dev); if (ret) { - if (ret != -ENOTSUPP) - dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); + dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); iommu_dev = ERR_PTR(ret); iommu_ignore_device(iommu, dev); } else { From 108042db53a1eb192b972098669f36c6379ef159 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Jan 2024 09:01:05 +0000 Subject: [PATCH 0219/1038] iommu/amd: Remove EXPORT_SYMBOL for perf counter related functions .. as IOMMU perf counters are always built as part of kernel. No functional change intended. Signed-off-by: Vasant Hegde Reviewed-by: Suravee Suthikulpanit Reviewed-by: Alejandro Jimenez Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240118090105.5864-7-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index c83bd0c2a1c9..e72fba4a8111 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3694,13 +3694,11 @@ u8 amd_iommu_pc_get_max_banks(unsigned int idx) return 0; } -EXPORT_SYMBOL(amd_iommu_pc_get_max_banks); bool amd_iommu_pc_supported(void) { return amd_iommu_pc_present; } -EXPORT_SYMBOL(amd_iommu_pc_supported); u8 amd_iommu_pc_get_max_counters(unsigned int idx) { @@ -3711,7 +3709,6 @@ u8 amd_iommu_pc_get_max_counters(unsigned int idx) return 0; } -EXPORT_SYMBOL(amd_iommu_pc_get_max_counters); static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value, bool is_write) From a6ffb9b3d71ea21c9d56ac5856fe321ef584bb19 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:55:59 +0000 Subject: [PATCH 0220/1038] iommu/amd: Pass struct iommu_dev_data to set_dte_entry() Pass iommu_dev_data structure instead of passing indivisual variables. No functional changes intended. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 2e50f116672f..743465cb677f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1700,12 +1700,14 @@ static int setup_gcr3_table(struct protection_domain *domain, int pasids) return 0; } -static void set_dte_entry(struct amd_iommu *iommu, u16 devid, - struct protection_domain *domain, bool ats, bool ppr) +static void set_dte_entry(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data) { u64 pte_root = 0; u64 flags = 0; u32 old_domid; + u16 devid = dev_data->devid; + struct protection_domain *domain = dev_data->domain; struct dev_table_entry *dev_table = get_dev_table(iommu); if (domain->iop.mode != PAGE_MODE_NONE) @@ -1725,10 +1727,10 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, flags = dev_table[devid].data[1]; - if (ats) + if (dev_data->ats_enabled) flags |= DTE_FLAG_IOTLB; - if (ppr) + if (dev_data->ppr) pte_root |= 1ULL << DEV_ENTRY_PPR; if (domain->dirty_tracking) @@ -1804,12 +1806,10 @@ static void do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { struct amd_iommu *iommu; - bool ats; iommu = rlookup_amd_iommu(dev_data->dev); if (!iommu) return; - ats = dev_data->ats_enabled; /* Update data structures */ dev_data->domain = domain; @@ -1824,8 +1824,7 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_cnt += 1; /* Update device table */ - set_dte_entry(iommu, dev_data->devid, domain, - ats, dev_data->ppr); + set_dte_entry(iommu, dev_data); clone_aliases(iommu, dev_data->dev); device_flush_dte(dev_data); @@ -2008,8 +2007,7 @@ static void update_device_table(struct protection_domain *domain) if (!iommu) continue; - set_dte_entry(iommu, dev_data->devid, domain, - dev_data->ats_enabled, dev_data->ppr); + set_dte_entry(iommu, dev_data); clone_aliases(iommu, dev_data->dev); } } From 8e017973317235468a330aa197134c1aebbf13da Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:00 +0000 Subject: [PATCH 0221/1038] iommu/amd: Enable Guest Translation before registering devices IOMMU Guest Translation (GT) feature needs to be enabled before invalidating guest translations (CMD_INV_IOMMU_PAGES with GN=1). Currently GT feature is enabled after setting up interrupt handler. So far it was fine as we were not invalidating guest page table before this point. Upcoming series will introduce per device GCR3 table and it will invalidate guest pages after configuring. Hence move GT feature enablement to early_enable_iommu(). Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index e72fba4a8111..cb8486c9f8f4 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2769,6 +2769,7 @@ static void early_enable_iommu(struct amd_iommu *iommu) iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); iommu_set_exclusion_range(iommu); + iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2825,6 +2826,7 @@ static void early_enable_iommus(void) iommu_disable_irtcachedis(iommu); iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); + iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2838,10 +2840,8 @@ static void enable_iommus_v2(void) { struct amd_iommu *iommu; - for_each_iommu(iommu) { + for_each_iommu(iommu) iommu_enable_ppr_log(iommu); - iommu_enable_gt(iommu); - } } static void enable_iommus_vapic(void) From 6f35fe5d8a0ad0125c52fb20f5d67000e369eb3a Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:01 +0000 Subject: [PATCH 0222/1038] iommu/amd: Introduce get_amd_iommu_from_dev() Introduce get_amd_iommu_from_dev() and get_amd_iommu_from_dev_data(). And replace rlookup_amd_iommu() with the new helper function where applicable to avoid unnecessary loop to look up struct amd_iommu from struct device. Suggested-by: Jason Gunthorpe Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-4-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 15 ++++++++++ drivers/iommu/amd/iommu.c | 54 +++++++++-------------------------- include/linux/iommu.h | 16 +++++++++++ 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 7464222ad8af..4e1f9a444a1a 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -138,6 +138,21 @@ static inline void *alloc_pgtable_page(int nid, gfp_t gfp) return page ? page_address(page) : NULL; } +/* + * This must be called after device probe completes. During probe + * use rlookup_amd_iommu() get the iommu. + */ +static inline struct amd_iommu *get_amd_iommu_from_dev(struct device *dev) +{ + return iommu_get_iommu_dev(dev, struct amd_iommu, iommu); +} + +/* This must be called after device probe completes. */ +static inline struct amd_iommu *get_amd_iommu_from_dev_data(struct iommu_dev_data *dev_data) +{ + return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu); +} + bool translation_pre_enabled(struct amd_iommu *iommu); bool amd_iommu_is_attach_deferred(struct device *dev); int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 743465cb677f..2a5ae7a841cc 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1384,14 +1384,9 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu) static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, size_t size, ioasid_t pasid, bool gn) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct iommu_cmd cmd; - int qdep; - - qdep = dev_data->ats_qdep; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return -EINVAL; + int qdep = dev_data->ats_qdep; build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size, pasid, gn); @@ -1411,16 +1406,12 @@ static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) */ static int device_flush_dte(struct iommu_dev_data *dev_data) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct pci_dev *pdev = NULL; struct amd_iommu_pci_seg *pci_seg; u16 alias; int ret; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return -EINVAL; - if (dev_is_pci(dev_data->dev)) pdev = to_pci_dev(dev_data->dev); @@ -1805,11 +1796,7 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) static void do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct amd_iommu *iommu; - - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); /* Update data structures */ dev_data->domain = domain; @@ -1833,11 +1820,7 @@ static void do_attach(struct iommu_dev_data *dev_data, static void do_detach(struct iommu_dev_data *dev_data) { struct protection_domain *domain = dev_data->domain; - struct amd_iommu *iommu; - - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); /* Update data structures */ dev_data->domain = NULL; @@ -2003,10 +1986,8 @@ static void update_device_table(struct protection_domain *domain) struct iommu_dev_data *dev_data; list_for_each_entry(dev_data, &domain->dev_list, list) { - struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); - if (!iommu) - continue; set_dte_entry(iommu, dev_data); clone_aliases(iommu, dev_data->dev); } @@ -2187,11 +2168,8 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type, struct protection_domain *domain; struct amd_iommu *iommu = NULL; - if (dev) { - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return ERR_PTR(-ENODEV); - } + if (dev) + iommu = get_amd_iommu_from_dev(dev); /* * Since DTE[Mode]=0 is prohibited on SNP-enabled system, @@ -2272,7 +2250,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); struct protection_domain *domain = to_pdomain(dom); - struct amd_iommu *iommu = rlookup_amd_iommu(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; /* @@ -2411,7 +2389,7 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) case IOMMU_CAP_DEFERRED_FLUSH: return true; case IOMMU_CAP_DIRTY_TRACKING: { - struct amd_iommu *iommu = rlookup_amd_iommu(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); return amd_iommu_hd_support(iommu); } @@ -2440,9 +2418,7 @@ static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, } list_for_each_entry(dev_data, &pdomain->dev_list, list) { - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - continue; + iommu = get_amd_iommu_from_dev_data(dev_data); dev_table = get_dev_table(iommu); pte_root = dev_table[dev_data->devid].data[0]; @@ -2502,9 +2478,7 @@ static void amd_iommu_get_resv_regions(struct device *dev, return; devid = PCI_SBDF_TO_DEVID(sbdf); - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return; + iommu = get_amd_iommu_from_dev(dev); pci_seg = iommu->pci_seg; list_for_each_entry(entry, &pci_seg->unity_map, list) { @@ -2838,9 +2812,7 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, struct iommu_cmd cmd; dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = rlookup_amd_iommu(&pdev->dev); - if (!iommu) - return -ENODEV; + iommu = get_amd_iommu_from_dev(&pdev->dev); build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, dev_data->pri_tlp); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1ea2a820e1eb..32bb121e8032 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -654,6 +654,22 @@ static inline struct iommu_device *dev_to_iommu_device(struct device *dev) return (struct iommu_device *)dev_get_drvdata(dev); } +/** + * iommu_get_iommu_dev - Get iommu_device for a device + * @dev: an end-point device + * + * Note that this function must be called from the iommu_ops + * to retrieve the iommu_device for a device, which the core code + * guarentees it will not invoke the op without an attached iommu. + */ +static inline struct iommu_device *__iommu_get_iommu_dev(struct device *dev) +{ + return dev->iommu->iommu_dev; +} + +#define iommu_get_iommu_dev(dev, type, member) \ + container_of(__iommu_get_iommu_dev(dev), type, member) + static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) { *gather = (struct iommu_iotlb_gather) { From fda5108ebafe7797dc8e1279f04e6e9145c9ad10 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:02 +0000 Subject: [PATCH 0223/1038] iommu/amd: Introduce struct protection_domain.pd_mode This enum variable is used to track the type of page table used by the protection domain. It will replace the protection_domain.flags in subsequent series. Suggested-by: Jason Gunthorpe Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-5-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 6 ++++++ drivers/iommu/amd/iommu.c | 2 ++ 2 files changed, 8 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index fdf7807280bd..d29d4a321e57 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -541,6 +541,11 @@ struct amd_io_pgtable { u64 *pgd; /* v2 pgtable pgd pointer */ }; +enum protection_domain_mode { + PD_MODE_V1 = 1, + PD_MODE_V2, +}; + /* * This structure contains generic data for IOMMU protection domains * independent of their use. @@ -556,6 +561,7 @@ struct protection_domain { int nid; /* Node ID */ u64 *gcr3_tbl; /* Guest CR3 table */ unsigned long flags; /* flags to find out type of domain */ + enum protection_domain_mode pd_mode; /* Track page table type */ bool dirty_tracking; /* dirty tracking is enabled in the domain */ unsigned dev_cnt; /* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 2a5ae7a841cc..a3c8f8ea926e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2068,6 +2068,7 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) return -ENOMEM; } + domain->pd_mode = PD_MODE_V1; amd_iommu_domain_set_pgtable(domain, pt_root, mode); return 0; @@ -2076,6 +2077,7 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) static int protection_domain_init_v2(struct protection_domain *domain) { domain->flags |= PD_GIOV_MASK; + domain->pd_mode = PD_MODE_V2; domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; From b7731065523048276a80c8e16120208153438930 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:03 +0000 Subject: [PATCH 0224/1038] iommu/amd: Introduce per-device GCR3 table AMD IOMMU GCR3 table is indexed by PASID. Each entry stores guest CR3 register value, which is an address to the root of guest IO page table. The GCR3 table can be programmed per-device. However, Linux AMD IOMMU driver currently managing the table on a per-domain basis. PASID is a device feature. When SVA is enabled it will bind PASID to device, not domain. Hence it makes sense to have per device GCR3 table. Introduce struct iommu_dev_data.gcr3_tbl_info to keep track of GCR3 table configuration. This will eventually replaces gcr3 related variables in protection_domain structure. Suggested-by: Jason Gunthorpe Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-6-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index d29d4a321e57..aed3e88d23c7 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -533,6 +533,12 @@ struct amd_irte_ops; #define io_pgtable_cfg_to_data(x) \ container_of((x), struct amd_io_pgtable, pgtbl_cfg) +struct gcr3_tbl_info { + u64 *gcr3_tbl; /* Guest CR3 table */ + int glx; /* Number of levels for GCR3 table */ + u32 pasid_cnt; /* Track attached PASIDs */ +}; + struct amd_io_pgtable { struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable iop; @@ -814,6 +820,7 @@ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct llist_node dev_data_list; /* For global dev_data_list */ struct protection_domain *domain; /* Domain the device is bound to */ + struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */ struct device *dev; u16 devid; /* PCI Device ID */ From 7b4e5623d8e4f1475b7deac42035d0129fb36ed0 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:04 +0000 Subject: [PATCH 0225/1038] iommu/amd: Use protection_domain.flags to check page table mode Page table mode (v1, v2 or pt) is per domain property. Recently we have enhanced protection_domain.pd_mode to track per domain page table mode. Use that variable to check the page table mode instead of global 'amd_iommu_pgtable' in {map/unmap}_pages path. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-7-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a3c8f8ea926e..498323930892 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2310,7 +2310,7 @@ static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, int prot = 0; int ret = -EINVAL; - if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + if ((domain->pd_mode == PD_MODE_V1) && (domain->iop.mode == PAGE_MODE_NONE)) return -EINVAL; @@ -2356,7 +2356,7 @@ static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova struct io_pgtable_ops *ops = &domain->iop.iop.ops; size_t r; - if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + if ((domain->pd_mode == PD_MODE_V1) && (domain->iop.mode == PAGE_MODE_NONE)) return 0; From 474bf01ed9f0358b243b264339baafcfbe6c6670 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:05 +0000 Subject: [PATCH 0226/1038] iommu/amd: Add support for device based TLB invalidation Add support to invalidate TLB/IOTLB for the given device. These functions will be used in subsequent patches where we will introduce per device GCR3 table and SVA support. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-8-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 5 +++++ drivers/iommu/amd/iommu.c | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 4e1f9a444a1a..1f4bbc6bf1e5 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -55,6 +55,11 @@ void amd_iommu_domain_update(struct protection_domain *domain); void amd_iommu_domain_flush_complete(struct protection_domain *domain); void amd_iommu_domain_flush_pages(struct protection_domain *domain, u64 address, size_t size); +void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, + ioasid_t pasid, u64 address, size_t size); +void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, + ioasid_t pasid); + int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, unsigned long cr3); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 498323930892..dc8afcefd3cc 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1538,6 +1538,29 @@ static void amd_iommu_domain_flush_all(struct protection_domain *domain) CMD_INV_IOMMU_ALL_PAGES_ADDRESS); } +void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, + ioasid_t pasid, u64 address, size_t size) +{ + struct iommu_cmd cmd; + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); + + build_inv_iommu_pages(&cmd, address, size, + dev_data->domain->id, pasid, true); + iommu_queue_command(iommu, &cmd); + + if (dev_data->ats_enabled) + device_flush_iotlb(dev_data, address, size, pasid, true); + + iommu_completion_wait(iommu); +} + +void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, + ioasid_t pasid) +{ + amd_iommu_dev_flush_pasid_pages(dev_data, 0, + CMD_INV_IOMMU_ALL_PAGES_ADDRESS, pasid); +} + void amd_iommu_domain_flush_complete(struct protection_domain *domain) { int i; From b2e8a7f5d2c3d99285e13cdd330d339d9b040599 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:06 +0000 Subject: [PATCH 0227/1038] iommu/amd: Rearrange GCR3 table setup code Consolidate GCR3 table related code in one place so that its easy to maintain. Note that this patch doesn't move __set_gcr3/__clear_gcr3. We are moving GCR3 table from per domain to per device. Following series will rework these functions. During that time I will move these functions as well. No functional changes intended. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-9-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index dc8afcefd3cc..d8b42b0a2539 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1714,6 +1714,38 @@ static int setup_gcr3_table(struct protection_domain *domain, int pasids) return 0; } +static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) +{ + int index; + u64 *pte; + + while (true) { + + index = (pasid >> (9 * level)) & 0x1ff; + pte = &root[index]; + + if (level == 0) + break; + + if (!(*pte & GCR3_VALID)) { + if (!alloc) + return NULL; + + root = (void *)get_zeroed_page(GFP_ATOMIC); + if (root == NULL) + return NULL; + + *pte = iommu_virt_to_phys(root) | GCR3_VALID; + } + + root = iommu_phys_to_virt(*pte & PAGE_MASK); + + level -= 1; + } + + return pte; +} + static void set_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data) { @@ -2737,38 +2769,6 @@ int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid) return ret; } -static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) -{ - int index; - u64 *pte; - - while (true) { - - index = (pasid >> (9 * level)) & 0x1ff; - pte = &root[index]; - - if (level == 0) - break; - - if (!(*pte & GCR3_VALID)) { - if (!alloc) - return NULL; - - root = (void *)get_zeroed_page(GFP_ATOMIC); - if (root == NULL) - return NULL; - - *pte = iommu_virt_to_phys(root) | GCR3_VALID; - } - - root = iommu_phys_to_virt(*pte & PAGE_MASK); - - level -= 1; - } - - return pte; -} - static int __set_gcr3(struct protection_domain *domain, u32 pasid, unsigned long cr3) { From bf8aff2945ba4091f503df673b9df33002546e6a Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:07 +0000 Subject: [PATCH 0228/1038] iommu: Introduce iommu_group_mutex_assert() Add function to check iommu group mutex lock. So that device drivers can rely on group mutex lock instead of adding another driver level lock before modifying driver specific device data structure. Suggested-by: Jason Gunthorpe Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-10-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 19 +++++++++++++++++++ include/linux/iommu.h | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..8309e634977e 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1248,6 +1248,25 @@ void iommu_group_remove_device(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_group_remove_device); +#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API) +/** + * iommu_group_mutex_assert - Check device group mutex lock + * @dev: the device that has group param set + * + * This function is called by an iommu driver to check whether it holds + * group mutex lock for the given device or not. + * + * Note that this function must be called after device group param is set. + */ +void iommu_group_mutex_assert(struct device *dev) +{ + struct iommu_group *group = dev->iommu_group; + + lockdep_assert_held(&group->mutex); +} +EXPORT_SYMBOL_GPL(iommu_group_mutex_assert); +#endif + static struct device *iommu_group_first_dev(struct iommu_group *group) { lockdep_assert_held(&group->mutex); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 32bb121e8032..8141a37556d5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1356,6 +1356,14 @@ static inline ioasid_t iommu_alloc_global_pasid(struct device *dev) static inline void iommu_free_global_pasid(ioasid_t pasid) {} #endif /* CONFIG_IOMMU_API */ +#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API) +void iommu_group_mutex_assert(struct device *dev); +#else +static inline void iommu_group_mutex_assert(struct device *dev) +{ +} +#endif + /** * iommu_map_sgtable - Map the given buffer to the IOMMU domain * @domain: The IOMMU domain to perform the mapping From e8e1aac33458d80545e5ad37b2a3e4243eee278e Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:08 +0000 Subject: [PATCH 0229/1038] iommu/amd: Refactor helper function for setting / clearing GCR3 Refactor GCR3 helper functions in preparation to use per device GCR3 table. * Add new function update_gcr3 to update per device GCR3 table * Remove per domain default GCR3 setup during v2 page table allocation. Subsequent patch will add support to setup default gcr3 while attaching device to domain. * Remove amd_iommu_domain_update() from V2 page table path as device detach path will take care of updating the domain. * Consolidate GCR3 table related code in one place so that its easy to maintain. * Rename functions to reflect its usage. Signed-off-by: Suravee Suthikulpanit Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-11-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 8 ++- drivers/iommu/amd/io_pgtable_v2.c | 21 +----- drivers/iommu/amd/iommu.c | 115 ++++++++++++++---------------- 3 files changed, 61 insertions(+), 83 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 1f4bbc6bf1e5..6e03ff2aaebe 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -44,6 +44,11 @@ bool amd_iommu_v2_supported(void); int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev); +/* GCR3 setup */ +int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, + ioasid_t pasid, unsigned long gcr3); +int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); + int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address); /* * This function flushes all internal caches of @@ -61,9 +66,6 @@ void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, ioasid_t pasid); int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); -int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, - unsigned long cr3); -int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid); #ifdef CONFIG_IRQ_REMAP int amd_iommu_create_irq_domain(struct amd_iommu *iommu); diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index 6d69ba60744f..93489d2db4e8 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -350,38 +350,26 @@ static const struct iommu_flush_ops v2_flush_ops = { static void v2_free_pgtable(struct io_pgtable *iop) { - struct protection_domain *pdom; struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); - pdom = container_of(pgtable, struct protection_domain, iop); - if (!(pdom->flags & PD_IOMMUV2_MASK)) + if (!pgtable || !pgtable->pgd) return; - /* Clear gcr3 entry */ - amd_iommu_domain_clear_gcr3(&pdom->domain, 0); - - /* Make changes visible to IOMMUs */ - amd_iommu_domain_update(pdom); - /* Free page table */ free_pgtable(pgtable->pgd, get_pgtable_level()); + pgtable->pgd = NULL; } static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); struct protection_domain *pdom = (struct protection_domain *)cookie; - int ret; int ias = IOMMU_IN_ADDR_BIT_SIZE; pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC); if (!pgtable->pgd) return NULL; - ret = amd_iommu_domain_set_gcr3(&pdom->domain, 0, iommu_virt_to_phys(pgtable->pgd)); - if (ret) - goto err_free_pgd; - if (get_pgtable_level() == PAGE_MODE_5_LEVEL) ias = 57; @@ -395,11 +383,6 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo cfg->tlb = &v2_flush_ops; return &pgtable->iop; - -err_free_pgd: - free_pgtable_page(pgtable->pgd); - - return NULL; } struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = { diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index d8b42b0a2539..4fda158e043f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1714,10 +1714,13 @@ static int setup_gcr3_table(struct protection_domain *domain, int pasids) return 0; } -static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) +static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info, + ioasid_t pasid, bool alloc) { int index; u64 *pte; + u64 *root = gcr3_info->gcr3_tbl; + int level = gcr3_info->glx; while (true) { @@ -1746,6 +1749,56 @@ static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) return pte; } +static int update_gcr3(struct iommu_dev_data *dev_data, + ioasid_t pasid, unsigned long gcr3, bool set) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + u64 *pte; + + pte = __get_gcr3_pte(gcr3_info, pasid, true); + if (pte == NULL) + return -ENOMEM; + + if (set) + *pte = (gcr3 & PAGE_MASK) | GCR3_VALID; + else + *pte = 0; + + amd_iommu_dev_flush_pasid_all(dev_data, pasid); + return 0; +} + +int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, + unsigned long gcr3) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + int ret; + + iommu_group_mutex_assert(dev_data->dev); + + ret = update_gcr3(dev_data, pasid, gcr3, true); + if (ret) + return ret; + + gcr3_info->pasid_cnt++; + return ret; +} + +int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + int ret; + + iommu_group_mutex_assert(dev_data->dev); + + ret = update_gcr3(dev_data, pasid, 0, false); + if (ret) + return ret; + + gcr3_info->pasid_cnt--; + return ret; +} + static void set_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data) { @@ -2769,66 +2822,6 @@ int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid) return ret; } -static int __set_gcr3(struct protection_domain *domain, u32 pasid, - unsigned long cr3) -{ - u64 *pte; - - if (domain->iop.mode != PAGE_MODE_NONE) - return -EINVAL; - - pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true); - if (pte == NULL) - return -ENOMEM; - - *pte = (cr3 & PAGE_MASK) | GCR3_VALID; - - return __amd_iommu_flush_tlb(domain, pasid); -} - -static int __clear_gcr3(struct protection_domain *domain, u32 pasid) -{ - u64 *pte; - - if (domain->iop.mode != PAGE_MODE_NONE) - return -EINVAL; - - pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false); - if (pte == NULL) - return 0; - - *pte = 0; - - return __amd_iommu_flush_tlb(domain, pasid); -} - -int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, - unsigned long cr3) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __set_gcr3(domain, pasid, cr3); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __clear_gcr3(domain, pasid); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, int status, int tag) { From 4ebd4c7f2501cc5876717e0f2382d5b825e59405 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:09 +0000 Subject: [PATCH 0230/1038] iommu/amd: Refactor attaching / detaching device functions If domain is configured with V2 page table then setup default GCR3 with domain GCR3 pointer. So that all devices in the domain uses same page table for translation. Also return page table setup status from do_attach() function. Signed-off-by: Suravee Suthikulpanit Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-12-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4fda158e043f..929a0e8d41d0 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1901,10 +1901,11 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) amd_iommu_apply_erratum_63(iommu, devid); } -static void do_attach(struct iommu_dev_data *dev_data, - struct protection_domain *domain) +static int do_attach(struct iommu_dev_data *dev_data, + struct protection_domain *domain) { struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + int ret = 0; /* Update data structures */ dev_data->domain = domain; @@ -1918,11 +1919,28 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; + /* Init GCR3 table and update device table */ + if (domain->pd_mode == PD_MODE_V2) { + /* By default, setup GCR3 table to support single PASID */ + ret = setup_gcr3_table(dev_data->domain, 1); + if (ret) + return ret; + + ret = update_gcr3(dev_data, 0, + iommu_virt_to_phys(domain->iop.pgd), true); + if (ret) { + free_gcr3_table(dev_data->domain); + return ret; + } + } + /* Update device table */ set_dte_entry(iommu, dev_data); clone_aliases(iommu, dev_data->dev); device_flush_dte(dev_data); + + return ret; } static void do_detach(struct iommu_dev_data *dev_data) @@ -1930,6 +1948,12 @@ static void do_detach(struct iommu_dev_data *dev_data) struct protection_domain *domain = dev_data->domain; struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + /* Clear GCR3 table */ + if (domain->pd_mode == PD_MODE_V2) { + update_gcr3(dev_data, 0, 0, false); + free_gcr3_table(dev_data->domain); + } + /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); @@ -1972,7 +1996,7 @@ static int attach_device(struct device *dev, if (dev_is_pci(dev)) pdev_enable_caps(to_pci_dev(dev)); - do_attach(dev_data, domain); + ret = do_attach(dev_data, domain); out: spin_unlock(&dev_data->lock); From fb575d17813f6aff2e8b7e051dac8def2c291bff Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:10 +0000 Subject: [PATCH 0231/1038] iommu/amd: Refactor protection_domain helper functions To removes the code to setup GCR3 table, and only handle domain create / destroy, since GCR3 is no longer part of a domain. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-13-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 929a0e8d41d0..e2f546926b6f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2176,9 +2176,6 @@ static void protection_domain_free(struct protection_domain *domain) if (domain->iop.pgtbl_cfg.tlb) free_io_pgtable_ops(&domain->iop.iop.ops); - if (domain->flags & PD_IOMMUV2_MASK) - free_gcr3_table(domain); - if (domain->iop.root) free_page((unsigned long)domain->iop.root); @@ -2206,15 +2203,10 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) return 0; } -static int protection_domain_init_v2(struct protection_domain *domain) +static int protection_domain_init_v2(struct protection_domain *pdom) { - domain->flags |= PD_GIOV_MASK; - domain->pd_mode = PD_MODE_V2; - - domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; - - if (setup_gcr3_table(domain, 1)) - return -ENOMEM; + pdom->pd_mode = PD_MODE_V2; + pdom->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; return 0; } From cf70873e3d01653df69115576501f8f7f6f2b203 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:11 +0000 Subject: [PATCH 0232/1038] iommu/amd: Refactor GCR3 table helper functions To use the new per-device struct gcr3_tbl_info. Use GFP_KERNEL flag instead of GFP_ATOMIC for GCR3 table allocation. Also modify set_dte_entry() to use new per device GCR3 table. Also in free_gcr3_table() path replace BUG_ON with WARN_ON_ONCE(). Signed-off-by: Suravee Suthikulpanit Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-14-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 53 +++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e2f546926b6f..b742983007f2 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -75,6 +75,9 @@ struct kmem_cache *amd_iommu_irq_cache; static void detach_device(struct device *dev); +static void set_dte_entry(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data); + /**************************************************************************** * * Helper functions @@ -1666,16 +1669,19 @@ static void free_gcr3_tbl_level2(u64 *tbl) } } -static void free_gcr3_table(struct protection_domain *domain) +static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) { - if (domain->glx == 2) - free_gcr3_tbl_level2(domain->gcr3_tbl); - else if (domain->glx == 1) - free_gcr3_tbl_level1(domain->gcr3_tbl); + if (gcr3_info->glx == 2) + free_gcr3_tbl_level2(gcr3_info->gcr3_tbl); + else if (gcr3_info->glx == 1) + free_gcr3_tbl_level1(gcr3_info->gcr3_tbl); else - BUG_ON(domain->glx != 0); + WARN_ON_ONCE(gcr3_info->glx != 0); - free_page((unsigned long)domain->gcr3_tbl); + gcr3_info->glx = 0; + + free_page((unsigned long)gcr3_info->gcr3_tbl); + gcr3_info->gcr3_tbl = NULL; } /* @@ -1694,22 +1700,23 @@ static int get_gcr3_levels(int pasids) return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels; } -/* Note: This function expects iommu_domain->lock to be held prior calling the function. */ -static int setup_gcr3_table(struct protection_domain *domain, int pasids) +static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, + struct amd_iommu *iommu, int pasids) { int levels = get_gcr3_levels(pasids); + int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; if (levels > amd_iommu_max_glx_val) return -EINVAL; - domain->gcr3_tbl = alloc_pgtable_page(domain->nid, GFP_ATOMIC); - if (domain->gcr3_tbl == NULL) + if (gcr3_info->gcr3_tbl) + return -EBUSY; + + gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_KERNEL); + if (gcr3_info->gcr3_tbl == NULL) return -ENOMEM; - domain->glx = levels; - domain->flags |= PD_IOMMUV2_MASK; - - amd_iommu_domain_update(domain); + gcr3_info->glx = levels; return 0; } @@ -1808,6 +1815,7 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid = dev_data->devid; struct protection_domain *domain = dev_data->domain; struct dev_table_entry *dev_table = get_dev_table(iommu); + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; if (domain->iop.mode != PAGE_MODE_NONE) pte_root = iommu_virt_to_phys(domain->iop.root); @@ -1835,9 +1843,9 @@ static void set_dte_entry(struct amd_iommu *iommu, if (domain->dirty_tracking) pte_root |= DTE_FLAG_HAD; - if (domain->flags & PD_IOMMUV2_MASK) { - u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl); - u64 glx = domain->glx; + if (gcr3_info && gcr3_info->gcr3_tbl) { + u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl); + u64 glx = gcr3_info->glx; u64 tmp; pte_root |= DTE_FLAG_GV; @@ -1865,7 +1873,8 @@ static void set_dte_entry(struct amd_iommu *iommu, ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT); } - if (domain->flags & PD_GIOV_MASK) + /* GIOV is supported with V2 page table mode only */ + if (pdom_is_v2_pgtbl_mode(domain)) pte_root |= DTE_FLAG_GIOV; } @@ -1922,14 +1931,14 @@ static int do_attach(struct iommu_dev_data *dev_data, /* Init GCR3 table and update device table */ if (domain->pd_mode == PD_MODE_V2) { /* By default, setup GCR3 table to support single PASID */ - ret = setup_gcr3_table(dev_data->domain, 1); + ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 1); if (ret) return ret; ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(domain->iop.pgd), true); if (ret) { - free_gcr3_table(dev_data->domain); + free_gcr3_table(&dev_data->gcr3_info); return ret; } } @@ -1951,7 +1960,7 @@ static void do_detach(struct iommu_dev_data *dev_data) /* Clear GCR3 table */ if (domain->pd_mode == PD_MODE_V2) { update_gcr3(dev_data, 0, 0, false); - free_gcr3_table(dev_data->domain); + free_gcr3_table(&dev_data->gcr3_info); } /* Update data structures */ From 02b990253db7cbdc3ef0631e9c202a69512a14c4 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:12 +0000 Subject: [PATCH 0233/1038] iommu/amd: Remove unused flush pasid functions We have removed iommu_v2 module and converted v2 page table to use common flush functions. Also we have moved GCR3 table to per device. PASID related functions are not used. Hence remove these unused functions. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-15-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 3 - drivers/iommu/amd/iommu.c | 100 ---------------------------------- 2 files changed, 103 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 6e03ff2aaebe..bea910d9ace5 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -49,7 +49,6 @@ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsigned long gcr3); int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); -int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address); /* * This function flushes all internal caches of * the IOMMU used by this driver. @@ -65,8 +64,6 @@ void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, ioasid_t pasid); -int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); - #ifdef CONFIG_IRQ_REMAP int amd_iommu_create_irq_domain(struct amd_iommu *iommu); #else diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index b742983007f2..4384d8304bea 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2747,106 +2747,6 @@ const struct iommu_ops amd_iommu_ops = { } }; -static int __flush_pasid(struct protection_domain *domain, u32 pasid, - u64 address, size_t size) -{ - struct iommu_dev_data *dev_data; - struct iommu_cmd cmd; - int i, ret; - - if (!(domain->flags & PD_IOMMUV2_MASK)) - return -EINVAL; - - build_inv_iommu_pages(&cmd, address, size, domain->id, pasid, true); - - /* - * IOMMU TLB needs to be flushed before Device TLB to - * prevent device TLB refill from IOMMU TLB - */ - for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { - if (domain->dev_iommu[i] == 0) - continue; - - ret = iommu_queue_command(amd_iommus[i], &cmd); - if (ret != 0) - goto out; - } - - /* Wait until IOMMU TLB flushes are complete */ - amd_iommu_domain_flush_complete(domain); - - /* Now flush device TLBs */ - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct amd_iommu *iommu; - int qdep; - - /* - There might be non-IOMMUv2 capable devices in an IOMMUv2 - * domain. - */ - if (!dev_data->ats_enabled) - continue; - - qdep = dev_data->ats_qdep; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - continue; - build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, - address, size, pasid, true); - - ret = iommu_queue_command(iommu, &cmd); - if (ret != 0) - goto out; - } - - /* Wait until all device TLBs are flushed */ - amd_iommu_domain_flush_complete(domain); - - ret = 0; - -out: - - return ret; -} - -static int __amd_iommu_flush_page(struct protection_domain *domain, u32 pasid, - u64 address) -{ - return __flush_pasid(domain, pasid, address, PAGE_SIZE); -} - -int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, - u64 address) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __amd_iommu_flush_page(domain, pasid, address); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid) -{ - return __flush_pasid(domain, pasid, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS); -} - -int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __amd_iommu_flush_tlb(domain, pasid); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, int status, int tag) { From a7b2aff3132562bd26657d9a707f8fa82967147f Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:13 +0000 Subject: [PATCH 0234/1038] iommu/amd: Rearrange device flush code Consolidate all flush related code in one place so that its easy to maintain. No functional changes intended. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-16-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 92 ++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4384d8304bea..d3ea4de9033a 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1605,6 +1605,49 @@ static void domain_flush_devices(struct protection_domain *domain) device_flush_dte(dev_data); } +static void update_device_table(struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data; + + list_for_each_entry(dev_data, &domain->dev_list, list) { + struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); + + set_dte_entry(iommu, dev_data); + clone_aliases(iommu, dev_data->dev); + } +} + +void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) +{ + update_device_table(domain); + domain_flush_devices(domain); +} + +void amd_iommu_domain_update(struct protection_domain *domain) +{ + /* Update device table */ + amd_iommu_update_and_flush_device_table(domain); + + /* Flush domain TLB(s) and wait for completion */ + amd_iommu_domain_flush_all(domain); +} + +int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, + int status, int tag) +{ + struct iommu_dev_data *dev_data; + struct amd_iommu *iommu; + struct iommu_cmd cmd; + + dev_data = dev_iommu_priv_get(&pdev->dev); + iommu = get_amd_iommu_from_dev(&pdev->dev); + + build_complete_ppr(&cmd, dev_data->devid, pasid, status, + tag, dev_data->pri_tlp); + + return iommu_queue_command(iommu, &cmd); +} + /**************************************************************************** * * The next functions belong to the domain allocation. A domain is @@ -2116,39 +2159,6 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev) return acpihid_device_group(dev); } -/***************************************************************************** - * - * The next functions belong to the dma_ops mapping/unmapping code. - * - *****************************************************************************/ - -static void update_device_table(struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data; - - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); - - set_dte_entry(iommu, dev_data); - clone_aliases(iommu, dev_data->dev); - } -} - -void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) -{ - update_device_table(domain); - domain_flush_devices(domain); -} - -void amd_iommu_domain_update(struct protection_domain *domain) -{ - /* Update device table */ - amd_iommu_update_and_flush_device_table(domain); - - /* Flush domain TLB(s) and wait for completion */ - amd_iommu_domain_flush_all(domain); -} - /***************************************************************************** * * The following functions belong to the exported interface of AMD IOMMU @@ -2747,22 +2757,6 @@ const struct iommu_ops amd_iommu_ops = { } }; -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag) -{ - struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; - struct iommu_cmd cmd; - - dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = get_amd_iommu_from_dev(&pdev->dev); - - build_complete_ppr(&cmd, dev_data->devid, pasid, status, - tag, dev_data->pri_tlp); - - return iommu_queue_command(iommu, &cmd); -} - #ifdef CONFIG_IRQ_REMAP /***************************************************************************** From c2a6af5e08c27350cb007db4cfeeb413017888b0 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2024 11:56:14 +0000 Subject: [PATCH 0235/1038] iommu/amd: Remove unused GCR3 table parameters from struct protection_domain Since they are moved to struct iommu_dev_data, and the driver has been ported to use them. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-17-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 12 ------------ drivers/iommu/amd/iommu.c | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index aed3e88d23c7..43f0f1b13f2d 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -453,15 +453,6 @@ #define MAX_DOMAIN_ID 65536 -/* Protection domain flags */ -#define PD_DMA_OPS_MASK BIT(0) /* domain used for dma_ops */ -#define PD_DEFAULT_MASK BIT(1) /* domain is a default dma_ops - domain for an IOMMU */ -#define PD_PASSTHROUGH_MASK BIT(2) /* domain has no page - translation */ -#define PD_IOMMUV2_MASK BIT(3) /* domain has gcr3 table */ -#define PD_GIOV_MASK BIT(4) /* domain enable GIOV support */ - /* Timeout stuff */ #define LOOP_TIMEOUT 100000 #define MMIO_STATUS_TIMEOUT 2000000 @@ -563,10 +554,7 @@ struct protection_domain { struct amd_io_pgtable iop; spinlock_t lock; /* mostly used to lock the page table*/ u16 id; /* the domain id written to the device table */ - int glx; /* Number of levels for GCR3 table */ int nid; /* Node ID */ - u64 *gcr3_tbl; /* Guest CR3 table */ - unsigned long flags; /* flags to find out type of domain */ enum protection_domain_mode pd_mode; /* Track page table type */ bool dirty_tracking; /* dirty tracking is enabled in the domain */ unsigned dev_cnt; /* devices assigned to this domain */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index d3ea4de9033a..ab6af861a3f1 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -86,7 +86,7 @@ static void set_dte_entry(struct amd_iommu *iommu, static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) { - return (pdom && (pdom->flags & PD_IOMMUV2_MASK)); + return (pdom && (pdom->pd_mode == PD_MODE_V2)); } static inline int get_acpihid_device_id(struct device *dev, From 87a6f1f22c9781b1b37847caf65e070bfa6dadb5 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 5 Feb 2024 11:56:15 +0000 Subject: [PATCH 0236/1038] iommu/amd: Introduce per-device domain ID to fix potential TLB aliasing issue With v1 page table, the AMD IOMMU spec states that the hardware must use the domain ID to tag its internal translation caches. I/O devices with different v1 page tables must be given different domain IDs. I/O devices that share the same v1 page table __may__ be given the same domain ID. This domain ID management policy is currently implemented by the AMD IOMMU driver. In this case, only the domain ID is needed when issuing the INVALIDATE_IOMMU_PAGES command to invalidate the IOMMU translation cache (TLB). With v2 page table, the hardware uses domain ID and PASID as parameters to tag and issue the INVALIDATE_IOMMU_PAGES command. Since the GCR3 table is setup per-device, and there is no guarantee for PASID to be unique across multiple devices. The same PASID for different devices could have different v2 page tables. In such case, if multiple devices share the same domain ID, IOMMU translation cache for these devices would be polluted due to TLB aliasing. Hence, avoid the TLB aliasing issue with v2 page table by allocating unique domain ID for each device even when multiple devices are sharing the same v1 page table. Please note that this fix would result in multiple INVALIDATE_IOMMU_PAGES commands (one per domain id) when unmapping a translation. Domain ID can be shared until device starts using PASID. We will enhance this code later where we will allocate per device domain ID only when its needed. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240205115615.6053-18-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 1 + drivers/iommu/amd/iommu.c | 80 +++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 43f0f1b13f2d..d1fed5fc219b 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -528,6 +528,7 @@ struct gcr3_tbl_info { u64 *gcr3_tbl; /* Guest CR3 table */ int glx; /* Number of levels for GCR3 table */ u32 pasid_cnt; /* Track attached PASIDs */ + u16 domid; /* Per device domain ID */ }; struct amd_io_pgtable { diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index ab6af861a3f1..f688443c2764 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1443,27 +1443,37 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) return ret; } -/* - * TLB invalidation function which is called from the mapping functions. - * It invalidates a single PTE if the range to flush is within a single - * page. Otherwise it flushes the whole TLB of the IOMMU. - */ -static void __domain_flush_pages(struct protection_domain *domain, +static int domain_flush_pages_v2(struct protection_domain *pdom, u64 address, size_t size) { struct iommu_dev_data *dev_data; + struct iommu_cmd cmd; + int ret = 0; + + list_for_each_entry(dev_data, &pdom->dev_list, list) { + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); + u16 domid = dev_data->gcr3_info.domid; + + build_inv_iommu_pages(&cmd, address, size, + domid, IOMMU_NO_PASID, true); + + ret |= iommu_queue_command(iommu, &cmd); + } + + return ret; +} + +static int domain_flush_pages_v1(struct protection_domain *pdom, + u64 address, size_t size) +{ struct iommu_cmd cmd; int ret = 0, i; - ioasid_t pasid = IOMMU_NO_PASID; - bool gn = false; - if (pdom_is_v2_pgtbl_mode(domain)) - gn = true; - - build_inv_iommu_pages(&cmd, address, size, domain->id, pasid, gn); + build_inv_iommu_pages(&cmd, address, size, + pdom->id, IOMMU_NO_PASID, false); for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { - if (!domain->dev_iommu[i]) + if (!pdom->dev_iommu[i]) continue; /* @@ -1473,6 +1483,28 @@ static void __domain_flush_pages(struct protection_domain *domain, ret |= iommu_queue_command(amd_iommus[i], &cmd); } + return ret; +} + +/* + * TLB invalidation function which is called from the mapping functions. + * It flushes range of PTEs of the domain. + */ +static void __domain_flush_pages(struct protection_domain *domain, + u64 address, size_t size) +{ + struct iommu_dev_data *dev_data; + int ret = 0; + ioasid_t pasid = IOMMU_NO_PASID; + bool gn = false; + + if (pdom_is_v2_pgtbl_mode(domain)) { + gn = true; + ret = domain_flush_pages_v2(domain, address, size); + } else { + ret = domain_flush_pages_v1(domain, address, size); + } + list_for_each_entry(dev_data, &domain->dev_list, list) { if (!dev_data->ats_enabled) @@ -1548,7 +1580,7 @@ void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); build_inv_iommu_pages(&cmd, address, size, - dev_data->domain->id, pasid, true); + dev_data->gcr3_info.domid, pasid, true); iommu_queue_command(iommu, &cmd); if (dev_data->ats_enabled) @@ -1723,6 +1755,9 @@ static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) gcr3_info->glx = 0; + /* Free per device domain ID */ + domain_id_free(gcr3_info->domid); + free_page((unsigned long)gcr3_info->gcr3_tbl); gcr3_info->gcr3_tbl = NULL; } @@ -1755,9 +1790,14 @@ static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, if (gcr3_info->gcr3_tbl) return -EBUSY; + /* Allocate per device domain ID */ + gcr3_info->domid = domain_id_alloc(); + gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_KERNEL); - if (gcr3_info->gcr3_tbl == NULL) + if (gcr3_info->gcr3_tbl == NULL) { + domain_id_free(gcr3_info->domid); return -ENOMEM; + } gcr3_info->glx = levels; @@ -1856,10 +1896,16 @@ static void set_dte_entry(struct amd_iommu *iommu, u64 flags = 0; u32 old_domid; u16 devid = dev_data->devid; + u16 domid; struct protection_domain *domain = dev_data->domain; struct dev_table_entry *dev_table = get_dev_table(iommu); struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + if (gcr3_info && gcr3_info->gcr3_tbl) + domid = dev_data->gcr3_info.domid; + else + domid = domain->id; + if (domain->iop.mode != PAGE_MODE_NONE) pte_root = iommu_virt_to_phys(domain->iop.root); @@ -1872,7 +1918,7 @@ static void set_dte_entry(struct amd_iommu *iommu, * When SNP is enabled, Only set TV bit when IOMMU * page translation is in use. */ - if (!amd_iommu_snp_en || (domain->id != 0)) + if (!amd_iommu_snp_en || (domid != 0)) pte_root |= DTE_FLAG_TV; flags = dev_table[devid].data[1]; @@ -1922,7 +1968,7 @@ static void set_dte_entry(struct amd_iommu *iommu, } flags &= ~DEV_DOMID_MASK; - flags |= domain->id; + flags |= domid; old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK; dev_table[devid].data[1] = flags; From fc5b764bef24d0cf722deb5c1a44948cd17d4afe Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 9 Feb 2024 18:54:23 +0200 Subject: [PATCH 0237/1038] spi: gpio: Follow renaming of SPI "master" to "controller" In commit 8caab75fd2c2 ("spi: Generalize SPI "master" to "controller"") some functions and struct members were renamed. Recent work by Uwe completes this renaming. However, there are plenty of leftovers in the comments and in-code documentation. Update them as well. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240209165423.2305493-1-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- include/linux/spi/spi_gpio.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/spi/spi_gpio.h b/include/linux/spi/spi_gpio.h index 9e7e83d8645b..5f0e1407917a 100644 --- a/include/linux/spi/spi_gpio.h +++ b/include/linux/spi/spi_gpio.h @@ -15,8 +15,8 @@ */ /** - * struct spi_gpio_platform_data - parameter for bitbanged SPI master - * @num_chipselect: how many slaves to allow + * struct spi_gpio_platform_data - parameter for bitbanged SPI host controller + * @num_chipselect: how many target devices to allow */ struct spi_gpio_platform_data { u16 num_chipselect; From b9f55084aa0962af6247fa971f0b1e9c7aa676ef Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 17 Jan 2024 13:44:05 +0200 Subject: [PATCH 0238/1038] hwmon: put HWMON_CHANNEL_INFO() initializers in rodata HWMON_CHANNEL_INFO() is supposed to be used as initializer for arrays of const struct hwmon_channel_info *. However, without explicit const, HWMON_CHANNEL_INFO() creates mutable compound literals, and the const pointers point at the mutable data. Add const to place the data in rodata. Cc: Jean Delvare Cc: Guenter Roeck Signed-off-by: Jani Nikula Link: https://lore.kernel.org/r/20240117114405.1506775-1-jani.nikula@intel.com Signed-off-by: Guenter Roeck --- include/linux/hwmon.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 8cd6a6b33593..c2c0da18dfa3 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -425,12 +425,12 @@ struct hwmon_channel_info { const u32 *config; }; -#define HWMON_CHANNEL_INFO(stype, ...) \ - (&(struct hwmon_channel_info) { \ - .type = hwmon_##stype, \ - .config = (u32 []) { \ - __VA_ARGS__, 0 \ - } \ +#define HWMON_CHANNEL_INFO(stype, ...) \ + (&(const struct hwmon_channel_info) { \ + .type = hwmon_##stype, \ + .config = (const u32 []) { \ + __VA_ARGS__, 0 \ + } \ }) /** From 7fce84a991cdc1b28e03d14c997ec65187df6da5 Mon Sep 17 00:00:00 2001 From: Forest Crossman Date: Sat, 13 Jan 2024 16:27:55 -0800 Subject: [PATCH 0239/1038] hwmon: (nct6683) Add another customer ID for MSI This value was found on an MSI PRO X670-P WIFI with an NCT6687D chip. Signed-off-by: Forest Crossman Link: https://lore.kernel.org/r/CAO3ALPwot01+bBisj7Roog7SD9UwV+y4NtiakKrBDE0tPvFhbw@mail.gmail.com Signed-off-by: Guenter Roeck --- Documentation/hwmon/nct6683.rst | 1 + drivers/hwmon/nct6683.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 3e7f6ee779c2..2a7a78eb1b46 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -64,4 +64,5 @@ Intel DB85FL NCT6683D EC firmware version 1.0 build 04/03/13 ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 ASRock X670E NCT6686D EC firmware version 1.0 build 05/19/22 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 +MSI X670-P NCT6687D EC firmware version 0.0 build 09/27/22 =============== =============================================== diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c index 3f3f7a88413e..0d016fedb9c2 100644 --- a/drivers/hwmon/nct6683.c +++ b/drivers/hwmon/nct6683.c @@ -174,6 +174,7 @@ superio_exit(int ioreg) #define NCT6683_CUSTOMER_ID_MITAC 0xa0e #define NCT6683_CUSTOMER_ID_MSI 0x201 #define NCT6683_CUSTOMER_ID_MSI2 0x200 +#define NCT6683_CUSTOMER_ID_MSI3 0x207 #define NCT6683_CUSTOMER_ID_ASROCK 0xe2c #define NCT6683_CUSTOMER_ID_ASROCK2 0xe1b #define NCT6683_CUSTOMER_ID_ASROCK3 0x1631 @@ -1224,6 +1225,8 @@ static int nct6683_probe(struct platform_device *pdev) break; case NCT6683_CUSTOMER_ID_MSI2: break; + case NCT6683_CUSTOMER_ID_MSI3: + break; case NCT6683_CUSTOMER_ID_ASROCK: break; case NCT6683_CUSTOMER_ID_ASROCK2: From eea32fafadd3e648537e2653a2e2291431c990ff Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Wed, 20 Dec 2023 14:12:13 +0100 Subject: [PATCH 0240/1038] dt-bindings: hwmon: ina2xx: Add label property Add a label property to allow a custom name to be used for identifying a device on the board. This is useful when multiple devices are present on the same board. Similar change was done by commit ffae65fb1ae4 ("dt-bindings: spi: spi-cadence: Add label property"). Signed-off-by: Michal Simek Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/6f3c57d08984c1978569d3918cb38eb295c0c67d.1703077926.git.michal.simek@amd.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml index 378d1f6aeeb3..8e5c1935b5f4 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml @@ -32,6 +32,9 @@ properties: reg: maxItems: 1 + label: + description: A descriptive name for this device. + shunt-resistor: description: Shunt resistor value in micro-Ohm. @@ -77,6 +80,7 @@ examples: power-sensor@44 { compatible = "ti,ina220"; reg = <0x44>; + label = "vdd_3v0"; shunt-resistor = <1000>; vs-supply = <&vdd_3v0>; }; From f29996d0295ec581d1e7cf5e72894656dc7f9f57 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Wed, 20 Dec 2023 14:12:14 +0100 Subject: [PATCH 0241/1038] dt-bindings: hwmon: ina2xx: Describe #io-channel-cells property There are two drivers in the Linux kernel. One is hwmon based and second IIO. IIO version requires to define #io-channel-cells to operate. Signed-off-by: Michal Simek Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/aa303b9fe3116e7f98d6b72822f7f57694366db3.1703077926.git.michal.simek@amd.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml index 8e5c1935b5f4..f324b627bf9c 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml @@ -32,6 +32,9 @@ properties: reg: maxItems: 1 + "#io-channel-cells": + const: 1 + label: description: A descriptive name for this device. @@ -80,6 +83,7 @@ examples: power-sensor@44 { compatible = "ti,ina220"; reg = <0x44>; + #io-channel-cells = <1>; label = "vdd_3v0"; shunt-resistor = <1000>; vs-supply = <&vdd_3v0>; From f7ab2d180e6a312b885d1aadb8ca6ea0933a55df Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Mon, 8 Jan 2024 15:30:51 +0100 Subject: [PATCH 0242/1038] dt-bindings: hwmon: ina2xx: Describe ina260 chip Describe ina260 chip which is precision digital current and power monitor with precision integrated shunt resistor. Signed-off-by: Michal Simek Acked-by: Conor Dooley Link: https://lore.kernel.org/r/4c82dc4d412e91d1601c1da5bca1cdf1a91cd9b8.1704724242.git.michal.simek@amd.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml index f324b627bf9c..a099bb71415e 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml @@ -28,6 +28,7 @@ properties: - ti,ina231 - ti,ina237 - ti,ina238 + - ti,ina260 reg: maxItems: 1 From e7b631d798716bba5cf1499caf62c548c7905439 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 27 Jan 2024 16:02:54 +0100 Subject: [PATCH 0243/1038] hwmon: Remove I2C_CLASS_HWMON from drivers w/o detect() and address_list Class-based I2C probing requires detect() and address_list to be set in the I2C client driver, see checks in i2c_detect(). It's misleading to declare I2C_CLASS_HWMON support if this precondition isn't met. Signed-off-by: Heiner Kallweit Link: https://lore.kernel.org/r/75747c6a-d414-4b07-8f66-5a5cdddc3c36@gmail.com Signed-off-by: Guenter Roeck --- drivers/hwmon/adm1177.c | 1 - drivers/hwmon/ds1621.c | 1 - drivers/hwmon/ds620.c | 1 - drivers/hwmon/ina209.c | 1 - drivers/hwmon/ina238.c | 1 - drivers/hwmon/max127.c | 1 - drivers/hwmon/max31760.c | 1 - drivers/hwmon/max31790.c | 1 - drivers/hwmon/max31827.c | 1 - drivers/hwmon/max6621.c | 1 - drivers/hwmon/max6697.c | 1 - drivers/hwmon/occ/p8_i2c.c | 1 - drivers/hwmon/pmbus/ir36021.c | 1 - drivers/hwmon/powr1220.c | 1 - drivers/hwmon/sbrmi.c | 1 - drivers/hwmon/sbtsi_temp.c | 1 - drivers/hwmon/w83773g.c | 1 - 17 files changed, 17 deletions(-) diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c index 60a893f27159..3390102d2d4a 100644 --- a/drivers/hwmon/adm1177.c +++ b/drivers/hwmon/adm1177.c @@ -250,7 +250,6 @@ static const struct of_device_id adm1177_dt_ids[] = { MODULE_DEVICE_TABLE(of, adm1177_dt_ids); static struct i2c_driver adm1177_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "adm1177", .of_match_table = adm1177_dt_ids, diff --git a/drivers/hwmon/ds1621.c b/drivers/hwmon/ds1621.c index 21b635046521..bffbc8040171 100644 --- a/drivers/hwmon/ds1621.c +++ b/drivers/hwmon/ds1621.c @@ -380,7 +380,6 @@ MODULE_DEVICE_TABLE(i2c, ds1621_id); /* This is the driver that will be inserted */ static struct i2c_driver ds1621_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ds1621", }, diff --git a/drivers/hwmon/ds620.c b/drivers/hwmon/ds620.c index 2b09536630cb..4fc4df012fac 100644 --- a/drivers/hwmon/ds620.c +++ b/drivers/hwmon/ds620.c @@ -241,7 +241,6 @@ MODULE_DEVICE_TABLE(i2c, ds620_id); /* This is the driver that will be inserted */ static struct i2c_driver ds620_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ds620", }, diff --git a/drivers/hwmon/ina209.c b/drivers/hwmon/ina209.c index c558143e5285..d9b57a4b3e41 100644 --- a/drivers/hwmon/ina209.c +++ b/drivers/hwmon/ina209.c @@ -589,7 +589,6 @@ MODULE_DEVICE_TABLE(of, ina209_of_match); /* This is the driver that will be inserted */ static struct i2c_driver ina209_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ina209", .of_match_table = of_match_ptr(ina209_of_match), diff --git a/drivers/hwmon/ina238.c b/drivers/hwmon/ina238.c index ca9f5d2c811b..69289293bc38 100644 --- a/drivers/hwmon/ina238.c +++ b/drivers/hwmon/ina238.c @@ -629,7 +629,6 @@ static const struct of_device_id __maybe_unused ina238_of_match[] = { MODULE_DEVICE_TABLE(of, ina238_of_match); static struct i2c_driver ina238_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ina238", .of_match_table = of_match_ptr(ina238_of_match), diff --git a/drivers/hwmon/max127.c b/drivers/hwmon/max127.c index ee5ead06d612..da2289e3560a 100644 --- a/drivers/hwmon/max127.c +++ b/drivers/hwmon/max127.c @@ -335,7 +335,6 @@ static const struct i2c_device_id max127_id[] = { MODULE_DEVICE_TABLE(i2c, max127_id); static struct i2c_driver max127_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max127", }, diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c index 79945eb466ae..1b6f71bc61cb 100644 --- a/drivers/hwmon/max31760.c +++ b/drivers/hwmon/max31760.c @@ -578,7 +578,6 @@ static DEFINE_SIMPLE_DEV_PM_OPS(max31760_pm_ops, max31760_suspend, max31760_resume); static struct i2c_driver max31760_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max31760", .of_match_table = max31760_of_match, diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c index 0cd44c1e998a..3dc95196b229 100644 --- a/drivers/hwmon/max31790.c +++ b/drivers/hwmon/max31790.c @@ -543,7 +543,6 @@ static const struct i2c_device_id max31790_id[] = { MODULE_DEVICE_TABLE(i2c, max31790_id); static struct i2c_driver max31790_driver = { - .class = I2C_CLASS_HWMON, .probe = max31790_probe, .driver = { .name = "max31790", diff --git a/drivers/hwmon/max31827.c b/drivers/hwmon/max31827.c index 4a8c3e37c5d3..f8a13b30f100 100644 --- a/drivers/hwmon/max31827.c +++ b/drivers/hwmon/max31827.c @@ -652,7 +652,6 @@ static const struct of_device_id max31827_of_match[] = { MODULE_DEVICE_TABLE(of, max31827_of_match); static struct i2c_driver max31827_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max31827", .of_match_table = max31827_of_match, diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c index af7e62685898..05426cde0e36 100644 --- a/drivers/hwmon/max6621.c +++ b/drivers/hwmon/max6621.c @@ -549,7 +549,6 @@ static const struct of_device_id __maybe_unused max6621_of_match[] = { MODULE_DEVICE_TABLE(of, max6621_of_match); static struct i2c_driver max6621_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = MAX6621_DRV_NAME, .of_match_table = of_match_ptr(max6621_of_match), diff --git a/drivers/hwmon/max6697.c b/drivers/hwmon/max6697.c index 7d10dd434f2e..d161ba0e7813 100644 --- a/drivers/hwmon/max6697.c +++ b/drivers/hwmon/max6697.c @@ -780,7 +780,6 @@ static const struct of_device_id __maybe_unused max6697_of_match[] = { MODULE_DEVICE_TABLE(of, max6697_of_match); static struct i2c_driver max6697_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max6697", .of_match_table = of_match_ptr(max6697_of_match), diff --git a/drivers/hwmon/occ/p8_i2c.c b/drivers/hwmon/occ/p8_i2c.c index 06095975f5c8..31159606cec7 100644 --- a/drivers/hwmon/occ/p8_i2c.c +++ b/drivers/hwmon/occ/p8_i2c.c @@ -241,7 +241,6 @@ static const struct of_device_id p8_i2c_occ_of_match[] = { MODULE_DEVICE_TABLE(of, p8_i2c_occ_of_match); static struct i2c_driver p8_i2c_occ_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "occ-hwmon", .of_match_table = p8_i2c_occ_of_match, diff --git a/drivers/hwmon/pmbus/ir36021.c b/drivers/hwmon/pmbus/ir36021.c index 382ba6b6031a..a263afeb8ac1 100644 --- a/drivers/hwmon/pmbus/ir36021.c +++ b/drivers/hwmon/pmbus/ir36021.c @@ -63,7 +63,6 @@ static const struct of_device_id __maybe_unused ir36021_of_id[] = { MODULE_DEVICE_TABLE(of, ir36021_of_id); static struct i2c_driver ir36021_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ir36021", .of_match_table = of_match_ptr(ir36021_of_id), diff --git a/drivers/hwmon/powr1220.c b/drivers/hwmon/powr1220.c index 4120cadb00ae..2388d0565e7e 100644 --- a/drivers/hwmon/powr1220.c +++ b/drivers/hwmon/powr1220.c @@ -323,7 +323,6 @@ static const struct i2c_device_id powr1220_ids[] = { MODULE_DEVICE_TABLE(i2c, powr1220_ids); static struct i2c_driver powr1220_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "powr1220", }, diff --git a/drivers/hwmon/sbrmi.c b/drivers/hwmon/sbrmi.c index 484703f0ea5f..4318f5121145 100644 --- a/drivers/hwmon/sbrmi.c +++ b/drivers/hwmon/sbrmi.c @@ -342,7 +342,6 @@ static const struct of_device_id __maybe_unused sbrmi_of_match[] = { MODULE_DEVICE_TABLE(of, sbrmi_of_match); static struct i2c_driver sbrmi_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "sbrmi", .of_match_table = of_match_ptr(sbrmi_of_match), diff --git a/drivers/hwmon/sbtsi_temp.c b/drivers/hwmon/sbtsi_temp.c index dd85cf89f008..a4181acb1aa6 100644 --- a/drivers/hwmon/sbtsi_temp.c +++ b/drivers/hwmon/sbtsi_temp.c @@ -232,7 +232,6 @@ static const struct of_device_id __maybe_unused sbtsi_of_match[] = { MODULE_DEVICE_TABLE(of, sbtsi_of_match); static struct i2c_driver sbtsi_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "sbtsi", .of_match_table = of_match_ptr(sbtsi_of_match), diff --git a/drivers/hwmon/w83773g.c b/drivers/hwmon/w83773g.c index 045eea8378c2..401a28f55f93 100644 --- a/drivers/hwmon/w83773g.c +++ b/drivers/hwmon/w83773g.c @@ -290,7 +290,6 @@ static int w83773_probe(struct i2c_client *client) } static struct i2c_driver w83773_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "w83773g", .of_match_table = of_match_ptr(w83773_of_match), From 3317f2385f093d7dc17cf492a6676e7be5597349 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Sat, 27 Jan 2024 07:40:58 -0800 Subject: [PATCH 0244/1038] MAINTAINERS: Drop entries for hwmon devices with unreachable maintainers Drop maintainer entries for MAX31760 and MAX31827 since the e-mail addresses of their maintainers is no longer reachable and there is no known alternative means to contact them. HWMON drivers have a subsystem maintainer, so individual maintainer entries are not mandatory. Reported-by: Heiner Kallweit Cc: Heiner Kallweit Signed-off-by: Guenter Roeck --- MAINTAINERS | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 73d898383e51..ea2debbed7cb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1384,15 +1384,6 @@ F: drivers/iio/amplifiers/hmc425a.c F: drivers/staging/iio/*/ad* X: drivers/iio/*/adjd* -ANALOG DEVICES INC MAX31760 DRIVER -M: Ibrahim Tilki -S: Maintained -W: http://wiki.analog.com/ -W: https://ez.analog.com/linux-software-drivers -F: Documentation/devicetree/bindings/hwmon/adi,max31760.yaml -F: Documentation/hwmon/max31760.rst -F: drivers/hwmon/max31760.c - ANALOGBITS PLL LIBRARIES M: Paul Walmsley S: Supported @@ -13133,15 +13124,6 @@ F: Documentation/userspace-api/media/drivers/max2175.rst F: drivers/media/i2c/max2175* F: include/uapi/linux/max2175.h -MAX31827 TEMPERATURE SWITCH DRIVER -M: Daniel Matyas -L: linux-hwmon@vger.kernel.org -S: Supported -W: https://ez.analog.com/linux-software-drivers -F: Documentation/devicetree/bindings/hwmon/adi,max31827.yaml -F: Documentation/hwmon/max31827.rst -F: drivers/hwmon/max31827.c - MAX31335 RTC DRIVER M: Antoniu Miclaus L: linux-rtc@vger.kernel.org From 7a0564448a861952e9dc032de146abd19ea5a096 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sat, 27 Jan 2024 18:45:00 +0100 Subject: [PATCH 0245/1038] hwmon: Drop non-functional I2C_CLASS_HWMON support for drivers w/o detect() Class-based I2C probing requires detect() and address_list both to be set in the I2C client driver, see checks in i2c_detect(). It's misleading to declare I2C_CLASS_HWMON support if the driver doesn't implement detect(). Class-based probing is a legacy mechanism, in addition apparently nobody ever noticed that class-based probing has been non-functional in both drivers from the very beginning. So drop the fragments of class-based probing support. Signed-off-by: Heiner Kallweit Link: https://lore.kernel.org/r/13ce7c11-a958-4892-ada9-faf5bfdcb89d@gmail.com Signed-off-by: Guenter Roeck --- Documentation/hwmon/emc2305.rst | 1 - drivers/hwmon/adt7410.c | 2 -- drivers/hwmon/emc2305.c | 5 ----- 3 files changed, 8 deletions(-) diff --git a/Documentation/hwmon/emc2305.rst b/Documentation/hwmon/emc2305.rst index 2403dbaf2728..d0bfffe46358 100644 --- a/Documentation/hwmon/emc2305.rst +++ b/Documentation/hwmon/emc2305.rst @@ -6,7 +6,6 @@ Kernel driver emc2305 Supported chips: Microchip EMC2305, EMC2303, EMC2302, EMC2301 - Addresses scanned: I2C 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d Prefixes: 'emc2305' Datasheet: Publicly available at the Microchip website : diff --git a/drivers/hwmon/adt7410.c b/drivers/hwmon/adt7410.c index 952506779336..fd214d9b3a89 100644 --- a/drivers/hwmon/adt7410.c +++ b/drivers/hwmon/adt7410.c @@ -95,14 +95,12 @@ static const struct i2c_device_id adt7410_ids[] = { MODULE_DEVICE_TABLE(i2c, adt7410_ids); static struct i2c_driver adt7410_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "adt7410", .pm = pm_sleep_ptr(&adt7x10_dev_pm_ops), }, .probe = adt7410_i2c_probe, .id_table = adt7410_ids, - .address_list = I2C_ADDRS(0x48, 0x49, 0x4a, 0x4b), }; module_i2c_driver(adt7410_driver); diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c index 29f0e4945f19..6ef733c0be16 100644 --- a/drivers/hwmon/emc2305.c +++ b/drivers/hwmon/emc2305.c @@ -12,9 +12,6 @@ #include #include -static const unsigned short -emc2305_normal_i2c[] = { 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d, I2C_CLIENT_END }; - #define EMC2305_REG_DRIVE_FAIL_STATUS 0x27 #define EMC2305_REG_VENDOR 0xfe #define EMC2305_FAN_MAX 0xff @@ -611,14 +608,12 @@ static void emc2305_remove(struct i2c_client *client) } static struct i2c_driver emc2305_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "emc2305", }, .probe = emc2305_probe, .remove = emc2305_remove, .id_table = emc2305_ids, - .address_list = emc2305_normal_i2c, }; module_i2c_driver(emc2305_driver); From 9c6df63a66c1fdf99d6e1ad278d140080c724120 Mon Sep 17 00:00:00 2001 From: Patrick Rudolph Date: Tue, 30 Jan 2024 20:59:03 +0530 Subject: [PATCH 0246/1038] hwmon: (pmbus_core) Allow to hook PMBUS_SMBALERT_MASK Use _pmbus_write_word_data to allow intercepting writes to PMBUS_SMBALERT_MASK in the custom chip specific code. This is required for MP2971/MP2973 which doesn't follow the PMBUS specification for PMBUS_SMBALERT_MASK. Signed-off-by: Patrick Rudolph Signed-off-by: Naresh Solanki Link: https://lore.kernel.org/r/20240130152903.3651341-1-naresh.solanki@9elements.com Signed-off-by: Guenter Roeck --- drivers/hwmon/pmbus/pmbus_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index 1363d9f89181..cb4c65a7f288 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -3188,7 +3188,7 @@ static int pmbus_regulator_notify(struct pmbus_data *data, int page, int event) static int pmbus_write_smbalert_mask(struct i2c_client *client, u8 page, u8 reg, u8 val) { - return pmbus_write_word_data(client, page, PMBUS_SMBALERT_MASK, reg | (val << 8)); + return _pmbus_write_word_data(client, page, PMBUS_SMBALERT_MASK, reg | (val << 8)); } static irqreturn_t pmbus_fault_handler(int irq, void *pdata) From d07c41eb963a4f621d1867c36ad3b79b2f5faa20 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Mon, 29 Jan 2024 17:13:23 +0100 Subject: [PATCH 0247/1038] dt-bindings: hwmon: Add LTC4282 bindings Add bindings for the LTC4282 High Current Hot Swap Controller with I2C Compatible Monitoring. Reviewed-by: Conor Dooley Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240129-b4-ltc4282-support-v4-1-fe75798164cc@analog.com Signed-off-by: Guenter Roeck --- .../bindings/hwmon/adi,ltc4282.yaml | 159 ++++++++++++++++++ MAINTAINERS | 6 + 2 files changed, 165 insertions(+) create mode 100644 Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml diff --git a/Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml b/Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml new file mode 100644 index 000000000000..4854b95a93e3 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/adi,ltc4282.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices LTC4282 I2C High Current Hot Swap Controller over I2C + +maintainers: + - Nuno Sa + +description: | + Analog Devices LTC4282 I2C High Current Hot Swap Controller over I2C. + + https://www.analog.com/media/en/technical-documentation/data-sheets/ltc4282.pdf + +properties: + compatible: + enum: + - adi,ltc4282 + + reg: + maxItems: 1 + + vdd-supply: true + + clocks: + maxItems: 1 + + '#clock-cells': + const: 0 + + adi,rsense-nano-ohms: + description: Value of the sense resistor. + + adi,vin-mode-microvolt: + description: + Selects operating range for the Undervoltage, Overvoltage and Foldback + pins. Also for the ADC. Should be set to the nominal input voltage. + enum: [3300000, 5000000, 12000000, 24000000] + default: 12000000 + + adi,fet-bad-timeout-ms: + description: + From the moment a FET bad conditions is present, this property selects the + wait time/timeout for a FET-bad fault to be signaled. Setting this to 0, + disables FET bad faults to be reported. + default: 255 + maximum: 255 + + adi,overvoltage-dividers: + description: | + Select which dividers to use for VDD Overvoltage detection. Note that + when the internal dividers are used the threshold is referenced to VDD. + The percentages in the datasheet are misleading since the actual values + to look for are in the "Absolute Maximum Ratings" table in the + "Comparator Inputs" section. In there there's a line for each of the 5%, + 10% and 15% settings with the actual min, typical and max tolerances. + $ref: /schemas/types.yaml#/definitions/string + enum: [external, vdd_5_percent, vdd_10_percent, vdd_15_percent] + default: external + + adi,undervoltage-dividers: + description: | + Select which dividers to use for VDD Overvoltage detection. Note that + when the internal dividers are used the threshold is referenced to VDD. + The percentages in the datasheet are misleading since the actual values + to look for are in the "Absolute Maximum Ratings" table in the + "Comparator Inputs" section. In there there's a line for each of the 5%, + 10% and 15% settings with the actual min, typical and max tolerances. + $ref: /schemas/types.yaml#/definitions/string + enum: [external, vdd_5_percent, vdd_10_percent, vdd_15_percent] + default: external + + adi,current-limit-sense-microvolt: + description: + The current limit sense voltage of the chip is adjustable between + 12.5mV and 34.4mV in 3.1mV steps. This effectively limits the current + on the load. + enum: [12500, 15625, 18750, 21875, 25000, 28125, 31250, 34375] + default: 25000 + + adi,overcurrent-retry: + description: + If set, enables the chip to auto-retry 256 timer cycles after an + Overcurrent fault. + type: boolean + + adi,overvoltage-retry-disable: + description: + If set, disables the chip to auto-retry 50ms after an Overvoltage fault. + It's enabled by default. + type: boolean + + adi,undervoltage-retry-disable: + description: + If set, disables the chip to auto-retry 50ms after an Undervoltage fault. + It's enabled by default. + type: boolean + + adi,fault-log-enable: + description: + If set, enables the FAULT_LOG and ADC_ALERT_LOG registers to be written + to the EEPROM when a fault bit transitions high and hence, will be + available after a power cycle (the chip loads the contents of + the EE_FAULT_LOG register - the one in EEPROM - into FAULT_LOG at boot). + type: boolean + + adi,gpio1-mode: + description: Defines the function of the Pin. It can indicate that power is + good (PULL the pin low when power is not good) or that power is bad (Go + into high-z when power is not good). + $ref: /schemas/types.yaml#/definitions/string + enum: [power_bad, power_good] + default: power_good + + adi,gpio2-mode: + description: Defines the function of the Pin. It can be set as the input for + the ADC or indicating that the MOSFET is in stress (dissipating power). + $ref: /schemas/types.yaml#/definitions/string + enum: [adc_input, stress_fet] + default: adc_input + + adi,gpio3-monitor-enable: + description: If set, gpio3 is set as input for the ADC instead of gpio2. + type: boolean + +allOf: + - if: + required: + - adi,gpio3-monitor-enable + then: + properties: + adi,gpio2-mode: + const: stress_fet + +required: + - compatible + - reg + - adi,rsense-nano-ohms + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + hwmon@50 { + compatible = "adi,ltc4282"; + reg = <0x50>; + adi,rsense-nano-ohms = <500>; + + adi,gpio1-mode = "power_good"; + adi,gpio2-mode = "adc_input"; + }; + }; +... diff --git a/MAINTAINERS b/MAINTAINERS index ea2debbed7cb..3625754d7d1b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12760,6 +12760,12 @@ S: Maintained F: Documentation/hwmon/ltc4261.rst F: drivers/hwmon/ltc4261.c +LTC4282 HARDWARE MONITOR DRIVER +M: Nuno Sa +L: linux-hwmon@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml + LTC4286 HARDWARE MONITOR DRIVER M: Delphine CC Chiu L: linux-i2c@vger.kernel.org From 35c1bfb99fef9c71f9df5c1325db99a79300bc97 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Mon, 29 Jan 2024 17:13:24 +0100 Subject: [PATCH 0248/1038] hwmon: add fault attribute for voltage channels Sometimes a voltage channel might have an hard failure (eg: a shorted MOSFET). Hence, add a fault attribute to report such failures. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240129-b4-ltc4282-support-v4-2-fe75798164cc@analog.com Signed-off-by: Guenter Roeck --- Documentation/ABI/testing/sysfs-class-hwmon | 9 +++++++++ drivers/hwmon/hwmon.c | 1 + include/linux/hwmon.h | 2 ++ 3 files changed, 12 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon index 3dac923c9b0e..6c4e68ad4a83 100644 --- a/Documentation/ABI/testing/sysfs-class-hwmon +++ b/Documentation/ABI/testing/sysfs-class-hwmon @@ -149,6 +149,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/inY_fault +Description: + Reports a voltage hard failure (eg: shorted component) + + - 1: Failed + - 0: Ok + + RO + What: /sys/class/hwmon/hwmonX/cpuY_vid Description: CPU core reference voltage. diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index c7dd3f5b2bd5..18705049ad61 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -510,6 +510,7 @@ static const char * const hwmon_in_attr_templates[] = { [hwmon_in_rated_min] = "in%d_rated_min", [hwmon_in_rated_max] = "in%d_rated_max", [hwmon_in_beep] = "in%d_beep", + [hwmon_in_fault] = "in%d_fault", }; static const char * const hwmon_curr_attr_templates[] = { diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index c2c0da18dfa3..c7885fdce88f 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -141,6 +141,7 @@ enum hwmon_in_attributes { hwmon_in_rated_min, hwmon_in_rated_max, hwmon_in_beep, + hwmon_in_fault, }; #define HWMON_I_ENABLE BIT(hwmon_in_enable) @@ -162,6 +163,7 @@ enum hwmon_in_attributes { #define HWMON_I_RATED_MIN BIT(hwmon_in_rated_min) #define HWMON_I_RATED_MAX BIT(hwmon_in_rated_max) #define HWMON_I_BEEP BIT(hwmon_in_beep) +#define HWMON_I_FAULT BIT(hwmon_in_fault) enum hwmon_curr_attributes { hwmon_curr_enable, From faf6efd2e5e23d3319501132d9671c8606ef21bd Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:27:04 +0100 Subject: [PATCH 0249/1038] gpio: constify opaque pointer in gpio_device_find() match function The match function used in gpio_device_find() should not modify the contents of passed opaque pointer, because such modification would not be necessary for actual matching and it could lead to quite unreadable, spaghetti code. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Mika Westerberg Reviewed-by: Linus Walleij Reviewed-by: Andy Shevchenko [Bartosz: fix coding style in header] Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-acpi.c | 2 +- drivers/gpio/gpiolib-of.c | 7 ++++--- drivers/gpio/gpiolib.c | 6 +++--- include/linux/gpio/driver.h | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index cd3e9657cc36..899cd505073e 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -126,7 +126,7 @@ static DEFINE_MUTEX(acpi_gpio_deferred_req_irqs_lock); static LIST_HEAD(acpi_gpio_deferred_req_irqs_list); static bool acpi_gpio_deferred_req_irqs_done; -static int acpi_gpiochip_find(struct gpio_chip *gc, void *data) +static int acpi_gpiochip_find(struct gpio_chip *gc, const void *data) { return device_match_acpi_handle(&gc->gpiodev->dev, data); } diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 77509aa19900..35d717fd393f 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -118,9 +118,10 @@ int of_gpio_get_count(struct device *dev, const char *con_id) return ret ? ret : -ENOENT; } -static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, + const void *data) { - struct of_phandle_args *gpiospec = data; + const struct of_phandle_args *gpiospec = data; return device_match_of_node(&chip->gpiodev->dev, gpiospec->np) && chip->of_xlate && @@ -852,7 +853,7 @@ static void of_gpiochip_remove_hog(struct gpio_chip *chip, gpiochip_free_own_desc(desc); } -static int of_gpiochip_match_node(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node(struct gpio_chip *chip, const void *data) { return device_match_of_node(&chip->gpiodev->dev, data); } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index d068788adb10..288aed5a9f6b 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1119,7 +1119,7 @@ EXPORT_SYMBOL_GPL(gpiochip_remove); */ struct gpio_device *gpio_device_find(void *data, int (*match)(struct gpio_chip *gc, - void *data)) + const void *data)) { struct gpio_device *gdev; @@ -1141,7 +1141,7 @@ struct gpio_device *gpio_device_find(void *data, } EXPORT_SYMBOL_GPL(gpio_device_find); -static int gpio_chip_match_by_label(struct gpio_chip *gc, void *label) +static int gpio_chip_match_by_label(struct gpio_chip *gc, const void *label) { return gc->label && !strcmp(gc->label, label); } @@ -1161,7 +1161,7 @@ struct gpio_device *gpio_device_find_by_label(const char *label) } EXPORT_SYMBOL_GPL(gpio_device_find_by_label); -static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, void *fwnode) +static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, const void *fwnode) { return device_match_fwnode(&gc->gpiodev->dev, fwnode); } diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 3a37d058cfcf..9d0023f83a57 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -629,7 +629,8 @@ int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, struct lock_class_key *request_key); struct gpio_device *gpio_device_find(void *data, - int (*match)(struct gpio_chip *gc, void *data)); + int (*match)(struct gpio_chip *gc, + const void *data)); struct gpio_device *gpio_device_find_by_label(const char *label); struct gpio_device *gpio_device_find_by_fwnode(const struct fwnode_handle *fwnode); From e348544f7994d252427ed3ae637c7081cbb90f66 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 19 Jan 2024 16:43:13 +0100 Subject: [PATCH 0250/1038] gpio: protect the list of GPIO devices with SRCU We're working towards removing the "multi-function" GPIO spinlock that's implemented terribly wrong. We tried using an RW-semaphore to protect the list of GPIO devices but it turned out that we still have old code using legacy GPIO calls that need to translate the global GPIO number to the address of the associated descriptor and - to that end - traverse the list while holding the lock. If we change the spinlock to a sleeping lock then we'll end up with "scheduling while atomic" bugs. Let's allow lockless traversal of the list using SRCU and only use the mutex when modyfing the list. While at it: let's protect the period between when we start the lookup and when we finally request the descriptor (increasing the reference count of the GPIO device) with the SRCU read lock. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 251 ++++++++++++++++++++++------------------- 1 file changed, 137 insertions(+), 114 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 288aed5a9f6b..a5598c3299ff 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -14,12 +15,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -81,7 +84,12 @@ DEFINE_SPINLOCK(gpio_lock); static DEFINE_MUTEX(gpio_lookup_lock); static LIST_HEAD(gpio_lookup_list); + LIST_HEAD(gpio_devices); +/* Protects the GPIO device list against concurrent modifications. */ +static DEFINE_MUTEX(gpio_devices_lock); +/* Ensures coherence during read-only accesses to the list of GPIO devices. */ +DEFINE_STATIC_SRCU(gpio_devices_srcu); static DEFINE_MUTEX(gpio_machine_hogs_mutex); static LIST_HEAD(gpio_machine_hogs); @@ -113,20 +121,16 @@ static inline void desc_set_label(struct gpio_desc *d, const char *label) struct gpio_desc *gpio_to_desc(unsigned gpio) { struct gpio_device *gdev; - unsigned long flags; - spin_lock_irqsave(&gpio_lock, flags); - - list_for_each_entry(gdev, &gpio_devices, list) { - if (gdev->base <= gpio && - gdev->base + gdev->ngpio > gpio) { - spin_unlock_irqrestore(&gpio_lock, flags); - return &gdev->descs[gpio - gdev->base]; + scoped_guard(srcu, &gpio_devices_srcu) { + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + if (gdev->base <= gpio && + gdev->base + gdev->ngpio > gpio) + return &gdev->descs[gpio - gdev->base]; } } - spin_unlock_irqrestore(&gpio_lock, flags); - if (!gpio_is_valid(gpio)) pr_warn("invalid GPIO %d\n", gpio); @@ -282,7 +286,8 @@ static int gpiochip_find_base_unlocked(int ngpio) struct gpio_device *gdev; int base = GPIO_DYNAMIC_BASE; - list_for_each_entry(gdev, &gpio_devices, list) { + list_for_each_entry_srcu(gdev, &gpio_devices, list, + lockdep_is_held(&gpio_devices_lock)) { /* found a free space? */ if (gdev->base >= base + ngpio) break; @@ -354,23 +359,25 @@ static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev) { struct gpio_device *prev, *next; + lockdep_assert_held(&gpio_devices_lock); + if (list_empty(&gpio_devices)) { /* initial entry in list */ - list_add_tail(&gdev->list, &gpio_devices); + list_add_tail_rcu(&gdev->list, &gpio_devices); return 0; } next = list_first_entry(&gpio_devices, struct gpio_device, list); if (gdev->base + gdev->ngpio <= next->base) { /* add before first entry */ - list_add(&gdev->list, &gpio_devices); + list_add_rcu(&gdev->list, &gpio_devices); return 0; } prev = list_last_entry(&gpio_devices, struct gpio_device, list); if (prev->base + prev->ngpio <= gdev->base) { /* add behind last entry */ - list_add_tail(&gdev->list, &gpio_devices); + list_add_tail_rcu(&gdev->list, &gpio_devices); return 0; } @@ -382,11 +389,13 @@ static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev) /* add between prev and next */ if (prev->base + prev->ngpio <= gdev->base && gdev->base + gdev->ngpio <= next->base) { - list_add(&gdev->list, &prev->list); + list_add_rcu(&gdev->list, &prev->list); return 0; } } + synchronize_srcu(&gpio_devices_srcu); + return -EBUSY; } @@ -399,26 +408,21 @@ static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev) static struct gpio_desc *gpio_name_to_desc(const char * const name) { struct gpio_device *gdev; - unsigned long flags; + struct gpio_desc *desc; if (!name) return NULL; - spin_lock_irqsave(&gpio_lock, flags); - - list_for_each_entry(gdev, &gpio_devices, list) { - struct gpio_desc *desc; + guard(srcu)(&gpio_devices_srcu); + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { for_each_gpio_desc(gdev->chip, desc) { - if (desc->name && !strcmp(desc->name, name)) { - spin_unlock_irqrestore(&gpio_lock, flags); + if (desc->name && !strcmp(desc->name, name)) return desc; - } } } - spin_unlock_irqrestore(&gpio_lock, flags); - return NULL; } @@ -752,7 +756,10 @@ static void gpiochip_setup_devs(void) struct gpio_device *gdev; int ret; - list_for_each_entry(gdev, &gpio_devices, list) { + guard(srcu)(&gpio_devices_srcu); + + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { ret = gpiochip_setup_dev(gdev); if (ret) dev_err(&gdev->dev, @@ -817,7 +824,6 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, struct lock_class_key *request_key) { struct gpio_device *gdev; - unsigned long flags; unsigned int i; int base = 0; int ret = 0; @@ -883,49 +889,47 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev->ngpio = gc->ngpio; - spin_lock_irqsave(&gpio_lock, flags); - - /* - * TODO: this allocates a Linux GPIO number base in the global - * GPIO numberspace for this chip. In the long run we want to - * get *rid* of this numberspace and use only descriptors, but - * it may be a pipe dream. It will not happen before we get rid - * of the sysfs interface anyways. - */ - base = gc->base; - if (base < 0) { - base = gpiochip_find_base_unlocked(gc->ngpio); + scoped_guard(mutex, &gpio_devices_lock) { + /* + * TODO: this allocates a Linux GPIO number base in the global + * GPIO numberspace for this chip. In the long run we want to + * get *rid* of this numberspace and use only descriptors, but + * it may be a pipe dream. It will not happen before we get rid + * of the sysfs interface anyways. + */ + base = gc->base; if (base < 0) { - spin_unlock_irqrestore(&gpio_lock, flags); - ret = base; - base = 0; + base = gpiochip_find_base_unlocked(gc->ngpio); + if (base < 0) { + ret = base; + base = 0; + goto err_free_label; + } + + /* + * TODO: it should not be necessary to reflect the + * assigned base outside of the GPIO subsystem. Go over + * drivers and see if anyone makes use of this, else + * drop this and assign a poison instead. + */ + gc->base = base; + } else { + dev_warn(&gdev->dev, + "Static allocation of GPIO base is deprecated, use dynamic allocation.\n"); + } + + gdev->base = base; + + ret = gpiodev_add_to_list_unlocked(gdev); + if (ret) { + chip_err(gc, "GPIO integer space overlap, cannot add chip\n"); goto err_free_label; } - /* - * TODO: it should not be necessary to reflect the assigned - * base outside of the GPIO subsystem. Go over drivers and - * see if anyone makes use of this, else drop this and assign - * a poison instead. - */ - gc->base = base; - } else { - dev_warn(&gdev->dev, - "Static allocation of GPIO base is deprecated, use dynamic allocation.\n"); - } - gdev->base = base; - - ret = gpiodev_add_to_list_unlocked(gdev); - if (ret) { - spin_unlock_irqrestore(&gpio_lock, flags); - chip_err(gc, "GPIO integer space overlap, cannot add chip\n"); - goto err_free_label; } for (i = 0; i < gc->ngpio; i++) gdev->descs[i].gdev = gdev; - spin_unlock_irqrestore(&gpio_lock, flags); - BLOCKING_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); init_rwsem(&gdev->sem); @@ -1011,9 +1015,9 @@ err_free_gpiochip_mask: gpiochip_remove_pin_ranges(gc); gpiochip_free_valid_mask(gc); err_remove_from_list: - spin_lock_irqsave(&gpio_lock, flags); - list_del(&gdev->list); - spin_unlock_irqrestore(&gpio_lock, flags); + scoped_guard(mutex, &gpio_devices_lock) + list_del_rcu(&gdev->list); + synchronize_srcu(&gpio_devices_srcu); if (gdev->dev.release) { /* release() has been registered by gpiochip_setup_dev() */ gpio_device_put(gdev); @@ -1057,6 +1061,11 @@ void gpiochip_remove(struct gpio_chip *gc) /* FIXME: should the legacy sysfs handling be moved to gpio_device? */ gpiochip_sysfs_unregister(gdev); gpiochip_free_hogs(gc); + + scoped_guard(mutex, &gpio_devices_lock) + list_del_rcu(&gdev->list); + synchronize_srcu(&gpio_devices_srcu); + /* Numb the device, cancelling all outstanding operations */ gdev->chip = NULL; gpiochip_irqchip_remove(gc); @@ -1081,9 +1090,6 @@ void gpiochip_remove(struct gpio_chip *gc) dev_crit(&gdev->dev, "REMOVING GPIOCHIP WITH GPIOS STILL REQUESTED\n"); - scoped_guard(spinlock_irqsave, &gpio_lock) - list_del(&gdev->list); - /* * The gpiochip side puts its use of the device to rest here: * if there are no userspace clients, the chardev and device will @@ -1130,7 +1136,7 @@ struct gpio_device *gpio_device_find(void *data, */ might_sleep(); - guard(spinlock_irqsave)(&gpio_lock); + guard(srcu)(&gpio_devices_srcu); list_for_each_entry(gdev, &gpio_devices, list) { if (gdev->chip && match(gdev->chip, data)) @@ -4138,30 +4144,39 @@ static struct gpio_desc *gpiod_find_and_request(struct device *consumer, bool platform_lookup_allowed) { unsigned long lookupflags = GPIO_LOOKUP_FLAGS_DEFAULT; - struct gpio_desc *desc; - int ret; - - desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, &flags, &lookupflags); - if (gpiod_not_found(desc) && platform_lookup_allowed) { - /* - * Either we are not using DT or ACPI, or their lookup did not - * return a result. In that case, use platform lookup as a - * fallback. - */ - dev_dbg(consumer, "using lookup tables for GPIO lookup\n"); - desc = gpiod_find(consumer, con_id, idx, &lookupflags); - } - - if (IS_ERR(desc)) { - dev_dbg(consumer, "No GPIO consumer %s found\n", con_id); - return desc; - } - /* - * If a connection label was passed use that, else attempt to use - * the device name as label + * scoped_guard() is implemented as a for loop, meaning static + * analyzers will complain about these two not being initialized. */ - ret = gpiod_request(desc, label); + struct gpio_desc *desc = NULL; + int ret = 0; + + scoped_guard(srcu, &gpio_devices_srcu) { + desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, + &flags, &lookupflags); + if (gpiod_not_found(desc) && platform_lookup_allowed) { + /* + * Either we are not using DT or ACPI, or their lookup + * did not return a result. In that case, use platform + * lookup as a fallback. + */ + dev_dbg(consumer, + "using lookup tables for GPIO lookup\n"); + desc = gpiod_find(consumer, con_id, idx, &lookupflags); + } + + if (IS_ERR(desc)) { + dev_dbg(consumer, "No GPIO consumer %s found\n", + con_id); + return desc; + } + + /* + * If a connection label was passed use that, else attempt to use + * the device name as label + */ + ret = gpiod_request(desc, label); + } if (ret) { if (!(ret == -EBUSY && flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE)) return ERR_PTR(ret); @@ -4730,61 +4745,69 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) } } +struct gpiolib_seq_priv { + bool newline; + int idx; +}; + static void *gpiolib_seq_start(struct seq_file *s, loff_t *pos) { - unsigned long flags; - struct gpio_device *gdev = NULL; + struct gpiolib_seq_priv *priv; + struct gpio_device *gdev; loff_t index = *pos; - s->private = ""; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; - spin_lock_irqsave(&gpio_lock, flags); - list_for_each_entry(gdev, &gpio_devices, list) - if (index-- == 0) { - spin_unlock_irqrestore(&gpio_lock, flags); + s->private = priv; + priv->idx = srcu_read_lock(&gpio_devices_srcu); + + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + if (index-- == 0) return gdev; - } - spin_unlock_irqrestore(&gpio_lock, flags); + } return NULL; } static void *gpiolib_seq_next(struct seq_file *s, void *v, loff_t *pos) { - unsigned long flags; - struct gpio_device *gdev = v; - void *ret = NULL; + struct gpiolib_seq_priv *priv = s->private; + struct gpio_device *gdev = v, *next; - spin_lock_irqsave(&gpio_lock, flags); - if (list_is_last(&gdev->list, &gpio_devices)) - ret = NULL; - else - ret = list_first_entry(&gdev->list, struct gpio_device, list); - spin_unlock_irqrestore(&gpio_lock, flags); - - s->private = "\n"; + next = list_entry_rcu(gdev->list.next, struct gpio_device, list); + gdev = &next->list == &gpio_devices ? NULL : next; + priv->newline = true; ++*pos; - return ret; + return gdev; } static void gpiolib_seq_stop(struct seq_file *s, void *v) { + struct gpiolib_seq_priv *priv = s->private; + + srcu_read_unlock(&gpio_devices_srcu, priv->idx); + kfree(priv); } static int gpiolib_seq_show(struct seq_file *s, void *v) { + struct gpiolib_seq_priv *priv = s->private; struct gpio_device *gdev = v; struct gpio_chip *gc = gdev->chip; struct device *parent; if (!gc) { - seq_printf(s, "%s%s: (dangling chip)", (char *)s->private, + seq_printf(s, "%s%s: (dangling chip)", + priv->newline ? "\n" : "", dev_name(&gdev->dev)); return 0; } - seq_printf(s, "%s%s: GPIOs %d-%d", (char *)s->private, + seq_printf(s, "%s%s: GPIOs %d-%d", priv->newline ? "\n" : "", dev_name(&gdev->dev), gdev->base, gdev->base + gdev->ngpio - 1); parent = gc->parent; From 8ce6fd81a452d554889874e50cd34e92cf4547d6 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 5 Jan 2024 13:12:45 +0100 Subject: [PATCH 0251/1038] gpio: of: assign and read the hog pointer atomically The device nodes representing GPIO hogs cannot be deleted without unregistering the GPIO chip so there's no need to serialize their access. However we must ensure that users can get the right address so write and read it atomically. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-of.c | 4 ++-- drivers/gpio/gpiolib.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 35d717fd393f..523b047a2803 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -801,7 +801,7 @@ static int of_gpiochip_add_hog(struct gpio_chip *chip, struct device_node *hog) return ret; #ifdef CONFIG_OF_DYNAMIC - desc->hog = hog; + WRITE_ONCE(desc->hog, hog); #endif } @@ -849,7 +849,7 @@ static void of_gpiochip_remove_hog(struct gpio_chip *chip, struct gpio_desc *desc; for_each_gpio_desc_with_flag(chip, desc, FLAG_IS_HOGGED) - if (desc->hog == hog) + if (READ_ONCE(desc->hog) == hog) gpiochip_free_own_desc(desc); } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index a5598c3299ff..03d7c3f92f5e 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2330,7 +2330,7 @@ static bool gpiod_free_commit(struct gpio_desc *desc) clear_bit(FLAG_EDGE_FALLING, &desc->flags); clear_bit(FLAG_IS_HOGGED, &desc->flags); #ifdef CONFIG_OF_DYNAMIC - desc->hog = NULL; + WRITE_ONCE(desc->hog, NULL); #endif ret = true; } From 0857c39bfd09183792c680858b4416498ee1d1ca Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 5 Jan 2024 19:51:19 +0100 Subject: [PATCH 0252/1038] gpio: remove unused logging helpers The general rule of the kernel is to not provide symbols that have no users upstream. Let's remove logging helpers that are not used anywhere. This will save us work later when we'll be modifying them to use the upcoming SRCU infrastructure. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.h | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index a4a2520b5f31..c3ae5bfa3f2e 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -219,31 +219,18 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc) /* With descriptor prefix */ -#define gpiod_emerg(desc, fmt, ...) \ - pr_emerg("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?",\ - ##__VA_ARGS__) -#define gpiod_crit(desc, fmt, ...) \ - pr_crit("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) #define gpiod_err(desc, fmt, ...) \ pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ ##__VA_ARGS__) #define gpiod_warn(desc, fmt, ...) \ pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ ##__VA_ARGS__) -#define gpiod_info(desc, fmt, ...) \ - pr_info("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) #define gpiod_dbg(desc, fmt, ...) \ pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?",\ ##__VA_ARGS__) /* With chip prefix */ -#define chip_emerg(gc, fmt, ...) \ - dev_emerg(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) -#define chip_crit(gc, fmt, ...) \ - dev_crit(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) #define chip_err(gc, fmt, ...) \ dev_err(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) #define chip_warn(gc, fmt, ...) \ From d23dc4a9a88f16d0327e7b4bb03e6f43c1a1b166 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 29 Jan 2024 10:11:41 +0100 Subject: [PATCH 0253/1038] gpio: provide and use gpiod_get_label() We will soon serialize access to the descriptor label using SRCU. The write-side of the protection will require calling synchronize_srcu() which must not be called from atomic context. We have two irq helpers: gpiochip_lock_as_irq() and gpiochip_unlock_as_irq() that set the label if the GPIO is not requested but is being used as interrupt. They are called with a spinlock held from the interrupt subsystem. They must not do it if we are to use SRCU so instead let's move the special corner case to a dedicated getter. First: let's implement and use the getter where it's applicable. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-cdev.c | 4 ++-- drivers/gpio/gpiolib.c | 9 +++++++-- drivers/gpio/gpiolib.h | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 34d6712fa07c..2c0a0700762d 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2305,8 +2305,8 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, if (desc->name) strscpy(info->name, desc->name, sizeof(info->name)); - if (desc->label) - strscpy(info->consumer, desc->label, + if (gpiod_get_label(desc)) + strscpy(info->consumer, gpiod_get_label(desc), sizeof(info->consumer)); dflags = READ_ONCE(desc->flags); diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 03d7c3f92f5e..c014404c3bed 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -105,6 +105,11 @@ static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc); static bool gpiolib_initialized; +const char *gpiod_get_label(struct gpio_desc *desc) +{ + return desc->label; +} + static inline void desc_set_label(struct gpio_desc *d, const char *label) { d->label = label; @@ -2391,7 +2396,7 @@ char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset) * * Until this happens, this allocation needs to be atomic. */ - label = kstrdup(desc->label, GFP_ATOMIC); + label = kstrdup(gpiod_get_label(desc), GFP_ATOMIC); if (!label) return ERR_PTR(-ENOMEM); @@ -4732,7 +4737,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) is_irq = test_bit(FLAG_USED_AS_IRQ, &desc->flags); active_low = test_bit(FLAG_ACTIVE_LOW, &desc->flags); seq_printf(s, " gpio-%-3d (%-20.20s|%-20.20s) %s %s %s%s\n", - gpio, desc->name ?: "", desc->label, + gpio, desc->name ?: "", gpiod_get_label(desc), is_out ? "out" : "in ", value >= 0 ? (value ? "hi" : "lo") : "? ", is_irq ? "IRQ " : "", diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index c3ae5bfa3f2e..1058f326fe2b 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -208,6 +208,7 @@ int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce); int gpiod_hog(struct gpio_desc *desc, const char *name, unsigned long lflags, enum gpiod_flags dflags); int gpiochip_get_ngpios(struct gpio_chip *gc, struct device *dev); +const char *gpiod_get_label(struct gpio_desc *desc); /* * Return the GPIO number of the passed descriptor relative to its chip From ccfb6ff4f6c0574e01fb16934fb60a46285c5f3f Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 29 Jan 2024 10:23:27 +0100 Subject: [PATCH 0254/1038] gpio: don't set label from irq helpers We will soon serialize access to the descriptor label using SRCU. The write-side of the protection will require calling synchronize_srcu() which must not be called from atomic context. We have two irq helpers: gpiochip_lock_as_irq() and gpiochip_unlock_as_irq() that set the label if the GPIO is not requested but is being used as interrupt. They are called with a spinlock held from the interrupt subsystem. They must not do it if we are to use SRCU so instead let's move the special corner case to a dedicated getter. Don't actually set the label to "interrupt" in the above case but rather use the newly added gpiod_get_label() helper to hide the logic that atomically checks the descriptor flags and returns the address of a static "interrupt" string. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index c014404c3bed..79717776be84 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -107,7 +107,14 @@ static bool gpiolib_initialized; const char *gpiod_get_label(struct gpio_desc *desc) { - return desc->label; + unsigned long flags; + + flags = READ_ONCE(desc->flags); + if (test_bit(FLAG_USED_AS_IRQ, &flags) && + !test_bit(FLAG_REQUESTED, &flags)) + return "interrupt"; + + return test_bit(FLAG_REQUESTED, &flags) ? desc->label : NULL; } static inline void desc_set_label(struct gpio_desc *d, const char *label) @@ -3599,14 +3606,6 @@ int gpiochip_lock_as_irq(struct gpio_chip *gc, unsigned int offset) set_bit(FLAG_USED_AS_IRQ, &desc->flags); set_bit(FLAG_IRQ_IS_ENABLED, &desc->flags); - /* - * If the consumer has not set up a label (such as when the - * IRQ is referenced from .to_irq()) we set up a label here - * so it is clear this is used as an interrupt. - */ - if (!desc->label) - desc_set_label(desc, "interrupt"); - return 0; } EXPORT_SYMBOL_GPL(gpiochip_lock_as_irq); @@ -3629,10 +3628,6 @@ void gpiochip_unlock_as_irq(struct gpio_chip *gc, unsigned int offset) clear_bit(FLAG_USED_AS_IRQ, &desc->flags); clear_bit(FLAG_IRQ_IS_ENABLED, &desc->flags); - - /* If we only had this marking, erase it */ - if (desc->label && !strcmp(desc->label, "interrupt")) - desc_set_label(desc, NULL); } EXPORT_SYMBOL_GPL(gpiochip_unlock_as_irq); From be711caa87c5c81d5dc00b244cac3a0b775adb18 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 5 Jan 2024 15:46:27 +0100 Subject: [PATCH 0255/1038] gpio: add SRCU infrastructure to struct gpio_desc Extend the GPIO descriptor with an SRCU structure in order to serialize the access to the label. Initialize and clean it up where applicable. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 18 ++++++++++++++++-- drivers/gpio/gpiolib.h | 3 +++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 79717776be84..78b9e9dc268a 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -672,6 +672,10 @@ EXPORT_SYMBOL_GPL(gpiochip_line_is_valid); static void gpiodev_release(struct device *dev) { struct gpio_device *gdev = to_gpio_device(dev); + unsigned int i; + + for (i = 0; i < gdev->ngpio; i++) + cleanup_srcu_struct(&gdev->descs[i].srcu); ida_free(&gpio_ida, gdev->id); kfree_const(gdev->label); @@ -836,7 +840,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, struct lock_class_key *request_key) { struct gpio_device *gdev; - unsigned int i; + unsigned int i, j; int base = 0; int ret = 0; @@ -970,6 +974,13 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, for (i = 0; i < gc->ngpio; i++) { struct gpio_desc *desc = &gdev->descs[i]; + ret = init_srcu_struct(&desc->srcu); + if (ret) { + for (j = 0; j < i; j++) + cleanup_srcu_struct(&gdev->descs[j].srcu); + goto err_remove_of_chip; + } + if (gc->get_direction && gpiochip_line_is_valid(gc, i)) { assign_bit(FLAG_IS_OUT, &desc->flags, !gc->get_direction(gc, i)); @@ -981,7 +992,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, ret = gpiochip_add_pin_ranges(gc); if (ret) - goto err_remove_of_chip; + goto err_cleanup_desc_srcu; acpi_gpiochip_add(gc); @@ -1020,6 +1031,9 @@ err_remove_irqchip_mask: gpiochip_irqchip_free_valid_mask(gc); err_remove_acpi_chip: acpi_gpiochip_remove(gc); +err_cleanup_desc_srcu: + for (i = 0; i < gdev->ngpio; i++) + cleanup_srcu_struct(&gdev->descs[i].srcu); err_remove_of_chip: gpiochip_free_hogs(gc); of_gpiochip_remove(gc); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 1058f326fe2b..6e14b629c48b 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -17,6 +17,7 @@ #include #include #include +#include #define GPIOCHIP_NAME "gpiochip" @@ -147,6 +148,7 @@ void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); * @label: Name of the consumer * @name: Line name * @hog: Pointer to the device node that hogs this line (if any) + * @srcu: SRCU struct protecting the label pointer. * * These are obtained using gpiod_get() and are preferable to the old * integer-based handles. @@ -184,6 +186,7 @@ struct gpio_desc { #ifdef CONFIG_OF_DYNAMIC struct device_node *hog; #endif + struct srcu_struct srcu; }; #define gpiod_not_found(desc) (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) From 1f2bcb8c8ccdf9dc2e46f7986e1e22408506a6d6 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 5 Jan 2024 17:08:36 +0100 Subject: [PATCH 0256/1038] gpio: protect the descriptor label with SRCU In order to ensure that the label is not freed while it's being accessed, let's protect it with SRCU and synchronize it everytime it's changed. Let's modify desc_set_label() to manage the memory used for the label as it can only be freed once synchronize_srcu() returns. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-cdev.c | 10 +++++--- drivers/gpio/gpiolib.c | 47 +++++++++++++++++++++++-------------- drivers/gpio/gpiolib.h | 34 +++++++++++++++++++-------- 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 2c0a0700762d..75f4912339a6 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2297,6 +2297,7 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, { struct gpio_chip *gc = desc->gdev->chip; unsigned long dflags; + const char *label; memset(info, 0, sizeof(*info)); info->offset = gpio_chip_hwgpio(desc); @@ -2305,9 +2306,12 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, if (desc->name) strscpy(info->name, desc->name, sizeof(info->name)); - if (gpiod_get_label(desc)) - strscpy(info->consumer, gpiod_get_label(desc), - sizeof(info->consumer)); + scoped_guard(srcu, &desc->srcu) { + label = gpiod_get_label(desc); + if (label) + strscpy(info->consumer, label, + sizeof(info->consumer)); + } dflags = READ_ONCE(desc->flags); } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 78b9e9dc268a..c72f3ffbe6b4 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -114,12 +114,26 @@ const char *gpiod_get_label(struct gpio_desc *desc) !test_bit(FLAG_REQUESTED, &flags)) return "interrupt"; - return test_bit(FLAG_REQUESTED, &flags) ? desc->label : NULL; + return test_bit(FLAG_REQUESTED, &flags) ? + rcu_dereference(desc->label) : NULL; } -static inline void desc_set_label(struct gpio_desc *d, const char *label) +static int desc_set_label(struct gpio_desc *desc, const char *label) { - d->label = label; + const char *new = NULL, *old; + + if (label) { + /* FIXME: make this GFP_KERNEL once the spinlock is out. */ + new = kstrdup_const(label, GFP_ATOMIC); + if (!new) + return -ENOMEM; + } + + old = rcu_replace_pointer(desc->label, new, 1); + synchronize_srcu(&desc->srcu); + kfree_const(old); + + return 0; } /** @@ -2229,9 +2243,7 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) * before IRQs are enabled, for non-sleeping (SOC) GPIOs. */ - if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) { - desc_set_label(desc, label ? : "?"); - } else { + if (test_and_set_bit(FLAG_REQUESTED, &desc->flags)) { ret = -EBUSY; goto out_free_unlock; } @@ -2259,6 +2271,13 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) spin_lock_irqsave(&gpio_lock, flags); } spin_unlock_irqrestore(&gpio_lock, flags); + + ret = desc_set_label(desc, label ? : "?"); + if (ret) { + clear_bit(FLAG_REQUESTED, &desc->flags); + return ret; + } + return 0; out_free_unlock: @@ -2343,8 +2362,6 @@ static bool gpiod_free_commit(struct gpio_desc *desc) gc->free(gc, gpio_chip_hwgpio(desc)); spin_lock_irqsave(&gpio_lock, flags); } - kfree_const(desc->label); - desc_set_label(desc, NULL); clear_bit(FLAG_ACTIVE_LOW, &desc->flags); clear_bit(FLAG_REQUESTED, &desc->flags); clear_bit(FLAG_OPEN_DRAIN, &desc->flags); @@ -2362,6 +2379,7 @@ static bool gpiod_free_commit(struct gpio_desc *desc) } spin_unlock_irqrestore(&gpio_lock, flags); + desc_set_label(desc, NULL); gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED); return ret; @@ -2409,6 +2427,8 @@ char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset) if (!test_bit(FLAG_REQUESTED, &desc->flags)) return NULL; + guard(srcu)(&desc->srcu); + /* * FIXME: Once we mark gpiod_direction_input/output() and * gpiod_get_direction() with might_sleep(), we'll be able to protect @@ -3520,16 +3540,8 @@ EXPORT_SYMBOL_GPL(gpiod_cansleep); int gpiod_set_consumer_name(struct gpio_desc *desc, const char *name) { VALIDATE_DESC(desc); - if (name) { - name = kstrdup_const(name, GFP_KERNEL); - if (!name) - return -ENOMEM; - } - kfree_const(desc->label); - desc_set_label(desc, name); - - return 0; + return desc_set_label(desc, name); } EXPORT_SYMBOL_GPL(gpiod_set_consumer_name); @@ -4739,6 +4751,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) int value; for_each_gpio_desc(gc, desc) { + guard(srcu)(&desc->srcu); if (test_bit(FLAG_REQUESTED, &desc->flags)) { gpiod_get_direction(desc); is_out = test_bit(FLAG_IS_OUT, &desc->flags); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 6e14b629c48b..d2e73eea9e92 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -180,7 +180,7 @@ struct gpio_desc { #define FLAG_EVENT_CLOCK_HTE 19 /* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ - const char *label; + const char __rcu *label; /* Name of the GPIO */ const char *name; #ifdef CONFIG_OF_DYNAMIC @@ -223,15 +223,29 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc) /* With descriptor prefix */ -#define gpiod_err(desc, fmt, ...) \ - pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) -#define gpiod_warn(desc, fmt, ...) \ - pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) -#define gpiod_dbg(desc, fmt, ...) \ - pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?",\ - ##__VA_ARGS__) +#define gpiod_err(desc, fmt, ...) \ +do { \ + scoped_guard(srcu, &desc->srcu) { \ + pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ + gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ + } \ +} while (0) + +#define gpiod_warn(desc, fmt, ...) \ +do { \ + scoped_guard(srcu, &desc->srcu) { \ + pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ + gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ + } \ +} while (0) + +#define gpiod_dbg(desc, fmt, ...) \ +do { \ + scoped_guard(srcu, &desc->srcu) { \ + pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ + gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ + } \ +} while (0) /* With chip prefix */ From 2a9101e875bc3aa6423b559e0ea43b2077f3be87 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 16:29:39 +0100 Subject: [PATCH 0257/1038] gpio: sysfs: use gpio_device_find() to iterate over existing devices With the list of GPIO devices now protected with SRCU we can use gpio_device_find() to traverse it from sysfs. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-sysfs.c | 46 ++++++++++++++++++------------------ drivers/gpio/gpiolib.c | 2 +- drivers/gpio/gpiolib.h | 1 - 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 6bf5332136e5..54b77ae1d176 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -790,11 +790,29 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) } } +/* + * We're not really looking for a device - we just want to iterate over the + * list and call this callback for each GPIO device. This is why this function + * always returns 0. + */ +static int gpiofind_sysfs_register(struct gpio_chip *gc, const void *data) +{ + struct gpio_device *gdev = gc->gpiodev; + int ret; + + if (gdev->mockdev) + return 0; + + ret = gpiochip_sysfs_register(gdev); + if (ret) + chip_err(gc, "failed to register the sysfs entry: %d\n", ret); + + return 0; +} + static int __init gpiolib_sysfs_init(void) { - int status; - unsigned long flags; - struct gpio_device *gdev; + int status; status = class_register(&gpio_class); if (status < 0) @@ -806,26 +824,8 @@ static int __init gpiolib_sysfs_init(void) * We run before arch_initcall() so chip->dev nodes can have * registered, and so arch_initcall() can always gpiod_export(). */ - spin_lock_irqsave(&gpio_lock, flags); - list_for_each_entry(gdev, &gpio_devices, list) { - if (gdev->mockdev) - continue; + (void)gpio_device_find(NULL, gpiofind_sysfs_register); - /* - * TODO we yield gpio_lock here because - * gpiochip_sysfs_register() acquires a mutex. This is unsafe - * and needs to be fixed. - * - * Also it would be nice to use gpio_device_find() here so we - * can keep gpio_chips local to gpiolib.c, but the yield of - * gpio_lock prevents us from doing this. - */ - spin_unlock_irqrestore(&gpio_lock, flags); - status = gpiochip_sysfs_register(gdev); - spin_lock_irqsave(&gpio_lock, flags); - } - spin_unlock_irqrestore(&gpio_lock, flags); - - return status; + return 0; } postcore_initcall(gpiolib_sysfs_init); diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index c72f3ffbe6b4..f0cbbab17dc3 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -85,7 +85,7 @@ DEFINE_SPINLOCK(gpio_lock); static DEFINE_MUTEX(gpio_lookup_lock); static LIST_HEAD(gpio_lookup_list); -LIST_HEAD(gpio_devices); +static LIST_HEAD(gpio_devices); /* Protects the GPIO device list against concurrent modifications. */ static DEFINE_MUTEX(gpio_devices_lock); /* Ensures coherence during read-only accesses to the list of GPIO devices. */ diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index d2e73eea9e92..2bf3f9e13ae4 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -136,7 +136,6 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, int gpiod_set_transitory(struct gpio_desc *desc, bool transitory); extern spinlock_t gpio_lock; -extern struct list_head gpio_devices; void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); From 35b545332b809a439aa1bf3fde5305e6eb243cf0 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 12 Jan 2024 14:49:04 +0100 Subject: [PATCH 0258/1038] gpio: remove gpio_lock The "multi-function" gpio_lock is pretty much useless with how it's used in GPIOLIB currently. Because many GPIO API calls can be called from all contexts but may also call into sleeping driver callbacks, there are many places with utterly broken workarounds like yielding the lock to call a possibly sleeping function and then re-acquiring it again without taking into account that the protected state may have changed. It was also used to protect several unrelated things: like individual descriptors AND the GPIO device list. We now serialize access to these two with SRCU and so can finally remove the spinlock. There is of course the question of consistency of lockless access to GPIO descriptors. Because we only support exclusive access to GPIOs (officially anyway, I'm looking at you broken GPIOD_FLAGS_BIT_NONEXCLUSIVE bit...) and the API contract with providers does not guarantee serialization, it's enough to ensure we cannot accidentally dereference an invalid pointer and that the state we present to both users and providers remains consistent. To achieve that: read the flags field atomically except for a few special cases. Read their current value before executing callback code and use this value for any subsequent logic. Modifying the flags depends on the particular use-case and can differ. For instance: when requesting a GPIO, we need to set the REQUESTED bit immediately so that the next user trying to request the same line sees -EBUSY. While at it: the allocations that used GFP_ATOMIC until this point can now switch to GFP_KERNEL. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-cdev.c | 18 +++--- drivers/gpio/gpiolib-sysfs.c | 17 ++---- drivers/gpio/gpiolib.c | 106 +++++++++++------------------------ drivers/gpio/gpiolib.h | 2 - 4 files changed, 46 insertions(+), 97 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 75f4912339a6..3588aaf90e45 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2302,18 +2302,16 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, memset(info, 0, sizeof(*info)); info->offset = gpio_chip_hwgpio(desc); - scoped_guard(spinlock_irqsave, &gpio_lock) { - if (desc->name) - strscpy(info->name, desc->name, sizeof(info->name)); + if (desc->name) + strscpy(info->name, desc->name, sizeof(info->name)); - scoped_guard(srcu, &desc->srcu) { - label = gpiod_get_label(desc); - if (label) - strscpy(info->consumer, label, - sizeof(info->consumer)); - } + dflags = READ_ONCE(desc->flags); - dflags = READ_ONCE(desc->flags); + scoped_guard(srcu, &desc->srcu) { + label = gpiod_get_label(desc); + if (label && test_bit(FLAG_REQUESTED, &dflags)) + strscpy(info->consumer, label, + sizeof(info->consumer)); } /* diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 54b77ae1d176..3ae7704b2996 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -563,7 +563,6 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) struct gpio_device *gdev; struct gpiod_data *data; struct gpio_chip *chip; - unsigned long flags; struct device *dev; int status, offset; @@ -578,6 +577,9 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) return -EINVAL; } + if (!test_and_set_bit(FLAG_EXPORT, &desc->flags)) + return -EPERM; + gdev = desc->gdev; chip = gdev->chip; @@ -589,18 +591,11 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) goto err_unlock; } - spin_lock_irqsave(&gpio_lock, flags); - if (!test_bit(FLAG_REQUESTED, &desc->flags) || - test_bit(FLAG_EXPORT, &desc->flags)) { - spin_unlock_irqrestore(&gpio_lock, flags); - gpiod_dbg(desc, "%s: unavailable (requested=%d, exported=%d)\n", - __func__, - test_bit(FLAG_REQUESTED, &desc->flags), - test_bit(FLAG_EXPORT, &desc->flags)); + if (!test_bit(FLAG_REQUESTED, &desc->flags)) { + gpiod_dbg(desc, "%s: unavailable (not requested)\n", __func__); status = -EPERM; goto err_unlock; } - spin_unlock_irqrestore(&gpio_lock, flags); data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { @@ -628,7 +623,6 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) goto err_free_data; } - set_bit(FLAG_EXPORT, &desc->flags); mutex_unlock(&sysfs_lock); return 0; @@ -636,6 +630,7 @@ err_free_data: kfree(data); err_unlock: mutex_unlock(&sysfs_lock); + clear_bit(FLAG_EXPORT, &desc->flags); gpiod_dbg(desc, "%s: status %d\n", __func__, status); return status; } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index f0cbbab17dc3..04b4636baff1 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -76,12 +76,6 @@ static const struct bus_type gpio_bus_type = { */ #define FASTPATH_NGPIO CONFIG_GPIOLIB_FASTPATH_LIMIT -/* gpio_lock prevents conflicts during gpio_desc[] table updates. - * While any GPIO is requested, its gpio_chip is not removable; - * each GPIO's "requested" flag serves as a lock and refcount. - */ -DEFINE_SPINLOCK(gpio_lock); - static DEFINE_MUTEX(gpio_lookup_lock); static LIST_HEAD(gpio_lookup_list); @@ -123,8 +117,7 @@ static int desc_set_label(struct gpio_desc *desc, const char *label) const char *new = NULL, *old; if (label) { - /* FIXME: make this GFP_KERNEL once the spinlock is out. */ - new = kstrdup_const(label, GFP_ATOMIC); + new = kstrdup_const(label, GFP_KERNEL); if (!new) return -ENOMEM; } @@ -1093,7 +1086,6 @@ EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key); void gpiochip_remove(struct gpio_chip *gc) { struct gpio_device *gdev = gc->gpiodev; - unsigned long flags; unsigned int i; down_write(&gdev->sem); @@ -1119,12 +1111,10 @@ void gpiochip_remove(struct gpio_chip *gc) */ gpiochip_set_data(gc, NULL); - spin_lock_irqsave(&gpio_lock, flags); for (i = 0; i < gdev->ngpio; i++) { if (test_bit(FLAG_REQUESTED, &gdev->descs[i].flags)) break; } - spin_unlock_irqrestore(&gpio_lock, flags); if (i != gdev->ngpio) dev_crit(&gdev->dev, @@ -2227,62 +2217,43 @@ EXPORT_SYMBOL_GPL(gpiochip_remove_pin_ranges); static int gpiod_request_commit(struct gpio_desc *desc, const char *label) { struct gpio_chip *gc = desc->gdev->chip; - unsigned long flags; unsigned int offset; int ret; + if (test_and_set_bit(FLAG_REQUESTED, &desc->flags)) + return -EBUSY; + if (label) { label = kstrdup_const(label, GFP_KERNEL); if (!label) return -ENOMEM; } - spin_lock_irqsave(&gpio_lock, flags); - /* NOTE: gpio_request() can be called in early boot, * before IRQs are enabled, for non-sleeping (SOC) GPIOs. */ - if (test_and_set_bit(FLAG_REQUESTED, &desc->flags)) { - ret = -EBUSY; - goto out_free_unlock; - } - if (gc->request) { - /* gc->request may sleep */ - spin_unlock_irqrestore(&gpio_lock, flags); offset = gpio_chip_hwgpio(desc); if (gpiochip_line_is_valid(gc, offset)) ret = gc->request(gc, offset); else ret = -EINVAL; - spin_lock_irqsave(&gpio_lock, flags); + if (ret) + goto out_clear_bit; + } - if (ret) { - desc_set_label(desc, NULL); - clear_bit(FLAG_REQUESTED, &desc->flags); - goto out_free_unlock; - } - } - if (gc->get_direction) { - /* gc->get_direction may sleep */ - spin_unlock_irqrestore(&gpio_lock, flags); + if (gc->get_direction) gpiod_get_direction(desc); - spin_lock_irqsave(&gpio_lock, flags); - } - spin_unlock_irqrestore(&gpio_lock, flags); ret = desc_set_label(desc, label ? : "?"); - if (ret) { - clear_bit(FLAG_REQUESTED, &desc->flags); - return ret; - } + if (ret) + goto out_clear_bit; return 0; -out_free_unlock: - spin_unlock_irqrestore(&gpio_lock, flags); - kfree_const(label); +out_clear_bit: + clear_bit(FLAG_REQUESTED, &desc->flags); return ret; } @@ -2352,35 +2323,32 @@ static bool gpiod_free_commit(struct gpio_desc *desc) might_sleep(); - spin_lock_irqsave(&gpio_lock, flags); - gc = desc->gdev->chip; - if (gc && test_bit(FLAG_REQUESTED, &desc->flags)) { - if (gc->free) { - spin_unlock_irqrestore(&gpio_lock, flags); - might_sleep_if(gc->can_sleep); + flags = READ_ONCE(desc->flags); + + if (gc && test_bit(FLAG_REQUESTED, &flags)) { + if (gc->free) gc->free(gc, gpio_chip_hwgpio(desc)); - spin_lock_irqsave(&gpio_lock, flags); - } - clear_bit(FLAG_ACTIVE_LOW, &desc->flags); - clear_bit(FLAG_REQUESTED, &desc->flags); - clear_bit(FLAG_OPEN_DRAIN, &desc->flags); - clear_bit(FLAG_OPEN_SOURCE, &desc->flags); - clear_bit(FLAG_PULL_UP, &desc->flags); - clear_bit(FLAG_PULL_DOWN, &desc->flags); - clear_bit(FLAG_BIAS_DISABLE, &desc->flags); - clear_bit(FLAG_EDGE_RISING, &desc->flags); - clear_bit(FLAG_EDGE_FALLING, &desc->flags); - clear_bit(FLAG_IS_HOGGED, &desc->flags); + + clear_bit(FLAG_ACTIVE_LOW, &flags); + clear_bit(FLAG_REQUESTED, &flags); + clear_bit(FLAG_OPEN_DRAIN, &flags); + clear_bit(FLAG_OPEN_SOURCE, &flags); + clear_bit(FLAG_PULL_UP, &flags); + clear_bit(FLAG_PULL_DOWN, &flags); + clear_bit(FLAG_BIAS_DISABLE, &flags); + clear_bit(FLAG_EDGE_RISING, &flags); + clear_bit(FLAG_EDGE_FALLING, &flags); + clear_bit(FLAG_IS_HOGGED, &flags); #ifdef CONFIG_OF_DYNAMIC WRITE_ONCE(desc->hog, NULL); #endif ret = true; - } + desc_set_label(desc, NULL); + WRITE_ONCE(desc->flags, flags); - spin_unlock_irqrestore(&gpio_lock, flags); - desc_set_label(desc, NULL); - gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED); + gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED); + } return ret; } @@ -2422,22 +2390,12 @@ char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset) if (IS_ERR(desc)) return NULL; - guard(spinlock_irqsave)(&gpio_lock); - if (!test_bit(FLAG_REQUESTED, &desc->flags)) return NULL; guard(srcu)(&desc->srcu); - /* - * FIXME: Once we mark gpiod_direction_input/output() and - * gpiod_get_direction() with might_sleep(), we'll be able to protect - * the GPIO descriptors with mutex (while value setting operations will - * become lockless). - * - * Until this happens, this allocation needs to be atomic. - */ - label = kstrdup(gpiod_get_label(desc), GFP_ATOMIC); + label = kstrdup(gpiod_get_label(desc), GFP_KERNEL); if (!label) return ERR_PTR(-ENOMEM); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 2bf3f9e13ae4..9b7afe87f1bd 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -135,8 +135,6 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, int gpiod_set_transitory(struct gpio_desc *desc, bool transitory); -extern spinlock_t gpio_lock; - void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); /** From 2559f2e09211dd4a5fde5bc5749ef4714fe38f7f Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 23 Jan 2024 14:26:09 +0100 Subject: [PATCH 0259/1038] gpio: reinforce desc->flags handling We now removed the gpio_lock spinlock and modified the places previously protected by it to handle desc->flags access in a consistent way. Let's improve other places that were previously unprotected by reading the flags field of gpio_desc once and using the stored value for logic consistency. If we need to modify the field, let's also write it back once with a consistent value resulting from the function's logic. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 04b4636baff1..cfbaf47b47d2 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -336,18 +336,20 @@ static int gpiochip_find_base_unlocked(int ngpio) int gpiod_get_direction(struct gpio_desc *desc) { struct gpio_chip *gc; + unsigned long flags; unsigned int offset; int ret; gc = gpiod_to_chip(desc); offset = gpio_chip_hwgpio(desc); + flags = READ_ONCE(desc->flags); /* * Open drain emulation using input mode may incorrectly report * input here, fix that up. */ - if (test_bit(FLAG_OPEN_DRAIN, &desc->flags) && - test_bit(FLAG_IS_OUT, &desc->flags)) + if (test_bit(FLAG_OPEN_DRAIN, &flags) && + test_bit(FLAG_IS_OUT, &flags)) return 0; if (!gc->get_direction) @@ -361,7 +363,8 @@ int gpiod_get_direction(struct gpio_desc *desc) if (ret > 0) ret = 1; - assign_bit(FLAG_IS_OUT, &desc->flags, !ret); + assign_bit(FLAG_IS_OUT, &flags, !ret); + WRITE_ONCE(desc->flags, flags); return ret; } @@ -751,9 +754,6 @@ static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog) return; } - if (test_bit(FLAG_IS_HOGGED, &desc->flags)) - return; - rv = gpiod_hog(desc, hog->line_name, hog->lflags, hog->dflags); if (rv) gpiod_err(desc, "%s: unable to hog GPIO line (%s:%u): %d\n", @@ -2528,13 +2528,16 @@ static int gpio_set_config(struct gpio_desc *desc, enum pin_config_param mode) static int gpio_set_bias(struct gpio_desc *desc) { enum pin_config_param bias; + unsigned long flags; unsigned int arg; - if (test_bit(FLAG_BIAS_DISABLE, &desc->flags)) + flags = READ_ONCE(desc->flags); + + if (test_bit(FLAG_BIAS_DISABLE, &flags)) bias = PIN_CONFIG_BIAS_DISABLE; - else if (test_bit(FLAG_PULL_UP, &desc->flags)) + else if (test_bit(FLAG_PULL_UP, &flags)) bias = PIN_CONFIG_BIAS_PULL_UP; - else if (test_bit(FLAG_PULL_DOWN, &desc->flags)) + else if (test_bit(FLAG_PULL_DOWN, &flags)) bias = PIN_CONFIG_BIAS_PULL_DOWN; else return 0; @@ -2700,24 +2703,28 @@ EXPORT_SYMBOL_GPL(gpiod_direction_output_raw); */ int gpiod_direction_output(struct gpio_desc *desc, int value) { + unsigned long flags; int ret; VALIDATE_DESC(desc); - if (test_bit(FLAG_ACTIVE_LOW, &desc->flags)) + + flags = READ_ONCE(desc->flags); + + if (test_bit(FLAG_ACTIVE_LOW, &flags)) value = !value; else value = !!value; /* GPIOs used for enabled IRQs shall not be set as output */ - if (test_bit(FLAG_USED_AS_IRQ, &desc->flags) && - test_bit(FLAG_IRQ_IS_ENABLED, &desc->flags)) { + if (test_bit(FLAG_USED_AS_IRQ, &flags) && + test_bit(FLAG_IRQ_IS_ENABLED, &flags)) { gpiod_err(desc, "%s: tried to set a GPIO tied to an IRQ as output\n", __func__); return -EIO; } - if (test_bit(FLAG_OPEN_DRAIN, &desc->flags)) { + if (test_bit(FLAG_OPEN_DRAIN, &flags)) { /* First see if we can enable open drain in hardware */ ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_DRAIN); if (!ret) @@ -2727,7 +2734,7 @@ int gpiod_direction_output(struct gpio_desc *desc, int value) ret = gpiod_direction_input(desc); goto set_output_flag; } - } else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags)) { + } else if (test_bit(FLAG_OPEN_SOURCE, &flags)) { ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_SOURCE); if (!ret) goto set_output_value; @@ -4424,21 +4431,22 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, int hwnum; int ret; + if (test_and_set_bit(FLAG_IS_HOGGED, &desc->flags)) + return 0; + gc = gpiod_to_chip(desc); hwnum = gpio_chip_hwgpio(desc); local_desc = gpiochip_request_own_desc(gc, hwnum, name, lflags, dflags); if (IS_ERR(local_desc)) { + clear_bit(FLAG_IS_HOGGED, &desc->flags); ret = PTR_ERR(local_desc); pr_err("requesting hog GPIO %s (chip %s, offset %d) failed, %d\n", name, gc->label, hwnum, ret); return ret; } - /* Mark GPIO as hogged so it can be identified and removed later */ - set_bit(FLAG_IS_HOGGED, &desc->flags); - gpiod_dbg(desc, "hogged as %s%s\n", (dflags & GPIOD_FLAGS_BIT_DIR_OUT) ? "output" : "input", (dflags & GPIOD_FLAGS_BIT_DIR_OUT) ? From b6f87adbacfab9001d08e56ac869e1c75734633d Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 23 Jan 2024 17:52:36 +0100 Subject: [PATCH 0260/1038] gpio: remove unneeded code from gpio_device_get_desc() The GPIO chip pointer is unused. Let's remove it. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index cfbaf47b47d2..c6966c55816a 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -184,16 +184,6 @@ EXPORT_SYMBOL_GPL(gpiochip_get_desc); struct gpio_desc * gpio_device_get_desc(struct gpio_device *gdev, unsigned int hwnum) { - struct gpio_chip *gc; - - /* - * FIXME: This will be locked once we protect gdev->chip everywhere - * with SRCU. - */ - gc = gdev->chip; - if (!gc) - return ERR_PTR(-ENODEV); - if (hwnum >= gdev->ngpio) return ERR_PTR(-EINVAL); From 59cba4a0e6ca1058fbf88fec22530a4e2841802a Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 14:08:45 +0100 Subject: [PATCH 0261/1038] gpio: sysfs: extend the critical section for unregistering sysfs devices Checking the gdev->mockdev pointer for NULL must be part of the critical section. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-sysfs.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 3ae7704b2996..12ecb49fe75d 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -768,15 +769,15 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) struct gpio_desc *desc; struct gpio_chip *chip = gdev->chip; - if (!gdev->mockdev) - return; + scoped_guard(mutex, &sysfs_lock) { + if (!gdev->mockdev) + return; - device_unregister(gdev->mockdev); + device_unregister(gdev->mockdev); - /* prevent further gpiod exports */ - mutex_lock(&sysfs_lock); - gdev->mockdev = NULL; - mutex_unlock(&sysfs_lock); + /* prevent further gpiod exports */ + gdev->mockdev = NULL; + } /* unregister gpiod class devices owned by sysfs */ for_each_gpio_desc_with_flag(chip, desc, FLAG_SYSFS) { From b93bca4bd6d221d5f34a20d69ec0f1ec636ae7df Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 15:11:38 +0100 Subject: [PATCH 0262/1038] gpio: sysfs: pass the GPIO device - not chip - to sysfs callbacks We're working towards protecting the chip pointer in struct gpio_device with SRCU. In order to use it in sysfs callbacks we must pass the pointer to the GPIO device that wraps the chip instead of the address of the chip itself as the user data. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-sysfs.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 12ecb49fe75d..84b8a7555791 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -401,27 +401,27 @@ static const struct attribute_group *gpio_groups[] = { static ssize_t base_show(struct device *dev, struct device_attribute *attr, char *buf) { - const struct gpio_chip *chip = dev_get_drvdata(dev); + const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%d\n", chip->base); + return sysfs_emit(buf, "%d\n", gdev->chip->base); } static DEVICE_ATTR_RO(base); static ssize_t label_show(struct device *dev, struct device_attribute *attr, char *buf) { - const struct gpio_chip *chip = dev_get_drvdata(dev); + const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%s\n", chip->label ?: ""); + return sysfs_emit(buf, "%s\n", gdev->chip->label ?: ""); } static DEVICE_ATTR_RO(label); static ssize_t ngpio_show(struct device *dev, struct device_attribute *attr, char *buf) { - const struct gpio_chip *chip = dev_get_drvdata(dev); + const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%u\n", chip->ngpio); + return sysfs_emit(buf, "%u\n", gdev->chip->ngpio); } static DEVICE_ATTR_RO(ngpio); @@ -751,7 +751,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) parent = &gdev->dev; /* use chip->base for the ID; it's already known to be unique */ - dev = device_create_with_groups(&gpio_class, parent, MKDEV(0, 0), chip, + dev = device_create_with_groups(&gpio_class, parent, MKDEV(0, 0), gdev, gpiochip_groups, GPIOCHIP_NAME "%d", chip->base); if (IS_ERR(dev)) From f4e14d45d7fe95f6c1c136baf9e232234669dfed Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 17:13:51 +0100 Subject: [PATCH 0263/1038] gpio: cdev: replace gpiochip_get_desc() with gpio_device_get_desc() gpio_device_get_desc() is the safer alternative to gpiochip_get_desc(). As we don't really need to dereference the chip pointer to retrieve the descriptors in character device code, let's use it. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-cdev.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 3588aaf90e45..8e37e3befa08 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -332,7 +332,7 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) /* Request each GPIO */ for (i = 0; i < handlereq.lines; i++) { u32 offset = handlereq.lineoffsets[i]; - struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset); + struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) { ret = PTR_ERR(desc); @@ -1739,7 +1739,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) /* Request each GPIO */ for (i = 0; i < ulr.num_lines; i++) { u32 offset = ulr.offsets[i]; - struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset); + struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) { ret = PTR_ERR(desc); @@ -2123,7 +2123,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) lflags = eventreq.handleflags; eflags = eventreq.eventflags; - desc = gpiochip_get_desc(gdev->chip, offset); + desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2419,7 +2419,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, return -EFAULT; /* this doubles as a range check on line_offset */ - desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.line_offset); + desc = gpio_device_get_desc(cdev->gdev, lineinfo.line_offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2456,7 +2456,7 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, if (memchr_inv(lineinfo.padding, 0, sizeof(lineinfo.padding))) return -EINVAL; - desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.offset); + desc = gpio_device_get_desc(cdev->gdev, lineinfo.offset); if (IS_ERR(desc)) return PTR_ERR(desc); From 3c7a47f6c5f0d3e3c6149b7eb920f420a741e969 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 17:21:16 +0100 Subject: [PATCH 0264/1038] gpio: cdev: don't access gdev->chip if it's not needed The variable holding the number of GPIO lines is duplicated in GPIO device so read it instead of unnecessarily dereferencing the chip pointer. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-cdev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 8e37e3befa08..e993c6a7215a 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2701,7 +2701,7 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file) if (!cdev) return -ENODEV; - cdev->watched_lines = bitmap_zalloc(gdev->chip->ngpio, GFP_KERNEL); + cdev->watched_lines = bitmap_zalloc(gdev->ngpio, GFP_KERNEL); if (!cdev->watched_lines) goto out_free_cdev; From 5694f274a060ebd0e7128c11501281197fb7a5ce Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 5 Feb 2024 20:28:48 +0100 Subject: [PATCH 0265/1038] gpio: sysfs: don't access gdev->chip if it's not needed Don't dereference gdev->chip if the same information can be obtained from struct gpio_device. Suggested-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-sysfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 84b8a7555791..c1f96a14fe7d 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -403,7 +403,7 @@ static ssize_t base_show(struct device *dev, { const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%d\n", gdev->chip->base); + return sysfs_emit(buf, "%d\n", gdev->base); } static DEVICE_ATTR_RO(base); @@ -412,7 +412,7 @@ static ssize_t label_show(struct device *dev, { const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%s\n", gdev->chip->label ?: ""); + return sysfs_emit(buf, "%s\n", gdev->label); } static DEVICE_ATTR_RO(label); @@ -421,7 +421,7 @@ static ssize_t ngpio_show(struct device *dev, { const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%u\n", gdev->chip->ngpio); + return sysfs_emit(buf, "%u\n", gdev->ngpio); } static DEVICE_ATTR_RO(ngpio); From 7fe595b3c3cf3f9b8f21fce72f1f48a2cb41522e Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 31 Jan 2024 11:40:57 +0100 Subject: [PATCH 0266/1038] gpio: don't dereference gdev->chip in gpiochip_setup_dev() We don't need to dereference gdev->chip in gpiochip_setup_dev() as at the time it's called, the label in the associated struct gpio_device is already set. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index c6966c55816a..9fc9cfac7081 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -723,7 +723,7 @@ static int gpiochip_setup_dev(struct gpio_device *gdev) goto err_remove_device; dev_dbg(&gdev->dev, "registered GPIOs %d to %d on %s\n", gdev->base, - gdev->base + gdev->ngpio - 1, gdev->chip->label ? : "generic"); + gdev->base + gdev->ngpio - 1, gdev->label); return 0; From 6c82e737ab21c82982eaf5591b07268927a085d1 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 21:22:19 +0100 Subject: [PATCH 0267/1038] gpio: reduce the functionality of validate_desc() Checking desc->gdev->chip for NULL without holding it in place with some serializing mechanism is pointless. Remove this check. Also don't check desc->gdev for NULL as it can never happen. We'll be protecting gdev->chip with SRCU soon but we will provide a dedicated, automatic class for that. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 9fc9cfac7081..5db3ed29dae6 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2257,19 +2257,12 @@ static int validate_desc(const struct gpio_desc *desc, const char *func) { if (!desc) return 0; + if (IS_ERR(desc)) { pr_warn("%s: invalid GPIO (errorpointer)\n", func); return PTR_ERR(desc); } - if (!desc->gdev) { - pr_warn("%s: invalid GPIO (no device)\n", func); - return -EINVAL; - } - if (!desc->gdev->chip) { - dev_warn(&desc->gdev->dev, - "%s: backing chip is gone\n", func); - return 0; - } + return 1; } @@ -2345,12 +2338,7 @@ static bool gpiod_free_commit(struct gpio_desc *desc) void gpiod_free(struct gpio_desc *desc) { - /* - * We must not use VALIDATE_DESC_VOID() as the underlying gdev->chip - * may already be NULL but we still want to put the references. - */ - if (!desc) - return; + VALIDATE_DESC_VOID(desc); if (!gpiod_free_commit(desc)) WARN_ON(1); From c5cf334dcc78185da08b1f5df2d754cc7bf02669 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 25 Jan 2024 15:05:19 +0100 Subject: [PATCH 0268/1038] gpio: remove unnecessary checks from gpiod_to_chip() We don't need to check the gdev pointer in struct gpio_desc - it's always assigned and never cleared. It's also pointless to check gdev->chip before we actually serialize access to it. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 5db3ed29dae6..76c27ba9ee64 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -214,7 +214,7 @@ EXPORT_SYMBOL_GPL(desc_to_gpio); */ struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc) { - if (!desc || !desc->gdev) + if (!desc) return NULL; return desc->gdev->chip; } @@ -3505,7 +3505,7 @@ int gpiod_to_irq(const struct gpio_desc *desc) * requires this function to not return zero on an invalid descriptor * but rather a negative error number. */ - if (!desc || IS_ERR(desc) || !desc->gdev || !desc->gdev->chip) + if (!desc || IS_ERR(desc)) return -EINVAL; gc = desc->gdev->chip; From 8a5b477bb3e9c891eb064accabb3162ccf2c590e Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Sat, 27 Jan 2024 22:14:15 +0100 Subject: [PATCH 0269/1038] gpio: add the can_sleep flag to struct gpio_device Duplicating the can_sleep value in GPIO device will allow us to not needlessly dereference the chip pointer in several places and reduce the number of SRCU read-only critical sections. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 11 ++++++----- drivers/gpio/gpiolib.h | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 76c27ba9ee64..40052de2361c 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -901,6 +901,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, } gdev->ngpio = gc->ngpio; + gdev->can_sleep = gc->can_sleep; scoped_guard(mutex, &gpio_devices_lock) { /* @@ -3072,7 +3073,7 @@ int gpiod_get_raw_value(const struct gpio_desc *desc) { VALIDATE_DESC(desc); /* Should be using gpiod_get_raw_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); return gpiod_get_raw_value_commit(desc); } EXPORT_SYMBOL_GPL(gpiod_get_raw_value); @@ -3093,7 +3094,7 @@ int gpiod_get_value(const struct gpio_desc *desc) VALIDATE_DESC(desc); /* Should be using gpiod_get_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); value = gpiod_get_raw_value_commit(desc); if (value < 0) @@ -3366,7 +3367,7 @@ void gpiod_set_raw_value(struct gpio_desc *desc, int value) { VALIDATE_DESC_VOID(desc); /* Should be using gpiod_set_raw_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); gpiod_set_raw_value_commit(desc, value); } EXPORT_SYMBOL_GPL(gpiod_set_raw_value); @@ -3407,7 +3408,7 @@ void gpiod_set_value(struct gpio_desc *desc, int value) { VALIDATE_DESC_VOID(desc); /* Should be using gpiod_set_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); gpiod_set_value_nocheck(desc, value); } EXPORT_SYMBOL_GPL(gpiod_set_value); @@ -3471,7 +3472,7 @@ EXPORT_SYMBOL_GPL(gpiod_set_array_value); int gpiod_cansleep(const struct gpio_desc *desc) { VALIDATE_DESC(desc); - return desc->gdev->chip->can_sleep; + return desc->gdev->can_sleep; } EXPORT_SYMBOL_GPL(gpiod_cansleep); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 9b7afe87f1bd..43ff4931e2c3 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -34,6 +34,8 @@ * @descs: array of ngpio descriptors. * @ngpio: the number of GPIO lines on this GPIO device, equal to the size * of the @descs array. + * @can_sleep: indicate whether the GPIO chip driver's callbacks can sleep + * implying that they cannot be used from atomic context * @base: GPIO base in the DEPRECATED global Linux GPIO numberspace, assigned * at device creation time. * @label: a descriptive name for the GPIO device, such as the part number @@ -64,6 +66,7 @@ struct gpio_device { struct gpio_desc *descs; int base; u16 ngpio; + bool can_sleep; const char *label; void *data; struct list_head list; From 47d8b4c1d868148c8fb51b785a89e58ca2d02c4d Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 5 Jan 2024 20:42:44 +0100 Subject: [PATCH 0270/1038] gpio: add SRCU infrastructure to struct gpio_device Add the SRCU struct to GPIO device. It will be used to serialize access to the GPIO chip pointer. Initialize and clean it up where applicable. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 13 ++++++++++--- drivers/gpio/gpiolib.h | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 40052de2361c..4e0c6df4a880 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -680,6 +680,7 @@ static void gpiodev_release(struct device *dev) ida_free(&gpio_ida, gdev->id); kfree_const(gdev->label); kfree(gdev->descs); + cleanup_srcu_struct(&gdev->srcu); kfree(gdev); } @@ -948,6 +949,10 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); init_rwsem(&gdev->sem); + ret = init_srcu_struct(&gdev->srcu); + if (ret) + goto err_remove_from_list; + #ifdef CONFIG_PINCTRL INIT_LIST_HEAD(&gdev->pin_ranges); #endif @@ -955,15 +960,15 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (gc->names) { ret = gpiochip_set_desc_names(gc); if (ret) - goto err_remove_from_list; + goto err_cleanup_gdev_srcu; } ret = gpiochip_set_names(gc); if (ret) - goto err_remove_from_list; + goto err_cleanup_gdev_srcu; ret = gpiochip_init_valid_mask(gc); if (ret) - goto err_remove_from_list; + goto err_cleanup_gdev_srcu; ret = of_gpiochip_add(gc); if (ret) @@ -1038,6 +1043,8 @@ err_remove_of_chip: err_free_gpiochip_mask: gpiochip_remove_pin_ranges(gc); gpiochip_free_valid_mask(gc); +err_cleanup_gdev_srcu: + cleanup_srcu_struct(&gdev->srcu); err_remove_from_list: scoped_guard(mutex, &gpio_devices_lock) list_del_rcu(&gdev->list); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 43ff4931e2c3..35d71e30c546 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -49,6 +49,7 @@ * @sem: protects the structure from a NULL-pointer dereference of @chip by * user-space operations when the device gets unregistered during * a hot-unplug event + * @srcu: protects the pointer to the underlying GPIO chip * @pin_ranges: range of pins served by the GPIO driver * * This state container holds most of the runtime variable data @@ -73,6 +74,7 @@ struct gpio_device { struct blocking_notifier_head line_state_notifier; struct blocking_notifier_head device_notifier; struct rw_semaphore sem; + struct srcu_struct srcu; #ifdef CONFIG_PINCTRL /* From d83cee3d2bb131c7fca7e19f1e33dc7530fd7083 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 23 Jan 2024 12:01:10 +0100 Subject: [PATCH 0271/1038] gpio: protect the pointer to gpio_chip in gpio_device with SRCU Ensure we cannot crash if the GPIO device gets unregistered (and the chip pointer set to NULL) during any of the API calls. To that end: wait for all users of gdev->chip to exit their read-only SRCU critical sections in gpiochip_remove(). For brevity: add a guard class which can be instantiated at the top of every function requiring read-only access to the chip pointer and use it in all API calls taking a GPIO descriptor as argument. In places where we only deal with the GPIO device - use regular guard() helpers and rcu_dereference() for chip access. Do the same in API calls taking a const pointer to gpio_desc. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib-cdev.c | 64 +++++---- drivers/gpio/gpiolib-sysfs.c | 57 +++++--- drivers/gpio/gpiolib.c | 257 +++++++++++++++++++++++------------ drivers/gpio/gpiolib.h | 22 ++- 4 files changed, 271 insertions(+), 129 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index e993c6a7215a..9323b357df43 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -205,9 +204,9 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, unsigned int i; int ret; - guard(rwsem_read)(&lh->gdev->sem); + guard(srcu)(&lh->gdev->srcu); - if (!lh->gdev->chip) + if (!rcu_dereference(lh->gdev->chip)) return -ENODEV; switch (cmd) { @@ -1520,9 +1519,9 @@ static long linereq_ioctl(struct file *file, unsigned int cmd, struct linereq *lr = file->private_data; void __user *ip = (void __user *)arg; - guard(rwsem_read)(&lr->gdev->sem); + guard(srcu)(&lr->gdev->srcu); - if (!lr->gdev->chip) + if (!rcu_dereference(lr->gdev->chip)) return -ENODEV; switch (cmd) { @@ -1551,9 +1550,9 @@ static __poll_t linereq_poll(struct file *file, struct linereq *lr = file->private_data; __poll_t events = 0; - guard(rwsem_read)(&lr->gdev->sem); + guard(srcu)(&lr->gdev->srcu); - if (!lr->gdev->chip) + if (!rcu_dereference(lr->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &lr->wait, wait); @@ -1573,9 +1572,9 @@ static ssize_t linereq_read(struct file *file, char __user *buf, ssize_t bytes_read = 0; int ret; - guard(rwsem_read)(&lr->gdev->sem); + guard(srcu)(&lr->gdev->srcu); - if (!lr->gdev->chip) + if (!rcu_dereference(lr->gdev->chip)) return -ENODEV; if (count < sizeof(le)) @@ -1874,9 +1873,9 @@ static __poll_t lineevent_poll(struct file *file, struct lineevent_state *le = file->private_data; __poll_t events = 0; - guard(rwsem_read)(&le->gdev->sem); + guard(srcu)(&le->gdev->srcu); - if (!le->gdev->chip) + if (!rcu_dereference(le->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &le->wait, wait); @@ -1912,9 +1911,9 @@ static ssize_t lineevent_read(struct file *file, char __user *buf, ssize_t ge_size; int ret; - guard(rwsem_read)(&le->gdev->sem); + guard(srcu)(&le->gdev->srcu); - if (!le->gdev->chip) + if (!rcu_dereference(le->gdev->chip)) return -ENODEV; /* @@ -1995,9 +1994,9 @@ static long lineevent_ioctl(struct file *file, unsigned int cmd, void __user *ip = (void __user *)arg; struct gpiohandle_data ghd; - guard(rwsem_read)(&le->gdev->sem); + guard(srcu)(&le->gdev->srcu); - if (!le->gdev->chip) + if (!rcu_dereference(le->gdev->chip)) return -ENODEV; /* @@ -2295,10 +2294,13 @@ static void gpio_v2_line_info_changed_to_v1( static void gpio_desc_to_lineinfo(struct gpio_desc *desc, struct gpio_v2_line_info *info) { - struct gpio_chip *gc = desc->gdev->chip; unsigned long dflags; const char *label; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; + memset(info, 0, sizeof(*info)); info->offset = gpio_chip_hwgpio(desc); @@ -2331,8 +2333,8 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, test_bit(FLAG_USED_AS_IRQ, &dflags) || test_bit(FLAG_EXPORT, &dflags) || test_bit(FLAG_SYSFS, &dflags) || - !gpiochip_line_is_valid(gc, info->offset) || - !pinctrl_gpio_can_use_line(gc, info->offset)) + !gpiochip_line_is_valid(guard.gc, info->offset) || + !pinctrl_gpio_can_use_line(guard.gc, info->offset)) info->flags |= GPIO_V2_LINE_FLAG_USED; if (test_bit(FLAG_IS_OUT, &dflags)) @@ -2505,10 +2507,10 @@ static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) struct gpio_device *gdev = cdev->gdev; void __user *ip = (void __user *)arg; - guard(rwsem_read)(&gdev->sem); + guard(srcu)(&gdev->srcu); /* We fail any subsequent ioctl():s when the chip is gone */ - if (!gdev->chip) + if (!rcu_dereference(gdev->chip)) return -ENODEV; /* Fill in the struct and pass to userspace */ @@ -2591,9 +2593,9 @@ static __poll_t lineinfo_watch_poll(struct file *file, struct gpio_chardev_data *cdev = file->private_data; __poll_t events = 0; - guard(rwsem_read)(&cdev->gdev->sem); + guard(srcu)(&cdev->gdev->srcu); - if (!cdev->gdev->chip) + if (!rcu_dereference(cdev->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &cdev->wait, pollt); @@ -2614,9 +2616,9 @@ static ssize_t lineinfo_watch_read(struct file *file, char __user *buf, int ret; size_t event_size; - guard(rwsem_read)(&cdev->gdev->sem); + guard(srcu)(&cdev->gdev->srcu); - if (!cdev->gdev->chip) + if (!rcu_dereference(cdev->gdev->chip)) return -ENODEV; #ifndef CONFIG_GPIO_CDEV_V1 @@ -2691,10 +2693,10 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file) struct gpio_chardev_data *cdev; int ret = -ENOMEM; - guard(rwsem_read)(&gdev->sem); + guard(srcu)(&gdev->srcu); /* Fail on open if the backing gpiochip is gone */ - if (!gdev->chip) + if (!rcu_dereference(gdev->chip)) return -ENODEV; cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); @@ -2781,6 +2783,7 @@ static const struct file_operations gpio_fileops = { int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) { + struct gpio_chip *gc; int ret; cdev_init(&gdev->chrdev, &gpio_fileops); @@ -2791,8 +2794,13 @@ int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) if (ret) return ret; - chip_dbg(gdev->chip, "added GPIO chardev (%d:%d)\n", - MAJOR(devt), gdev->id); + guard(srcu)(&gdev->srcu); + + gc = rcu_dereference(gdev->chip); + if (!gc) + return -ENODEV; + + chip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); return 0; } diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index c1f96a14fe7d..6285fa5afbb1 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -171,6 +171,10 @@ static int gpio_sysfs_request_irq(struct device *dev, unsigned char flags) unsigned long irq_flags; int ret; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + data->irq = gpiod_to_irq(desc); if (data->irq < 0) return -EIO; @@ -195,7 +199,7 @@ static int gpio_sysfs_request_irq(struct device *dev, unsigned char flags) * Remove this redundant call (along with the corresponding * unlock) when those drivers have been fixed. */ - ret = gpiochip_lock_as_irq(desc->gdev->chip, gpio_chip_hwgpio(desc)); + ret = gpiochip_lock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); if (ret < 0) goto err_put_kn; @@ -209,7 +213,7 @@ static int gpio_sysfs_request_irq(struct device *dev, unsigned char flags) return 0; err_unlock: - gpiochip_unlock_as_irq(desc->gdev->chip, gpio_chip_hwgpio(desc)); + gpiochip_unlock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); err_put_kn: sysfs_put(data->value_kn); @@ -225,9 +229,13 @@ static void gpio_sysfs_free_irq(struct device *dev) struct gpiod_data *data = dev_get_drvdata(dev); struct gpio_desc *desc = data->desc; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; + data->irq_flags = 0; free_irq(data->irq, data); - gpiochip_unlock_as_irq(desc->gdev->chip, gpio_chip_hwgpio(desc)); + gpiochip_unlock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); sysfs_put(data->value_kn); } @@ -444,13 +452,12 @@ static ssize_t export_store(const struct class *class, const char *buf, size_t len) { struct gpio_desc *desc; - struct gpio_chip *gc; int status, offset; long gpio; status = kstrtol(buf, 0, &gpio); - if (status < 0) - goto done; + if (status) + return status; desc = gpio_to_desc(gpio); /* reject invalid GPIOs */ @@ -458,9 +465,13 @@ static ssize_t export_store(const struct class *class, pr_warn("%s: invalid GPIO %ld\n", __func__, gpio); return -EINVAL; } - gc = desc->gdev->chip; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + offset = gpio_chip_hwgpio(desc); - if (!gpiochip_line_is_valid(gc, offset)) { + if (!gpiochip_line_is_valid(guard.gc, offset)) { pr_warn("%s: GPIO %ld masked\n", __func__, gpio); return -EINVAL; } @@ -563,7 +574,6 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) const char *ioname = NULL; struct gpio_device *gdev; struct gpiod_data *data; - struct gpio_chip *chip; struct device *dev; int status, offset; @@ -578,16 +588,19 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) return -EINVAL; } + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + if (!test_and_set_bit(FLAG_EXPORT, &desc->flags)) return -EPERM; gdev = desc->gdev; - chip = gdev->chip; mutex_lock(&sysfs_lock); /* check if chip is being removed */ - if (!chip || !gdev->mockdev) { + if (!gdev->mockdev) { status = -ENODEV; goto err_unlock; } @@ -606,14 +619,14 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) data->desc = desc; mutex_init(&data->mutex); - if (chip->direction_input && chip->direction_output) + if (guard.gc->direction_input && guard.gc->direction_output) data->direction_can_change = direction_may_change; else data->direction_can_change = false; offset = gpio_chip_hwgpio(desc); - if (chip->names && chip->names[offset]) - ioname = chip->names[offset]; + if (guard.gc->names && guard.gc->names[offset]) + ioname = guard.gc->names[offset]; dev = device_create_with_groups(&gpio_class, &gdev->dev, MKDEV(0, 0), data, gpio_groups, @@ -728,7 +741,7 @@ EXPORT_SYMBOL_GPL(gpiod_unexport); int gpiochip_sysfs_register(struct gpio_device *gdev) { - struct gpio_chip *chip = gdev->chip; + struct gpio_chip *chip; struct device *parent; struct device *dev; @@ -741,6 +754,12 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) if (!class_is_registered(&gpio_class)) return 0; + guard(srcu)(&gdev->srcu); + + chip = rcu_dereference(gdev->chip); + if (!chip) + return -ENODEV; + /* * For sysfs backward compatibility we need to preserve this * preferred parenting to the gpio_chip parent field, if set. @@ -767,7 +786,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) void gpiochip_sysfs_unregister(struct gpio_device *gdev) { struct gpio_desc *desc; - struct gpio_chip *chip = gdev->chip; + struct gpio_chip *chip; scoped_guard(mutex, &sysfs_lock) { if (!gdev->mockdev) @@ -779,6 +798,12 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) gdev->mockdev = NULL; } + guard(srcu)(&gdev->srcu); + + chip = rcu_dereference(gdev->chip); + if (chip) + return; + /* unregister gpiod class devices owned by sysfs */ for_each_gpio_desc_with_flag(chip, desc, FLAG_SYSFS) { gpiod_unexport(desc); diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 4e0c6df4a880..1a9d045b5a6b 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -216,7 +216,7 @@ struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc) { if (!desc) return NULL; - return desc->gdev->chip; + return rcu_dereference(desc->gdev->chip); } EXPORT_SYMBOL_GPL(gpiod_to_chip); @@ -285,7 +285,7 @@ EXPORT_SYMBOL(gpio_device_get_label); */ struct gpio_chip *gpio_device_get_chip(struct gpio_device *gdev) { - return gdev->chip; + return rcu_dereference(gdev->chip); } EXPORT_SYMBOL_GPL(gpio_device_get_chip); @@ -325,12 +325,21 @@ static int gpiochip_find_base_unlocked(int ngpio) */ int gpiod_get_direction(struct gpio_desc *desc) { - struct gpio_chip *gc; unsigned long flags; unsigned int offset; int ret; - gc = gpiod_to_chip(desc); + /* + * We cannot use VALIDATE_DESC() as we must not return 0 for a NULL + * descriptor like we usually do. + */ + if (!desc || IS_ERR(desc)) + return -EINVAL; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + offset = gpio_chip_hwgpio(desc); flags = READ_ONCE(desc->flags); @@ -342,10 +351,10 @@ int gpiod_get_direction(struct gpio_desc *desc) test_bit(FLAG_IS_OUT, &flags)) return 0; - if (!gc->get_direction) + if (!guard.gc->get_direction) return -ENOTSUPP; - ret = gc->get_direction(gc, offset); + ret = guard.gc->get_direction(guard.gc, offset); if (ret < 0) return ret; @@ -421,6 +430,7 @@ static struct gpio_desc *gpio_name_to_desc(const char * const name) { struct gpio_device *gdev; struct gpio_desc *desc; + struct gpio_chip *gc; if (!name) return NULL; @@ -429,7 +439,13 @@ static struct gpio_desc *gpio_name_to_desc(const char * const name) list_for_each_entry_srcu(gdev, &gpio_devices, list, srcu_read_lock_held(&gpio_devices_srcu)) { - for_each_gpio_desc(gdev->chip, desc) { + guard(srcu)(&gdev->srcu); + + gc = rcu_dereference(gdev->chip); + if (!gc) + continue; + + for_each_gpio_desc(gc, desc) { if (desc->name && !strcmp(desc->name, name)) return desc; } @@ -853,7 +869,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev->dev.type = &gpio_dev_type; gdev->dev.bus = &gpio_bus_type; gdev->dev.parent = gc->parent; - gdev->chip = gc; + rcu_assign_pointer(gdev->chip, gc); gc->gpiodev = gdev; gpiochip_set_data(gc, data); @@ -1097,7 +1113,8 @@ void gpiochip_remove(struct gpio_chip *gc) synchronize_srcu(&gpio_devices_srcu); /* Numb the device, cancelling all outstanding operations */ - gdev->chip = NULL; + rcu_assign_pointer(gdev->chip, NULL); + synchronize_srcu(&gdev->srcu); gpiochip_irqchip_remove(gc); acpi_gpiochip_remove(gc); of_gpiochip_remove(gc); @@ -1156,6 +1173,7 @@ struct gpio_device *gpio_device_find(void *data, const void *data)) { struct gpio_device *gdev; + struct gpio_chip *gc; /* * Not yet but in the future the spinlock below will become a mutex. @@ -1166,8 +1184,13 @@ struct gpio_device *gpio_device_find(void *data, guard(srcu)(&gpio_devices_srcu); - list_for_each_entry(gdev, &gpio_devices, list) { - if (gdev->chip && match(gdev->chip, data)) + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + guard(srcu)(&gdev->srcu); + + gc = rcu_dereference(gdev->chip); + + if (gc && match(gc, data)) return gpio_device_get(gdev); } @@ -2214,10 +2237,13 @@ EXPORT_SYMBOL_GPL(gpiochip_remove_pin_ranges); */ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) { - struct gpio_chip *gc = desc->gdev->chip; unsigned int offset; int ret; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + if (test_and_set_bit(FLAG_REQUESTED, &desc->flags)) return -EBUSY; @@ -2231,17 +2257,17 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) * before IRQs are enabled, for non-sleeping (SOC) GPIOs. */ - if (gc->request) { + if (guard.gc->request) { offset = gpio_chip_hwgpio(desc); - if (gpiochip_line_is_valid(gc, offset)) - ret = gc->request(gc, offset); + if (gpiochip_line_is_valid(guard.gc, offset)) + ret = guard.gc->request(guard.gc, offset); else ret = -EINVAL; if (ret) goto out_clear_bit; } - if (gc->get_direction) + if (guard.gc->get_direction) gpiod_get_direction(desc); ret = desc_set_label(desc, label ? : "?"); @@ -2308,18 +2334,18 @@ int gpiod_request(struct gpio_desc *desc, const char *label) static bool gpiod_free_commit(struct gpio_desc *desc) { - struct gpio_chip *gc; unsigned long flags; bool ret = false; might_sleep(); - gc = desc->gdev->chip; + CLASS(gpio_chip_guard, guard)(desc); + flags = READ_ONCE(desc->flags); - if (gc && test_bit(FLAG_REQUESTED, &flags)) { - if (gc->free) - gc->free(gc, gpio_chip_hwgpio(desc)); + if (guard.gc && test_bit(FLAG_REQUESTED, &flags)) { + if (guard.gc->free) + guard.gc->free(guard.gc, gpio_chip_hwgpio(desc)); clear_bit(FLAG_ACTIVE_LOW, &flags); clear_bit(FLAG_REQUESTED, &flags); @@ -2476,11 +2502,14 @@ static int gpio_set_config_with_argument(struct gpio_desc *desc, enum pin_config_param mode, u32 argument) { - struct gpio_chip *gc = desc->gdev->chip; unsigned long config; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + config = pinconf_to_config_packed(mode, argument); - return gpio_do_set_config(gc, gpio_chip_hwgpio(desc), config); + return gpio_do_set_config(guard.gc, gpio_chip_hwgpio(desc), config); } static int gpio_set_config_with_argument_optional(struct gpio_desc *desc, @@ -2570,18 +2599,20 @@ int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce) */ int gpiod_direction_input(struct gpio_desc *desc) { - struct gpio_chip *gc; int ret = 0; VALIDATE_DESC(desc); - gc = desc->gdev->chip; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; /* * It is legal to have no .get() and .direction_input() specified if * the chip is output-only, but you can't specify .direction_input() * and not support the .get() operation, that doesn't make sense. */ - if (!gc->get && gc->direction_input) { + if (!guard.gc->get && guard.gc->direction_input) { gpiod_warn(desc, "%s: missing get() but have direction_input()\n", __func__); @@ -2594,10 +2625,12 @@ int gpiod_direction_input(struct gpio_desc *desc) * direction (if .get_direction() is supported) else we silently * assume we are in input mode after this. */ - if (gc->direction_input) { - ret = gc->direction_input(gc, gpio_chip_hwgpio(desc)); - } else if (gc->get_direction && - (gc->get_direction(gc, gpio_chip_hwgpio(desc)) != 1)) { + if (guard.gc->direction_input) { + ret = guard.gc->direction_input(guard.gc, + gpio_chip_hwgpio(desc)); + } else if (guard.gc->get_direction && + (guard.gc->get_direction(guard.gc, + gpio_chip_hwgpio(desc)) != 1)) { gpiod_warn(desc, "%s: missing direction_input() operation and line is output\n", __func__); @@ -2616,28 +2649,31 @@ EXPORT_SYMBOL_GPL(gpiod_direction_input); static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value) { - struct gpio_chip *gc = desc->gdev->chip; - int val = !!value; - int ret = 0; + int val = !!value, ret = 0; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; /* * It's OK not to specify .direction_output() if the gpiochip is * output-only, but if there is then not even a .set() operation it * is pretty tricky to drive the output line. */ - if (!gc->set && !gc->direction_output) { + if (!guard.gc->set && !guard.gc->direction_output) { gpiod_warn(desc, "%s: missing set() and direction_output() operations\n", __func__); return -EIO; } - if (gc->direction_output) { - ret = gc->direction_output(gc, gpio_chip_hwgpio(desc), val); + if (guard.gc->direction_output) { + ret = guard.gc->direction_output(guard.gc, + gpio_chip_hwgpio(desc), val); } else { /* Check that we are in output mode if we can */ - if (gc->get_direction && - gc->get_direction(gc, gpio_chip_hwgpio(desc))) { + if (guard.gc->get_direction && + guard.gc->get_direction(guard.gc, gpio_chip_hwgpio(desc))) { gpiod_warn(desc, "%s: missing direction_output() operation\n", __func__); @@ -2647,7 +2683,7 @@ static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value) * If we can't actively set the direction, we are some * output-only chip, so just drive the output as desired. */ - gc->set(gc, gpio_chip_hwgpio(desc), val); + guard.gc->set(guard.gc, gpio_chip_hwgpio(desc), val); } if (!ret) @@ -2763,17 +2799,20 @@ EXPORT_SYMBOL_GPL(gpiod_direction_output); int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags) { int ret = 0; - struct gpio_chip *gc; VALIDATE_DESC(desc); - gc = desc->gdev->chip; - if (!gc->en_hw_timestamp) { + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + if (!guard.gc->en_hw_timestamp) { gpiod_warn(desc, "%s: hw ts not supported\n", __func__); return -ENOTSUPP; } - ret = gc->en_hw_timestamp(gc, gpio_chip_hwgpio(desc), flags); + ret = guard.gc->en_hw_timestamp(guard.gc, + gpio_chip_hwgpio(desc), flags); if (ret) gpiod_warn(desc, "%s: hw ts request failed\n", __func__); @@ -2792,17 +2831,20 @@ EXPORT_SYMBOL_GPL(gpiod_enable_hw_timestamp_ns); int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags) { int ret = 0; - struct gpio_chip *gc; VALIDATE_DESC(desc); - gc = desc->gdev->chip; - if (!gc->dis_hw_timestamp) { + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + if (!guard.gc->dis_hw_timestamp) { gpiod_warn(desc, "%s: hw ts not supported\n", __func__); return -ENOTSUPP; } - ret = gc->dis_hw_timestamp(gc, gpio_chip_hwgpio(desc), flags); + ret = guard.gc->dis_hw_timestamp(guard.gc, gpio_chip_hwgpio(desc), + flags); if (ret) gpiod_warn(desc, "%s: hw ts release failed\n", __func__); @@ -2821,12 +2863,13 @@ EXPORT_SYMBOL_GPL(gpiod_disable_hw_timestamp_ns); */ int gpiod_set_config(struct gpio_desc *desc, unsigned long config) { - struct gpio_chip *gc; - VALIDATE_DESC(desc); - gc = desc->gdev->chip; - return gpio_do_set_config(gc, gpio_chip_hwgpio(desc), config); + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + return gpio_do_set_config(guard.gc, gpio_chip_hwgpio(desc), config); } EXPORT_SYMBOL_GPL(gpiod_set_config); @@ -2924,10 +2967,19 @@ static int gpio_chip_get_value(struct gpio_chip *gc, const struct gpio_desc *des static int gpiod_get_raw_value_commit(const struct gpio_desc *desc) { + struct gpio_device *gdev; struct gpio_chip *gc; int value; - gc = desc->gdev->chip; + /* FIXME Unable to use gpio_chip_guard due to const desc. */ + gdev = desc->gdev; + + guard(srcu)(&gdev->srcu); + + gc = rcu_dereference(gdev->chip); + if (!gc) + return -ENODEV; + value = gpio_chip_get_value(gc, desc); value = value < 0 ? value : !!value; trace_gpio_value(desc_to_gpio(desc), 1, value); @@ -2953,6 +3005,14 @@ static int gpio_chip_get_multiple(struct gpio_chip *gc, return -EIO; } +/* The 'other' chip must be protected with its GPIO device's SRCU. */ +static bool gpio_device_chip_cmp(struct gpio_device *gdev, struct gpio_chip *gc) +{ + guard(srcu)(&gdev->srcu); + + return gc == rcu_dereference(gdev->chip); +} + int gpiod_get_array_value_complex(bool raw, bool can_sleep, unsigned int array_size, struct gpio_desc **desc_array, @@ -2990,33 +3050,36 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep, } while (i < array_size) { - struct gpio_chip *gc = desc_array[i]->gdev->chip; DECLARE_BITMAP(fastpath_mask, FASTPATH_NGPIO); DECLARE_BITMAP(fastpath_bits, FASTPATH_NGPIO); unsigned long *mask, *bits; int first, j; - if (likely(gc->ngpio <= FASTPATH_NGPIO)) { + CLASS(gpio_chip_guard, guard)(desc_array[i]); + if (!guard.gc) + return -ENODEV; + + if (likely(guard.gc->ngpio <= FASTPATH_NGPIO)) { mask = fastpath_mask; bits = fastpath_bits; } else { gfp_t flags = can_sleep ? GFP_KERNEL : GFP_ATOMIC; - mask = bitmap_alloc(gc->ngpio, flags); + mask = bitmap_alloc(guard.gc->ngpio, flags); if (!mask) return -ENOMEM; - bits = bitmap_alloc(gc->ngpio, flags); + bits = bitmap_alloc(guard.gc->ngpio, flags); if (!bits) { bitmap_free(mask); return -ENOMEM; } } - bitmap_zero(mask, gc->ngpio); + bitmap_zero(mask, guard.gc->ngpio); if (!can_sleep) - WARN_ON(gc->can_sleep); + WARN_ON(guard.gc->can_sleep); /* collect all inputs belonging to the same chip */ first = i; @@ -3031,9 +3094,9 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep, i = find_next_zero_bit(array_info->get_mask, array_size, i); } while ((i < array_size) && - (desc_array[i]->gdev->chip == gc)); + gpio_device_chip_cmp(desc_array[i]->gdev, guard.gc)); - ret = gpio_chip_get_multiple(gc, mask, bits); + ret = gpio_chip_get_multiple(guard.gc, mask, bits); if (ret) { if (mask != fastpath_mask) bitmap_free(mask); @@ -3174,14 +3237,16 @@ EXPORT_SYMBOL_GPL(gpiod_get_array_value); */ static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value) { - int ret = 0; - struct gpio_chip *gc = desc->gdev->chip; - int offset = gpio_chip_hwgpio(desc); + int ret = 0, offset = gpio_chip_hwgpio(desc); + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; if (value) { - ret = gc->direction_input(gc, offset); + ret = guard.gc->direction_input(guard.gc, offset); } else { - ret = gc->direction_output(gc, offset, 0); + ret = guard.gc->direction_output(guard.gc, offset, 0); if (!ret) set_bit(FLAG_IS_OUT, &desc->flags); } @@ -3199,16 +3264,18 @@ static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value) */ static void gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value) { - int ret = 0; - struct gpio_chip *gc = desc->gdev->chip; - int offset = gpio_chip_hwgpio(desc); + int ret = 0, offset = gpio_chip_hwgpio(desc); + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; if (value) { - ret = gc->direction_output(gc, offset, 1); + ret = guard.gc->direction_output(guard.gc, offset, 1); if (!ret) set_bit(FLAG_IS_OUT, &desc->flags); } else { - ret = gc->direction_input(gc, offset); + ret = guard.gc->direction_input(guard.gc, offset); } trace_gpio_direction(desc_to_gpio(desc), !value, ret); if (ret < 0) @@ -3219,11 +3286,12 @@ static void gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value static void gpiod_set_raw_value_commit(struct gpio_desc *desc, bool value) { - struct gpio_chip *gc; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; - gc = desc->gdev->chip; trace_gpio_value(desc_to_gpio(desc), 0, value); - gc->set(gc, gpio_chip_hwgpio(desc), value); + guard.gc->set(guard.gc, gpio_chip_hwgpio(desc), value); } /* @@ -3284,33 +3352,36 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, } while (i < array_size) { - struct gpio_chip *gc = desc_array[i]->gdev->chip; DECLARE_BITMAP(fastpath_mask, FASTPATH_NGPIO); DECLARE_BITMAP(fastpath_bits, FASTPATH_NGPIO); unsigned long *mask, *bits; int count = 0; - if (likely(gc->ngpio <= FASTPATH_NGPIO)) { + CLASS(gpio_chip_guard, guard)(desc_array[i]); + if (!guard.gc) + return -ENODEV; + + if (likely(guard.gc->ngpio <= FASTPATH_NGPIO)) { mask = fastpath_mask; bits = fastpath_bits; } else { gfp_t flags = can_sleep ? GFP_KERNEL : GFP_ATOMIC; - mask = bitmap_alloc(gc->ngpio, flags); + mask = bitmap_alloc(guard.gc->ngpio, flags); if (!mask) return -ENOMEM; - bits = bitmap_alloc(gc->ngpio, flags); + bits = bitmap_alloc(guard.gc->ngpio, flags); if (!bits) { bitmap_free(mask); return -ENOMEM; } } - bitmap_zero(mask, gc->ngpio); + bitmap_zero(mask, guard.gc->ngpio); if (!can_sleep) - WARN_ON(gc->can_sleep); + WARN_ON(guard.gc->can_sleep); do { struct gpio_desc *desc = desc_array[i]; @@ -3346,10 +3417,10 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, i = find_next_zero_bit(array_info->set_mask, array_size, i); } while ((i < array_size) && - (desc_array[i]->gdev->chip == gc)); + gpio_device_chip_cmp(desc_array[i]->gdev, guard.gc)); /* push collected bits to outputs */ if (count != 0) - gpio_chip_set_multiple(gc, mask, bits); + gpio_chip_set_multiple(guard.gc, mask, bits); if (mask != fastpath_mask) bitmap_free(mask); @@ -3505,6 +3576,7 @@ EXPORT_SYMBOL_GPL(gpiod_set_consumer_name); */ int gpiod_to_irq(const struct gpio_desc *desc) { + struct gpio_device *gdev; struct gpio_chip *gc; int offset; @@ -3516,7 +3588,13 @@ int gpiod_to_irq(const struct gpio_desc *desc) if (!desc || IS_ERR(desc)) return -EINVAL; - gc = desc->gdev->chip; + gdev = desc->gdev; + /* FIXME Cannot use gpio_chip_guard due to const desc. */ + guard(srcu)(&gdev->srcu); + gc = rcu_dereference(gdev->chip); + if (!gc) + return -ENODEV; + offset = gpio_chip_hwgpio(desc); if (gc->to_irq) { int retirq = gc->to_irq(gc, offset); @@ -4696,12 +4774,20 @@ core_initcall(gpiolib_dev_init); static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) { - struct gpio_chip *gc = gdev->chip; bool active_low, is_irq, is_out; unsigned int gpio = gdev->base; struct gpio_desc *desc; + struct gpio_chip *gc; int value; + guard(srcu)(&gdev->srcu); + + gc = rcu_dereference(gdev->chip); + if (!gc) { + seq_puts(s, "Underlying GPIO chip is gone\n"); + return; + } + for_each_gpio_desc(gc, desc) { guard(srcu)(&desc->srcu); if (test_bit(FLAG_REQUESTED, &desc->flags)) { @@ -4776,9 +4862,12 @@ static int gpiolib_seq_show(struct seq_file *s, void *v) { struct gpiolib_seq_priv *priv = s->private; struct gpio_device *gdev = v; - struct gpio_chip *gc = gdev->chip; + struct gpio_chip *gc; struct device *parent; + guard(srcu)(&gdev->srcu); + + gc = rcu_dereference(gdev->chip); if (!gc) { seq_printf(s, "%s%s: (dangling chip)", priv->newline ? "\n" : "", diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 35d71e30c546..b3810f7d286a 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -63,7 +63,7 @@ struct gpio_device { int id; struct device *mockdev; struct module *owner; - struct gpio_chip *chip; + struct gpio_chip __rcu *chip; struct gpio_desc *descs; int base; u16 ngpio; @@ -193,6 +193,26 @@ struct gpio_desc { #define gpiod_not_found(desc) (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) +struct gpio_chip_guard { + struct gpio_device *gdev; + struct gpio_chip *gc; + int idx; +}; + +DEFINE_CLASS(gpio_chip_guard, + struct gpio_chip_guard, + srcu_read_unlock(&_T.gdev->srcu, _T.idx), + ({ + struct gpio_chip_guard _guard; + + _guard.gdev = desc->gdev; + _guard.idx = srcu_read_lock(&_guard.gdev->srcu); + _guard.gc = rcu_dereference(_guard.gdev->chip); + + _guard; + }), + struct gpio_desc *desc) + int gpiod_request(struct gpio_desc *desc, const char *label); void gpiod_free(struct gpio_desc *desc); From f067372c6a3c3b0c5cc775157408970b8fa6c010 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 17:02:03 +0100 Subject: [PATCH 0272/1038] gpio: remove the RW semaphore from the GPIO device With all accesses to gdev->chip being protected with SRCU, we can now remove the RW-semaphore specific to the character device which fulfilled the same role up to this point. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 4 ---- drivers/gpio/gpiolib.h | 5 ----- 2 files changed, 9 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 1a9d045b5a6b..875b40376c0b 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -963,7 +963,6 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, BLOCKING_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); - init_rwsem(&gdev->sem); ret = init_srcu_struct(&gdev->srcu); if (ret) @@ -1102,8 +1101,6 @@ void gpiochip_remove(struct gpio_chip *gc) struct gpio_device *gdev = gc->gpiodev; unsigned int i; - down_write(&gdev->sem); - /* FIXME: should the legacy sysfs handling be moved to gpio_device? */ gpiochip_sysfs_unregister(gdev); gpiochip_free_hogs(gc); @@ -1142,7 +1139,6 @@ void gpiochip_remove(struct gpio_chip *gc) * gone. */ gcdev_unregister(gdev); - up_write(&gdev->sem); gpio_device_put(gdev); } EXPORT_SYMBOL_GPL(gpiochip_remove); diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index b3810f7d286a..07443d26cbca 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -16,7 +16,6 @@ #include #include #include -#include #include #define GPIOCHIP_NAME "gpiochip" @@ -46,9 +45,6 @@ * requested, released or reconfigured * @device_notifier: used to notify character device wait queues about the GPIO * device being unregistered - * @sem: protects the structure from a NULL-pointer dereference of @chip by - * user-space operations when the device gets unregistered during - * a hot-unplug event * @srcu: protects the pointer to the underlying GPIO chip * @pin_ranges: range of pins served by the GPIO driver * @@ -73,7 +69,6 @@ struct gpio_device { struct list_head list; struct blocking_notifier_head line_state_notifier; struct blocking_notifier_head device_notifier; - struct rw_semaphore sem; struct srcu_struct srcu; #ifdef CONFIG_PINCTRL From 5e6284444024cf7e7d0ae538df8037a3a5efbec8 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 24 Jan 2024 19:27:39 +0100 Subject: [PATCH 0273/1038] gpio: mark unsafe gpio_chip manipulators as deprecated We still have some functions that return the address of the GPIO chip associated with the GPIO device. This is dangerous and the users should find a better solution. Let's add appropriate comments to the kernel docs. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Andy Shevchenko --- drivers/gpio/gpiolib.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 875b40376c0b..82811d9a4477 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -211,6 +211,11 @@ EXPORT_SYMBOL_GPL(desc_to_gpio); /** * gpiod_to_chip - Return the GPIO chip to which a GPIO descriptor belongs * @desc: descriptor to return the chip of + * + * *DEPRECATED* + * This function is unsafe and should not be used. Using the chip address + * without taking the SRCU read lock may result in dereferencing a dangling + * pointer. */ struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc) { @@ -275,6 +280,7 @@ EXPORT_SYMBOL(gpio_device_get_label); * Returns: * Address of the GPIO chip backing this device. * + * *DEPRECATED* * Until we can get rid of all non-driver users of struct gpio_chip, we must * provide a way of retrieving the pointer to it from struct gpio_device. This * is *NOT* safe as the GPIO API is considered to be hot-unpluggable and the From ccd975daa807543a0050ab0a84f8581e3aae62a6 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 9 Feb 2024 14:49:49 +0100 Subject: [PATCH 0274/1038] thermal: sysfs: Fix up white space in trip_point_temp_store() Remove an excess tab character from an otherwise empty code line. No functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka --- drivers/thermal/thermal_sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index f4033865b093..d55f9303afb5 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -136,7 +136,7 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, unlock: mutex_unlock(&tz->lock); - + return ret ? ret : count; } From 0fac6893ff6c08d6fab6f56b9550a2c4cee589fd Mon Sep 17 00:00:00 2001 From: Di Shen Date: Wed, 7 Feb 2024 10:09:23 +0800 Subject: [PATCH 0275/1038] thermal: gov_power_allocator: Avoid overwriting PID coefficients from setup time When the PID coefficients k_* are set via sysfs before the IPA algorithm is triggered then the coefficients would be overwritten after IPA throttle() is called. The old configuration values might be different than the new values estimated by the IPA internal algorithm. There might be a time delay when this overwriting happens. It depends on the thermal zone temperature value. The temperature value needs to cross the first trip point value then IPA algorithms start operating. Although, the PID coefficients setup time should not be affected or linked to any later operating phase and values must not be overwritten. This patch initializes params->sustainable_power when the governor binds to thermal zone to avoid overwriting k_*. The basic function won't be affected, as the k_* still can be estimated if the sustainable_power is modified. Signed-off-by: Di Shen Reviewed-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/thermal/gov_power_allocator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 81e061f183ad..1b17dc4c219c 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -711,6 +711,8 @@ static int power_allocator_bind(struct thermal_zone_device *tz) if (!tz->tzp->sustainable_power) dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n"); + else + params->sustainable_power = tz->tzp->sustainable_power; estimate_pid_constants(tz, tz->tzp->sustainable_power, params->trip_switch_on, From 073237281a508ac80ec025872ad7de50cfb5a28a Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 6 Feb 2024 20:33:45 +0100 Subject: [PATCH 0276/1038] ACPI: PM: s2idle: Enable Low-Power S0 Idle MSFT UUID for non-AMD systems Systems based on Intel platforms that use the MSFT UUID for Low-Power S0 Idle (LPS0) have started to ship, so allow the kernel to use the MSFT UUID in the non-AMD case too, but in that case make it avoid evaluating the same _DSM function for two different UUIDs and prioritize the MSFT one. While at it, combine two MSFT _DSM function mask checks in acpi_s2idle_restore_early() so as to make it reflect the acpi_s2idle_prepare_late() flow more closely and adjust the Modern Standby entry and exit comments slightly. Non-AMD systems that do not support MSFT UUID for Low-power S0 Idle are not expected to be affected by this change in any way. Signed-off-by: Rafael J. Wysocki Reviewed-by: Mario Limonciello --- drivers/acpi/x86/s2idle.c | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index 7d64e655f1b8..cd84af23f7ea 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -488,7 +488,21 @@ static int lps0_device_attach(struct acpi_device *adev, rev_id = 1; lps0_dsm_func_mask = validate_dsm(adev->handle, ACPI_LPS0_DSM_UUID, rev_id, &lps0_dsm_guid); - lps0_dsm_func_mask_microsoft = -EINVAL; + if (lps0_dsm_func_mask > 0 && lps0_dsm_func_mask_microsoft > 0) { + unsigned int func_mask; + + /* + * Avoid evaluating the same _DSM function for two + * different UUIDs and prioritize the MSFT one. + */ + func_mask = lps0_dsm_func_mask & lps0_dsm_func_mask_microsoft; + if (func_mask) { + acpi_handle_info(adev->handle, + "Duplicate LPS0 _DSM functions (mask: 0x%x)\n", + func_mask); + lps0_dsm_func_mask &= ~func_mask; + } + } } if (lps0_dsm_func_mask < 0 && lps0_dsm_func_mask_microsoft < 0) @@ -549,19 +563,22 @@ int acpi_s2idle_prepare_late(void) lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); /* LPS0 entry */ - if (lps0_dsm_func_mask > 0) - acpi_sleep_run_lps0_dsm(acpi_s2idle_vendor_amd() ? - ACPI_LPS0_ENTRY_AMD : - ACPI_LPS0_ENTRY, + if (lps0_dsm_func_mask > 0 && acpi_s2idle_vendor_amd()) + acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY_AMD, lps0_dsm_func_mask, lps0_dsm_guid); + if (lps0_dsm_func_mask_microsoft > 0) { - /* modern standby entry */ + /* Modern Standby entry */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_ENTRY, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); } + if (lps0_dsm_func_mask > 0 && !acpi_s2idle_vendor_amd()) + acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY, + lps0_dsm_func_mask, lps0_dsm_guid); + list_for_each_entry(handler, &lps0_s2idle_devops_head, list_node) { if (handler->prepare) handler->prepare(); @@ -600,14 +617,14 @@ void acpi_s2idle_restore_early(void) ACPI_LPS0_EXIT_AMD : ACPI_LPS0_EXIT, lps0_dsm_func_mask, lps0_dsm_guid); - if (lps0_dsm_func_mask_microsoft > 0) + + if (lps0_dsm_func_mask_microsoft > 0) { acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); - - /* Modern standby exit */ - if (lps0_dsm_func_mask_microsoft > 0) + /* Modern Standby exit */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); + } /* Screen on */ if (lps0_dsm_func_mask_microsoft > 0) From ca3afc2806046f626a1518f160a564b90f141f95 Mon Sep 17 00:00:00 2001 From: Nicolas Haye Date: Tue, 30 Jan 2024 09:57:04 +0000 Subject: [PATCH 0277/1038] ACPI: resource: Skip IRQ override on ASUS ExpertBook B1502CVA Like many b1502 models, the b1502CVA keyboard doesn't work because of an ACPI DSDT table that describes IRQ 1 as ActiveLow while the kernel overrides it to Edge_High. Signed-off-by: Nicolas Haye Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index dacad1d846c0..65ce43ecfa8c 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -468,6 +468,13 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "B1502CGA"), }, }, + { + /* Asus ExpertBook B1502CVA */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "B1502CVA"), + }, + }, { /* Asus ExpertBook B2402CBA */ .matches = { From 4c7dbd85218d6e2ed0495bbaf40c1b1c6bac506b Mon Sep 17 00:00:00 2001 From: Meng Li Date: Mon, 5 Feb 2024 14:03:05 +0800 Subject: [PATCH 0278/1038] Documentation: PM: amd-pstate: Fix section title underline Title under line too short Signed-off-by: Meng Li [ rjw: Subject edits ] Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/pm/amd-pstate.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 0a3aa6b8ffd5..1e0d101b020a 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -381,7 +381,7 @@ driver receives a message with the highest performance change, it will update the core ranking and set the cpu's priority. ``amd-pstate`` Preferred Core Switch -================================= +===================================== Kernel Parameters ----------------- From 592190b598c3a9fdf98c5fc649e5f6da6e8e7941 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 11 Feb 2024 12:37:11 -0300 Subject: [PATCH 0279/1038] ACPI: bus: make acpi_bus_type const Now that the driver core can properly handle constant struct bus_type, move the acpi_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Greg Kroah-Hartman Signed-off-by: Rafael J. Wysocki --- drivers/acpi/bus.c | 2 +- include/acpi/acpi_bus.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 569bd15f211b..d9fa730416f1 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -1097,7 +1097,7 @@ static void acpi_device_remove(struct device *dev) put_device(dev); } -struct bus_type acpi_bus_type = { +const struct bus_type acpi_bus_type = { .name = "acpi", .match = acpi_bus_match, .probe = acpi_device_probe, diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index e4d24d3f9abb..f67aa93ba53e 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -582,7 +582,7 @@ void acpi_initialize_hp_context(struct acpi_device *adev, void (*uevent)(struct acpi_device *, u32)); /* acpi_device.dev.bus == &acpi_bus_type */ -extern struct bus_type acpi_bus_type; +extern const struct bus_type acpi_bus_type; int acpi_bus_for_each_dev(int (*fn)(struct device *, void *), void *data); int acpi_dev_for_each_child(struct acpi_device *adev, From b26ffbf800ae3c8d01bdf90d9cd8a37e1606ff06 Mon Sep 17 00:00:00 2001 From: Tor Vic Date: Fri, 9 Feb 2024 16:42:26 +0100 Subject: [PATCH 0280/1038] cpufreq: amd-pstate: Fix min_perf assignment in amd_pstate_adjust_perf() In the function amd_pstate_adjust_perf(), the 'min_perf' variable is set to 'highest_perf' instead of 'lowest_perf'. Fixes: 1d215f0319c2 ("cpufreq: amd-pstate: Add fast switch function for AMD P-State") Reported-by: Oleksandr Natalenko Reviewed-by: Perry Yuan Signed-off-by: Tor Vic Reviewed-by: Mario Limonciello Cc: 6.1+ # 6.1+ Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 08e112444c27..aa5e57e27d2b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -577,7 +577,7 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (target_perf < capacity) des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); - min_perf = READ_ONCE(cpudata->highest_perf); + min_perf = READ_ONCE(cpudata->lowest_perf); if (_min_perf < capacity) min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); From e13aa799c2a6a0be23294b630109d23940cf8079 Mon Sep 17 00:00:00 2001 From: Qais Yousef Date: Mon, 5 Feb 2024 02:25:00 +0000 Subject: [PATCH 0281/1038] cpufreq: Change default transition delay to 2ms 10ms is too high for today's hardware, even low end ones. This default end up being used a lot on Arm machines at least. Pine64, mac mini and pixel 6 all end up with 10ms rate_limit_us when using schedutil, and it's too high for all of them. Change the default to 2ms which should be 'pessimistic' enough for worst case scenario, but not too high for platforms with fast DVFS hardware. Signed-off-by: Qais Yousef Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 44db4f59c4cc..66cef33c4ec7 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -582,11 +582,11 @@ unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) * for platforms where transition_latency is in milliseconds, it * ends up giving unrealistic values. * - * Cap the default transition delay to 10 ms, which seems to be + * Cap the default transition delay to 2 ms, which seems to be * a reasonable amount of time after which we should reevaluate * the frequency. */ - return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000); + return min(latency * LATENCY_MULTIPLIER, (unsigned int)(2 * MSEC_PER_SEC)); } return LATENCY_MULTIPLIER; From c0ef3df8dbaef51ee4cfd58a471adf2eaee6f6b3 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 30 Jan 2024 13:28:05 +0200 Subject: [PATCH 0282/1038] PM: runtime: Simplify pm_runtime_get_if_active() usage There are two ways to opportunistically increment a device's runtime PM usage count, calling either pm_runtime_get_if_active() or pm_runtime_get_if_in_use(). The former has an argument to tell whether to ignore the usage count or not, and the latter simply calls the former with ign_usage_count set to false. The other users that want to ignore the usage_count will have to explicitly set that argument to true which is a bit cumbersome. To make this function more practical to use, remove the ign_usage_count argument from the function. The main implementation is in a static function called pm_runtime_get_conditional() and implementations of pm_runtime_get_if_active() and pm_runtime_get_if_in_use() are moved to runtime.c. Signed-off-by: Sakari Ailus Reviewed-by: Alex Elder Reviewed-by: Laurent Pinchart Acked-by: Takashi Iwai # sound/ Reviewed-by: Jacek Lawrynowicz # drivers/accel/ivpu/ Acked-by: Rodrigo Vivi # drivers/gpu/drm/i915/ Reviewed-by: Rodrigo Vivi Acked-by: Bjorn Helgaas # drivers/pci/ Signed-off-by: Rafael J. Wysocki --- Documentation/power/runtime_pm.rst | 5 ++-- drivers/accel/ivpu/ivpu_pm.c | 2 +- drivers/base/power/runtime.c | 35 +++++++++++++++++++++++-- drivers/gpu/drm/i915/intel_runtime_pm.c | 5 +++- drivers/gpu/drm/xe/xe_pm.c | 2 +- drivers/media/i2c/ccs/ccs-core.c | 2 +- drivers/media/i2c/ov64a40.c | 2 +- drivers/media/i2c/thp7312.c | 2 +- drivers/net/ipa/ipa_smp2p.c | 2 +- drivers/pci/pci.c | 2 +- include/linux/pm_runtime.h | 18 +++---------- sound/hda/hdac_device.c | 2 +- 12 files changed, 50 insertions(+), 29 deletions(-) diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index 65b86e487afe..da99379071a4 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -396,10 +396,9 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: nonzero, increment the counter and return 1; otherwise return 0 without changing the counter - `int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count);` + `int pm_runtime_get_if_active(struct device *dev);` - return -EINVAL if 'power.disable_depth' is nonzero; otherwise, if the - runtime PM status is RPM_ACTIVE, and either ign_usage_count is true - or the device's usage_count is non-zero, increment the counter and + runtime PM status is RPM_ACTIVE, increment the counter and return 1; otherwise return 0 without changing the counter `void pm_runtime_put_noidle(struct device *dev);` diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c index f501f27ebafd..981777c93cf5 100644 --- a/drivers/accel/ivpu/ivpu_pm.c +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -304,7 +304,7 @@ int ivpu_rpm_get_if_active(struct ivpu_device *vdev) { int ret; - ret = pm_runtime_get_if_active(vdev->drm.dev, false); + ret = pm_runtime_get_if_in_use(vdev->drm.dev); drm_WARN_ON(&vdev->drm, ret < 0); return ret; diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 05793c9fbb84..5275a6b2e980 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -1176,7 +1176,7 @@ int __pm_runtime_resume(struct device *dev, int rpmflags) EXPORT_SYMBOL_GPL(__pm_runtime_resume); /** - * pm_runtime_get_if_active - Conditionally bump up device usage counter. + * pm_runtime_get_conditional - Conditionally bump up device usage counter. * @dev: Device to handle. * @ign_usage_count: Whether or not to look at the current usage counter value. * @@ -1197,7 +1197,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_resume); * The caller is responsible for decrementing the runtime PM usage counter of * @dev after this function has returned a positive value for it. */ -int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) +static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count) { unsigned long flags; int retval; @@ -1218,8 +1218,39 @@ int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) return retval; } + +/** + * pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is + * in active state + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE, in which case it returns 1. If the device is in a different + * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the + * device, in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_active(struct device *dev) +{ + return pm_runtime_get_conditional(dev, true); +} EXPORT_SYMBOL_GPL(pm_runtime_get_if_active); +/** + * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE and its runtime PM usage counter is greater than 0, in which case + * it returns 1. If the device is in a different state or its usage_count is 0, + * 0 is returned. -EINVAL is returned if runtime PM is disabled for the device, + * in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_in_use(struct device *dev) +{ + return pm_runtime_get_conditional(dev, false); +} +EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use); + /** * __pm_runtime_set_status - Set runtime PM status of a device. * @dev: Device to handle. diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 860b51b56a92..d4e844128826 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -246,7 +246,10 @@ static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm * function, since the power state is undefined. This applies * atm to the late/early system suspend/resume handlers. */ - if (pm_runtime_get_if_active(rpm->kdev, ignore_usecount) <= 0) + if ((ignore_usecount && + pm_runtime_get_if_active(rpm->kdev) <= 0) || + (!ignore_usecount && + pm_runtime_get_if_in_use(rpm->kdev) <= 0)) return 0; } diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index b429c2876a76..dd110058bf74 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -330,7 +330,7 @@ int xe_pm_runtime_put(struct xe_device *xe) int xe_pm_runtime_get_if_active(struct xe_device *xe) { - return pm_runtime_get_if_active(xe->drm.dev, true); + return pm_runtime_get_if_active(xe->drm.dev); } void xe_pm_assert_unbounded_bridge(struct xe_device *xe) diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index e21287d50c15..e1ae0f9fad43 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -674,7 +674,7 @@ static int ccs_set_ctrl(struct v4l2_ctrl *ctrl) break; } - pm_status = pm_runtime_get_if_active(&client->dev, true); + pm_status = pm_runtime_get_if_active(&client->dev); if (!pm_status) return 0; diff --git a/drivers/media/i2c/ov64a40.c b/drivers/media/i2c/ov64a40.c index 4fba4c2cb064..541bf74581d2 100644 --- a/drivers/media/i2c/ov64a40.c +++ b/drivers/media/i2c/ov64a40.c @@ -3287,7 +3287,7 @@ static int ov64a40_set_ctrl(struct v4l2_ctrl *ctrl) exp_max, 1, exp_val); } - pm_status = pm_runtime_get_if_active(ov64a40->dev, true); + pm_status = pm_runtime_get_if_active(ov64a40->dev); if (!pm_status) return 0; diff --git a/drivers/media/i2c/thp7312.c b/drivers/media/i2c/thp7312.c index 2806887514dc..19bd923a7315 100644 --- a/drivers/media/i2c/thp7312.c +++ b/drivers/media/i2c/thp7312.c @@ -1052,7 +1052,7 @@ static int thp7312_s_ctrl(struct v4l2_ctrl *ctrl) if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE) return -EINVAL; - if (!pm_runtime_get_if_active(thp7312->dev, true)) + if (!pm_runtime_get_if_active(thp7312->dev)) return 0; switch (ctrl->id) { diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c index 5620dc271fac..cbf3d4761ce3 100644 --- a/drivers/net/ipa/ipa_smp2p.c +++ b/drivers/net/ipa/ipa_smp2p.c @@ -92,7 +92,7 @@ static void ipa_smp2p_notify(struct ipa_smp2p *smp2p) return; dev = &smp2p->ipa->pdev->dev; - smp2p->power_on = pm_runtime_get_if_active(dev, true) > 0; + smp2p->power_on = pm_runtime_get_if_active(dev) > 0; /* Signal whether the IPA power is enabled */ mask = BIT(smp2p->enabled_bit); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 9ab9b1008d8b..cb51c4079013 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2536,7 +2536,7 @@ static void pci_pme_list_scan(struct work_struct *work) * If the device is in a low power state it * should not be polled either. */ - pm_status = pm_runtime_get_if_active(dev, true); + pm_status = pm_runtime_get_if_active(dev); if (!pm_status) continue; diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 7c9b35448563..436baa167498 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -72,7 +72,8 @@ extern int pm_runtime_force_resume(struct device *dev); extern int __pm_runtime_idle(struct device *dev, int rpmflags); extern int __pm_runtime_suspend(struct device *dev, int rpmflags); extern int __pm_runtime_resume(struct device *dev, int rpmflags); -extern int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count); +extern int pm_runtime_get_if_active(struct device *dev); +extern int pm_runtime_get_if_in_use(struct device *dev); extern int pm_schedule_suspend(struct device *dev, unsigned int delay); extern int __pm_runtime_set_status(struct device *dev, unsigned int status); extern int pm_runtime_barrier(struct device *dev); @@ -94,18 +95,6 @@ extern void pm_runtime_release_supplier(struct device_link *link); extern int devm_pm_runtime_enable(struct device *dev); -/** - * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. - * @dev: Target device. - * - * Increment the runtime PM usage counter of @dev if its runtime PM status is - * %RPM_ACTIVE and its runtime PM usage counter is greater than 0. - */ -static inline int pm_runtime_get_if_in_use(struct device *dev) -{ - return pm_runtime_get_if_active(dev, false); -} - /** * pm_suspend_ignore_children - Set runtime PM behavior regarding children. * @dev: Target device. @@ -275,8 +264,7 @@ static inline int pm_runtime_get_if_in_use(struct device *dev) { return -EINVAL; } -static inline int pm_runtime_get_if_active(struct device *dev, - bool ign_usage_count) +static inline int pm_runtime_get_if_active(struct device *dev) { return -EINVAL; } diff --git a/sound/hda/hdac_device.c b/sound/hda/hdac_device.c index 7f7b67fe1b65..068c16e52dff 100644 --- a/sound/hda/hdac_device.c +++ b/sound/hda/hdac_device.c @@ -612,7 +612,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_power_up_pm); int snd_hdac_keep_power_up(struct hdac_device *codec) { if (!atomic_inc_not_zero(&codec->in_pm)) { - int ret = pm_runtime_get_if_active(&codec->dev, true); + int ret = pm_runtime_get_if_active(&codec->dev); if (!ret) return -1; if (ret < 0) From b7d46644e554ed017dfabc0841acf418d0584bc9 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 30 Jan 2024 13:28:32 +0200 Subject: [PATCH 0283/1038] PM: runtime: Add pm_runtime_put_autosuspend() replacement Add __pm_runtime_put_autosuspend() that replaces pm_runtime_put_autosuspend() for new users. The intent is to later re-purpose pm_runtime_put_autosuspend() to also mark the device's last busy stamp---which is what the vast majority of users actually need. This is also described in pm_runtime_put_autosuspend() documentation. Signed-off-by: Sakari Ailus Reviewed-by: Laurent Pinchart Signed-off-by: Rafael J. Wysocki --- Documentation/power/runtime_pm.rst | 17 +++++++++++------ include/linux/pm_runtime.h | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index da99379071a4..6fa50e4f87ce 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -154,7 +154,7 @@ suspending the device are satisfied) and to queue up a suspend request for the device in that case. If there is no idle callback, or if the callback returns 0, then the PM core will attempt to carry out a runtime suspend of the device, also respecting devices configured for autosuspend. In essence this means a -call to pm_runtime_autosuspend() (do note that drivers needs to update the +call to __pm_runtime_autosuspend() (do note that drivers needs to update the device last busy mark, pm_runtime_mark_last_busy(), to control the delay under this circumstance). To prevent this (for example, if the callback routine has started a delayed suspend), the routine must return a non-zero value. Negative @@ -409,6 +409,10 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: pm_request_idle(dev) and return its result `int pm_runtime_put_autosuspend(struct device *dev);` + - does the same as __pm_runtime_put_autosuspend() for now, but in the + future, will also call pm_runtime_mark_last_busy() as well, DO NOT USE! + + `int __pm_runtime_put_autosuspend(struct device *dev);` - decrement the device's usage counter; if the result is 0 then run pm_request_autosuspend(dev) and return its result @@ -539,6 +543,7 @@ It is safe to execute the following helper functions from interrupt context: - pm_runtime_put_noidle() - pm_runtime_put() - pm_runtime_put_autosuspend() +- __pm_runtime_put_autosuspend() - pm_runtime_enable() - pm_suspend_ignore_children() - pm_runtime_set_active() @@ -864,9 +869,9 @@ automatically be delayed until the desired period of inactivity has elapsed. Inactivity is determined based on the power.last_busy field. Drivers should call pm_runtime_mark_last_busy() to update this field after carrying out I/O, -typically just before calling pm_runtime_put_autosuspend(). The desired length -of the inactivity period is a matter of policy. Subsystems can set this length -initially by calling pm_runtime_set_autosuspend_delay(), but after device +typically just before calling __pm_runtime_put_autosuspend(). The desired +length of the inactivity period is a matter of policy. Subsystems can set this +length initially by calling pm_runtime_set_autosuspend_delay(), but after device registration the length should be controlled by user space, using the /sys/devices/.../power/autosuspend_delay_ms attribute. @@ -877,7 +882,7 @@ instead of the non-autosuspend counterparts:: Instead of: pm_runtime_suspend use: pm_runtime_autosuspend; Instead of: pm_schedule_suspend use: pm_request_autosuspend; - Instead of: pm_runtime_put use: pm_runtime_put_autosuspend; + Instead of: pm_runtime_put use: __pm_runtime_put_autosuspend; Instead of: pm_runtime_put_sync use: pm_runtime_put_sync_autosuspend. Drivers may also continue to use the non-autosuspend helper functions; they @@ -916,7 +921,7 @@ Here is a schematic pseudo-code example:: lock(&foo->private_lock); if (--foo->num_pending_requests == 0) { pm_runtime_mark_last_busy(&foo->dev); - pm_runtime_put_autosuspend(&foo->dev); + __pm_runtime_put_autosuspend(&foo->dev); } else { foo_process_next_request(foo); } diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 436baa167498..d39dc863f612 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -448,6 +448,18 @@ static inline int pm_runtime_put(struct device *dev) return __pm_runtime_idle(dev, RPM_GET_PUT | RPM_ASYNC); } +/** + * __pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0. + * @dev: Target device. + * + * Decrement the runtime PM usage counter of @dev and if it turns out to be + * equal to 0, queue up a work item for @dev like in pm_request_autosuspend(). + */ +static inline int __pm_runtime_put_autosuspend(struct device *dev) +{ + return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_ASYNC | RPM_AUTO); +} + /** * pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0. * @dev: Target device. From 496d0a648509bce665a85e19a871375dfe4c8f2e Mon Sep 17 00:00:00 2001 From: Parshuram Sangle Date: Thu, 11 Jan 2024 19:29:50 +0530 Subject: [PATCH 0284/1038] cpuidle: haltpoll: do not shrink guest poll_limit_ns below grow_start While adjusting guest halt poll limit, grow block starts at guest_halt_poll_grow_start without taking intermediate values. Similar behavior is expected while shrinking the value. This avoids short interval values which are really not required. VCPU1 trace (guest_halt_poll_shrink equals 2): VCPU1 grow 10000 VCPU1 shrink 5000 VCPU1 shrink 2500 VCPU1 shrink 1250 VCPU1 shrink 625 VCPU1 shrink 312 VCPU1 shrink 156 VCPU1 shrink 78 VCPU1 shrink 39 VCPU1 shrink 19 VCPU1 shrink 9 VCPU1 shrink 4 Similar change is done in KVM halt poll flow with below patch: Link: https://lore.kernel.org/kvm/20211006133021.271905-3-sashal@kernel.org/ Co-developed-by: Rajendran Jaishankar Signed-off-by: Rajendran Jaishankar Signed-off-by: Parshuram Sangle Reviewed-by: Marcelo Tosatti [ rjw: Subject edits ] Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/governors/haltpoll.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c index 1dff3a52917d..663b7f164d20 100644 --- a/drivers/cpuidle/governors/haltpoll.c +++ b/drivers/cpuidle/governors/haltpoll.c @@ -98,10 +98,15 @@ static void adjust_poll_limit(struct cpuidle_device *dev, u64 block_ns) unsigned int shrink = guest_halt_poll_shrink; val = dev->poll_limit_ns; - if (shrink == 0) + if (shrink == 0) { val = 0; - else + } else { val /= shrink; + /* Reset value to 0 if shrunk below grow_start */ + if (val < guest_halt_poll_grow_start) + val = 0; + } + trace_guest_halt_poll_ns_shrink(val, dev->poll_limit_ns); dev->poll_limit_ns = val; } From 88390dd788db485912ee7f9a8d3d56fc5265d52f Mon Sep 17 00:00:00 2001 From: C Cheng Date: Tue, 19 Dec 2023 11:14:42 +0800 Subject: [PATCH 0285/1038] cpuidle: Avoid potential overflow in integer multiplication In detail: In C language, when you perform a multiplication operation, if both operands are of int type, the multiplication operation is performed on the int type, and then the result is converted to the target type. This means that if the product of int type multiplication exceeds the range that int type can represent, an overflow will occur even if you store the result in a variable of int64_t type. For a multiplication of two int values, it is better to use mul_u32_u32() rather than s->exit_latency_ns = s->exit_latency * NSEC_PER_USEC to avoid potential overflow happenning. Signed-off-by: C Cheng Signed-off-by: Bo Ye Reviewed-by: AngeloGioacchino Del Regno [ rjw: New subject ] Signed-off-by: Rafael J. Wysocki --- drivers/cpuidle/driver.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index d9cda7f6ccb9..cf5873cc45dc 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "cpuidle.h" @@ -187,7 +188,7 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) s->target_residency = div_u64(s->target_residency_ns, NSEC_PER_USEC); if (s->exit_latency > 0) - s->exit_latency_ns = s->exit_latency * NSEC_PER_USEC; + s->exit_latency_ns = mul_u32_u32(s->exit_latency, NSEC_PER_USEC); else if (s->exit_latency_ns < 0) s->exit_latency_ns = 0; else From 7251b9e8a007ddd834aa81f8c7ea338884629fec Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Tue, 6 Feb 2024 09:54:09 +0800 Subject: [PATCH 0286/1038] thermal/intel: Fix intel_tcc_get_temp() to support negative CPU temperature CPU temperature can be negative in some cases. Thus the negative CPU temperature should not be considered as a failure. Fix intel_tcc_get_temp() and its users to support negative CPU temperature. Fixes: a3c1f066e1c5 ("thermal/intel: Introduce Intel TCC library") Signed-off-by: Zhang Rui Reviewed-by: Stanislaw Gruszka Cc: 6.3+ # 6.3+ Signed-off-by: Rafael J. Wysocki --- .../intel/int340x_thermal/processor_thermal_device.c | 8 ++++---- drivers/thermal/intel/intel_tcc.c | 12 ++++++------ drivers/thermal/intel/x86_pkg_temp_thermal.c | 8 ++++---- include/linux/intel_tcc.h | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index 649f67fdf345..d75fae7b7ed2 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -176,14 +176,14 @@ static int proc_thermal_get_zone_temp(struct thermal_zone_device *zone, int *temp) { int cpu; - int curr_temp; + int curr_temp, ret; *temp = 0; for_each_online_cpu(cpu) { - curr_temp = intel_tcc_get_temp(cpu, false); - if (curr_temp < 0) - return curr_temp; + ret = intel_tcc_get_temp(cpu, &curr_temp, false); + if (ret < 0) + return ret; if (!*temp || curr_temp > *temp) *temp = curr_temp; } diff --git a/drivers/thermal/intel/intel_tcc.c b/drivers/thermal/intel/intel_tcc.c index 2e5c741c41ca..5e8b7f34b395 100644 --- a/drivers/thermal/intel/intel_tcc.c +++ b/drivers/thermal/intel/intel_tcc.c @@ -103,18 +103,19 @@ EXPORT_SYMBOL_NS_GPL(intel_tcc_set_offset, INTEL_TCC); /** * intel_tcc_get_temp() - returns the current temperature * @cpu: cpu that the MSR should be run on, nagative value means any cpu. + * @temp: pointer to the memory for saving cpu temperature. * @pkg: true: Package Thermal Sensor. false: Core Thermal Sensor. * * Get the current temperature returned by the CPU core/package level * thermal sensor, in degrees C. * - * Return: Temperature in degrees C on success, negative error code otherwise. + * Return: 0 on success, negative error code otherwise. */ -int intel_tcc_get_temp(int cpu, bool pkg) +int intel_tcc_get_temp(int cpu, int *temp, bool pkg) { u32 low, high; u32 msr = pkg ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; - int tjmax, temp, err; + int tjmax, err; tjmax = intel_tcc_get_tjmax(cpu); if (tjmax < 0) @@ -131,9 +132,8 @@ int intel_tcc_get_temp(int cpu, bool pkg) if (!(low & BIT(31))) return -ENODATA; - temp = tjmax - ((low >> 16) & 0x7f); + *temp = tjmax - ((low >> 16) & 0x7f); - /* Do not allow negative CPU temperature */ - return temp >= 0 ? temp : -ENODATA; + return 0; } EXPORT_SYMBOL_NS_GPL(intel_tcc_get_temp, INTEL_TCC); diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 11a7f8108bbb..61c3d450ee60 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -108,11 +108,11 @@ static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu) static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp) { struct zone_device *zonedev = thermal_zone_device_priv(tzd); - int val; + int val, ret; - val = intel_tcc_get_temp(zonedev->cpu, true); - if (val < 0) - return val; + ret = intel_tcc_get_temp(zonedev->cpu, &val, true); + if (ret < 0) + return ret; *temp = val * 1000; pr_debug("sys_get_curr_temp %d\n", *temp); diff --git a/include/linux/intel_tcc.h b/include/linux/intel_tcc.h index f422612c28d6..8ff8eabb4a98 100644 --- a/include/linux/intel_tcc.h +++ b/include/linux/intel_tcc.h @@ -13,6 +13,6 @@ int intel_tcc_get_tjmax(int cpu); int intel_tcc_get_offset(int cpu); int intel_tcc_set_offset(int cpu, int offset); -int intel_tcc_get_temp(int cpu, bool pkg); +int intel_tcc_get_temp(int cpu, int *temp, bool pkg); #endif /* __INTEL_TCC_H__ */ From 9bb6c395b0ffeac878fd1509b2bbb976346919fd Mon Sep 17 00:00:00 2001 From: RinHizakura Date: Fri, 2 Feb 2024 23:21:20 +0800 Subject: [PATCH 0287/1038] Documentation: PM: Fix PCI hibernation support description According to the context, 'pci_pm_suspend_noirq' is the right word for the changed sentence. Signed-off-by: Yiwei Lin [ rjw: Subject edits ] Signed-off-by: Rafael J. Wysocki --- Documentation/power/pci.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/power/pci.rst b/Documentation/power/pci.rst index a125544b4cb6..12070320307e 100644 --- a/Documentation/power/pci.rst +++ b/Documentation/power/pci.rst @@ -625,7 +625,7 @@ The PCI subsystem-level callbacks they correspond to:: pci_pm_poweroff() pci_pm_poweroff_noirq() -work in analogy with pci_pm_suspend() and pci_pm_poweroff_noirq(), respectively, +work in analogy with pci_pm_suspend() and pci_pm_suspend_noirq(), respectively, although they don't attempt to save the device's standard configuration registers. From 1e518e8333ca2134a53b96bc835432083f274350 Mon Sep 17 00:00:00 2001 From: Wentong Wu Date: Wed, 7 Feb 2024 08:59:08 +0800 Subject: [PATCH 0288/1038] ACPI: scan: Defer enumeration of devices with a _DEP pointing to IVSC device Inside IVSC, switching ownership requires an interface with two different hardware modules, ACE and CSI. The software interface to these modules is based on Intel MEI framework. Usually mei client devices are dynamically created, so the info of consumers depending on mei client devices is not present in the firmware tables. This causes problems with the probe ordering with respect to drivers for consumers of these MEI client devices. But on these camera sensor devices, the ACPI nodes describing the sensors all have a _DEP dependency on the matching MEI bus ACPI device, so adding IVSC MEI bus ACPI device to acpi_honor_dep_ids allows solving the probe-ordering problem by deferring the enumeration of ACPI-devices which have a _DEP dependency on an IVSC mei bus ACPI device. Add INTC10CF, the HID of IVSC MEI bus ACPI device on MTL platform, to acpi_honor_dep_ids. Signed-off-by: Wentong Wu Signed-off-by: Rafael J. Wysocki --- drivers/acpi/scan.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index e6ed1ba91e5c..f32a2c738c8b 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -798,6 +798,7 @@ static const char * const acpi_honor_dep_ids[] = { "INTC1059", /* IVSC (TGL) driver must be loaded to allow i2c access to camera sensors */ "INTC1095", /* IVSC (ADL) driver must be loaded to allow i2c access to camera sensors */ "INTC100A", /* IVSC (RPL) driver must be loaded to allow i2c access to camera sensors */ + "INTC10CF", /* IVSC (MTL) driver must be loaded to allow i2c access to camera sensors */ NULL }; From 7edbb6059274c0ee997947978a46d91664c64983 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Fri, 9 Feb 2024 14:03:06 +0100 Subject: [PATCH 0289/1038] ahci: clean up intel_pcs_quirk The comment in front of board_ahci_pcs7 is completely wrong. It claims that board_ahci_pcs7 is needing the quirk, but in fact, the logic implemented in ahci_intel_pcs_quirk() is the exact opposite, only board_ahci_pcs7 is _excluded_ from the quirk. This way of implementing a quirk is unconventional in several ways: First of all because it has a board ID for which the quirk should _not_ be applied (board_ahci_pcs7), instead of the usual way where we have a board ID for which the quirk should be applied. The second reason is that other than only excluding board_ahci_pcs7 from the quirk, PCI devices that make use of the generic entry in ahci_pci_tbl (which matches on AHCI class code) are also excluded. This can of course lead to very subtle breakage, and did indeed do so in: commit 104ff59af73a ("ata: ahci: Add Tiger Lake UP{3,4} AHCI controller"), which added an explicit entry with board_ahci_low_power to ahci_pci_tbl. This caused many users to complain that their SATA drives disappeared. The logical assumption was of course that the issue was related to LPM, and was therefore reverted in commit 6210038aeaf4 ("ata: ahci: Revert "ata: ahci: Add Tiger Lake UP{3,4} AHCI controller""). It took a lot of time to figure out that this was all completely unrelated to LPM, and was instead caused by an unconventional Intel quirk. Clean up the quirk so that it behaves like other quirks, i.e. define a board where the quirk is applied. Platforms that were using board_ahci_pcs7 are converted to use board_ahci, this is safe since the boards were identical, and board_ahci_pcs7 did not define any custom port_ops. This way, new Intel platforms can be added using the correct "board_ahci" board, without getting any unexpected quirks applied. This means that we currently have some modern platforms defined that are using the Intel PCS quirk, but that is identical to the behavior that was there before this commit. No functional changes intended. Link: https://bugzilla.kernel.org/show_bug.cgi?id=217114 Reviewed-by: Mika Westerberg Reviewed-by: Dan Williams Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 361 ++++++++++++++++++++++----------------------- drivers/ata/ahci.h | 1 + 2 files changed, 180 insertions(+), 182 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index ae0a592e2185..41b4c9777c85 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -54,6 +54,14 @@ enum board_ids { board_ahci_nomsi, board_ahci_noncq, board_ahci_nosntf, + /* + * board_ahci_pcs_quirk is for legacy Intel platforms. + * Modern Intel platforms should use board_ahci instead. + * (Some modern Intel platforms might have been added with + * board_ahci_pcs_quirk, however, we cannot change them to board_ahci + * without testing that the platform actually works without the quirk.) + */ + board_ahci_pcs_quirk, board_ahci_yes_fbs, /* board IDs for specific chipsets in alphabetical order */ @@ -67,12 +75,6 @@ enum board_ids { board_ahci_sb700, /* for SB700 and SB800 */ board_ahci_vt8251, - /* - * board IDs for Intel chipsets that support more than 6 ports - * *and* end up needing the PCS quirk. - */ - board_ahci_pcs7, - /* aliases */ board_ahci_mcp_linux = board_ahci_mcp65, board_ahci_mcp67 = board_ahci_mcp65, @@ -164,7 +166,15 @@ static const struct ata_port_info ahci_port_info[] = { .port_ops = &ahci_ops, }, [board_ahci_nosntf] = { - AHCI_HFLAGS (AHCI_HFLAG_NO_SNTF), + AHCI_HFLAGS (AHCI_HFLAG_NO_SNTF | + AHCI_HFLAG_INTEL_PCS_QUIRK), + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_ops, + }, + [board_ahci_pcs_quirk] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, @@ -186,6 +196,7 @@ static const struct ata_port_info ahci_port_info[] = { .port_ops = &ahci_ops, }, [board_ahci_avn] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, @@ -244,119 +255,113 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_vt8251_ops, }, - [board_ahci_pcs7] = { - .flags = AHCI_FLAG_COMMON, - .pio_mask = ATA_PIO4, - .udma_mask = ATA_UDMA6, - .port_ops = &ahci_ops, - }, }; static const struct pci_device_id ahci_pci_tbl[] = { /* Intel */ - { PCI_VDEVICE(INTEL, 0x06d6), board_ahci }, /* Comet Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */ - { PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */ - { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */ - { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */ - { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */ + { PCI_VDEVICE(INTEL, 0x06d6), board_ahci_pcs_quirk }, /* Comet Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x2652), board_ahci_pcs_quirk }, /* ICH6 */ + { PCI_VDEVICE(INTEL, 0x2653), board_ahci_pcs_quirk }, /* ICH6M */ + { PCI_VDEVICE(INTEL, 0x27c1), board_ahci_pcs_quirk }, /* ICH7 */ + { PCI_VDEVICE(INTEL, 0x27c5), board_ahci_pcs_quirk }, /* ICH7M */ + { PCI_VDEVICE(INTEL, 0x27c3), board_ahci_pcs_quirk }, /* ICH7R */ { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */ - { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */ - { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */ - { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */ - { PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */ - { PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */ + { PCI_VDEVICE(INTEL, 0x2681), board_ahci_pcs_quirk }, /* ESB2 */ + { PCI_VDEVICE(INTEL, 0x2682), board_ahci_pcs_quirk }, /* ESB2 */ + { PCI_VDEVICE(INTEL, 0x2683), board_ahci_pcs_quirk }, /* ESB2 */ + { PCI_VDEVICE(INTEL, 0x27c6), board_ahci_pcs_quirk }, /* ICH7-M DH */ + { PCI_VDEVICE(INTEL, 0x2821), board_ahci_pcs_quirk }, /* ICH8 */ { PCI_VDEVICE(INTEL, 0x2822), board_ahci_nosntf }, /* ICH8/Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */ - { PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */ - { PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */ - { PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */ - { PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */ - { PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */ - { PCI_VDEVICE(INTEL, 0x3a22), board_ahci }, /* ICH10 */ - { PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */ - { PCI_VDEVICE(INTEL, 0x3b22), board_ahci }, /* PCH AHCI */ - { PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */ - { PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b29), board_ahci }, /* PCH M AHCI */ - { PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci }, /* PCH M RAID */ - { PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */ - { PCI_VDEVICE(INTEL, 0x19b0), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b1), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b2), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b3), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b4), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b5), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b6), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b7), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19bE), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19bF), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c0), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c1), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c2), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c3), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c4), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c5), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c6), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c7), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19cE), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19cF), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */ - { PCI_VDEVICE(INTEL, 0x1c03), board_ahci }, /* CPT M AHCI */ - { PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1c05), board_ahci }, /* CPT M RAID */ - { PCI_VDEVICE(INTEL, 0x1c06), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1c07), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1d02), board_ahci }, /* PBG AHCI */ - { PCI_VDEVICE(INTEL, 0x1d04), board_ahci }, /* PBG RAID */ - { PCI_VDEVICE(INTEL, 0x1d06), board_ahci }, /* PBG RAID */ - { PCI_VDEVICE(INTEL, 0x2323), board_ahci }, /* DH89xxCC AHCI */ - { PCI_VDEVICE(INTEL, 0x1e02), board_ahci }, /* Panther Point AHCI */ - { PCI_VDEVICE(INTEL, 0x1e03), board_ahci }, /* Panther M AHCI */ - { PCI_VDEVICE(INTEL, 0x1e04), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e05), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e06), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e07), board_ahci }, /* Panther M RAID */ - { PCI_VDEVICE(INTEL, 0x1e0e), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c02), board_ahci }, /* Lynx Point AHCI */ - { PCI_VDEVICE(INTEL, 0x8c03), board_ahci }, /* Lynx M AHCI */ - { PCI_VDEVICE(INTEL, 0x8c04), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c05), board_ahci }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x8c06), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c07), board_ahci }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x8c0e), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c0f), board_ahci }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x9c02), board_ahci }, /* Lynx LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c03), board_ahci }, /* Lynx LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c04), board_ahci }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c05), board_ahci }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c06), board_ahci }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c07), board_ahci }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c0e), board_ahci }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c0f), board_ahci }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9dd3), board_ahci }, /* Cannon Lake PCH-LP AHCI */ - { PCI_VDEVICE(INTEL, 0x1f22), board_ahci }, /* Avoton AHCI */ - { PCI_VDEVICE(INTEL, 0x1f23), board_ahci }, /* Avoton AHCI */ - { PCI_VDEVICE(INTEL, 0x1f24), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f25), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f26), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f27), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f2e), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f2f), board_ahci }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x2824), board_ahci_pcs_quirk }, /* ICH8 */ + { PCI_VDEVICE(INTEL, 0x2829), board_ahci_pcs_quirk }, /* ICH8M */ + { PCI_VDEVICE(INTEL, 0x282a), board_ahci_pcs_quirk }, /* ICH8M */ + { PCI_VDEVICE(INTEL, 0x2922), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2923), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2924), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2925), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2927), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2929), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292a), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292b), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292c), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292f), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x294d), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x294e), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x502a), board_ahci_pcs_quirk }, /* Tolapai */ + { PCI_VDEVICE(INTEL, 0x502b), board_ahci_pcs_quirk }, /* Tolapai */ + { PCI_VDEVICE(INTEL, 0x3a05), board_ahci_pcs_quirk }, /* ICH10 */ + { PCI_VDEVICE(INTEL, 0x3a22), board_ahci_pcs_quirk }, /* ICH10 */ + { PCI_VDEVICE(INTEL, 0x3a25), board_ahci_pcs_quirk }, /* ICH10 */ + { PCI_VDEVICE(INTEL, 0x3b22), board_ahci_pcs_quirk }, /* PCH AHCI */ + { PCI_VDEVICE(INTEL, 0x3b23), board_ahci_pcs_quirk }, /* PCH AHCI */ + { PCI_VDEVICE(INTEL, 0x3b24), board_ahci_pcs_quirk }, /* PCH RAID */ + { PCI_VDEVICE(INTEL, 0x3b25), board_ahci_pcs_quirk }, /* PCH RAID */ + { PCI_VDEVICE(INTEL, 0x3b29), board_ahci_pcs_quirk }, /* PCH M AHCI */ + { PCI_VDEVICE(INTEL, 0x3b2b), board_ahci_pcs_quirk }, /* PCH RAID */ + { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_pcs_quirk }, /* PCH M RAID */ + { PCI_VDEVICE(INTEL, 0x3b2f), board_ahci_pcs_quirk }, /* PCH AHCI */ + { PCI_VDEVICE(INTEL, 0x19b0), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b1), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b2), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b3), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b4), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b5), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b6), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b7), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19bE), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19bF), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c0), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c1), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c2), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c3), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c4), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c5), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c6), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c7), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19cE), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19cF), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x1c02), board_ahci_pcs_quirk }, /* CPT AHCI */ + { PCI_VDEVICE(INTEL, 0x1c03), board_ahci_pcs_quirk }, /* CPT M AHCI */ + { PCI_VDEVICE(INTEL, 0x1c04), board_ahci_pcs_quirk }, /* CPT RAID */ + { PCI_VDEVICE(INTEL, 0x1c05), board_ahci_pcs_quirk }, /* CPT M RAID */ + { PCI_VDEVICE(INTEL, 0x1c06), board_ahci_pcs_quirk }, /* CPT RAID */ + { PCI_VDEVICE(INTEL, 0x1c07), board_ahci_pcs_quirk }, /* CPT RAID */ + { PCI_VDEVICE(INTEL, 0x1d02), board_ahci_pcs_quirk }, /* PBG AHCI */ + { PCI_VDEVICE(INTEL, 0x1d04), board_ahci_pcs_quirk }, /* PBG RAID */ + { PCI_VDEVICE(INTEL, 0x1d06), board_ahci_pcs_quirk }, /* PBG RAID */ + { PCI_VDEVICE(INTEL, 0x2323), board_ahci_pcs_quirk }, /* DH89xxCC AHCI */ + { PCI_VDEVICE(INTEL, 0x1e02), board_ahci_pcs_quirk }, /* Panther Point AHCI */ + { PCI_VDEVICE(INTEL, 0x1e03), board_ahci_pcs_quirk }, /* Panther M AHCI */ + { PCI_VDEVICE(INTEL, 0x1e04), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x1e05), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x1e06), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x1e07), board_ahci_pcs_quirk }, /* Panther M RAID */ + { PCI_VDEVICE(INTEL, 0x1e0e), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c02), board_ahci_pcs_quirk }, /* Lynx Point AHCI */ + { PCI_VDEVICE(INTEL, 0x8c03), board_ahci_pcs_quirk }, /* Lynx M AHCI */ + { PCI_VDEVICE(INTEL, 0x8c04), board_ahci_pcs_quirk }, /* Lynx Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c05), board_ahci_pcs_quirk }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x8c06), board_ahci_pcs_quirk }, /* Lynx Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c07), board_ahci_pcs_quirk }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x8c0e), board_ahci_pcs_quirk }, /* Lynx Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_pcs_quirk }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x9c02), board_ahci_pcs_quirk }, /* Lynx LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c03), board_ahci_pcs_quirk }, /* Lynx LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c04), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c05), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c06), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c07), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9dd3), board_ahci_pcs_quirk }, /* Cannon Lake PCH-LP AHCI */ + { PCI_VDEVICE(INTEL, 0x1f22), board_ahci_pcs_quirk }, /* Avoton AHCI */ + { PCI_VDEVICE(INTEL, 0x1f23), board_ahci_pcs_quirk }, /* Avoton AHCI */ + { PCI_VDEVICE(INTEL, 0x1f24), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f25), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f26), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f27), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f2e), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f2f), board_ahci_pcs_quirk }, /* Avoton RAID */ { PCI_VDEVICE(INTEL, 0x1f32), board_ahci_avn }, /* Avoton AHCI */ { PCI_VDEVICE(INTEL, 0x1f33), board_ahci_avn }, /* Avoton AHCI */ { PCI_VDEVICE(INTEL, 0x1f34), board_ahci_avn }, /* Avoton RAID */ @@ -365,65 +370,65 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x1f37), board_ahci_avn }, /* Avoton RAID */ { PCI_VDEVICE(INTEL, 0x1f3e), board_ahci_avn }, /* Avoton RAID */ { PCI_VDEVICE(INTEL, 0x1f3f), board_ahci_avn }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Wellsburg/Lewisburg AHCI*/ - { PCI_VDEVICE(INTEL, 0x2826), board_ahci }, /* *burg SATA0 'RAID' */ - { PCI_VDEVICE(INTEL, 0x2827), board_ahci }, /* *burg SATA1 'RAID' */ - { PCI_VDEVICE(INTEL, 0x282f), board_ahci }, /* *burg SATA2 'RAID' */ - { PCI_VDEVICE(INTEL, 0x43d4), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x43d5), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x43d6), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x43d7), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x8d02), board_ahci }, /* Wellsburg AHCI */ - { PCI_VDEVICE(INTEL, 0x8d04), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d06), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d0e), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d62), board_ahci }, /* Wellsburg AHCI */ - { PCI_VDEVICE(INTEL, 0x8d64), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d66), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d6e), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x23a3), board_ahci }, /* Coleto Creek AHCI */ - { PCI_VDEVICE(INTEL, 0x9c83), board_ahci }, /* Wildcat LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c85), board_ahci }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c87), board_ahci }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c8f), board_ahci }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x8c82), board_ahci }, /* 9 Series AHCI */ - { PCI_VDEVICE(INTEL, 0x8c83), board_ahci }, /* 9 Series M AHCI */ - { PCI_VDEVICE(INTEL, 0x8c84), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c85), board_ahci }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x8c86), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c87), board_ahci }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x8c8e), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c8f), board_ahci }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x9d03), board_ahci }, /* Sunrise LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9d05), board_ahci }, /* Sunrise LP RAID */ - { PCI_VDEVICE(INTEL, 0x9d07), board_ahci }, /* Sunrise LP RAID */ - { PCI_VDEVICE(INTEL, 0xa102), board_ahci }, /* Sunrise Point-H AHCI */ - { PCI_VDEVICE(INTEL, 0xa103), board_ahci }, /* Sunrise M AHCI */ - { PCI_VDEVICE(INTEL, 0xa105), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa106), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa107), board_ahci }, /* Sunrise M RAID */ - { PCI_VDEVICE(INTEL, 0xa10f), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa182), board_ahci }, /* Lewisburg AHCI*/ - { PCI_VDEVICE(INTEL, 0xa186), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa1d2), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa1d6), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa202), board_ahci }, /* Lewisburg AHCI*/ - { PCI_VDEVICE(INTEL, 0xa206), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa252), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa256), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa356), board_ahci }, /* Cannon Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x06d7), board_ahci }, /* Comet Lake-H RAID */ - { PCI_VDEVICE(INTEL, 0xa386), board_ahci }, /* Comet Lake PCH-V RAID */ - { PCI_VDEVICE(INTEL, 0x0f22), board_ahci }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x0f23), board_ahci }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x22a3), board_ahci }, /* Cherry Tr. AHCI */ - { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci }, /* ApolloLake AHCI */ - { PCI_VDEVICE(INTEL, 0x34d3), board_ahci }, /* Ice Lake LP AHCI */ - { PCI_VDEVICE(INTEL, 0x02d3), board_ahci }, /* Comet Lake PCH-U AHCI */ - { PCI_VDEVICE(INTEL, 0x02d7), board_ahci }, /* Comet Lake PCH RAID */ + { PCI_VDEVICE(INTEL, 0x2823), board_ahci_pcs_quirk }, /* Wellsburg/Lewisburg AHCI*/ + { PCI_VDEVICE(INTEL, 0x2826), board_ahci_pcs_quirk }, /* *burg SATA0 'RAID' */ + { PCI_VDEVICE(INTEL, 0x2827), board_ahci_pcs_quirk }, /* *burg SATA1 'RAID' */ + { PCI_VDEVICE(INTEL, 0x282f), board_ahci_pcs_quirk }, /* *burg SATA2 'RAID' */ + { PCI_VDEVICE(INTEL, 0x43d4), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x43d5), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x43d6), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x43d7), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x8d02), board_ahci_pcs_quirk }, /* Wellsburg AHCI */ + { PCI_VDEVICE(INTEL, 0x8d04), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d06), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d0e), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d62), board_ahci_pcs_quirk }, /* Wellsburg AHCI */ + { PCI_VDEVICE(INTEL, 0x8d64), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d66), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d6e), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x23a3), board_ahci_pcs_quirk }, /* Coleto Creek AHCI */ + { PCI_VDEVICE(INTEL, 0x9c83), board_ahci_pcs_quirk }, /* Wildcat LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c85), board_ahci_pcs_quirk }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c87), board_ahci_pcs_quirk }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_pcs_quirk }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x8c82), board_ahci_pcs_quirk }, /* 9 Series AHCI */ + { PCI_VDEVICE(INTEL, 0x8c83), board_ahci_pcs_quirk }, /* 9 Series M AHCI */ + { PCI_VDEVICE(INTEL, 0x8c84), board_ahci_pcs_quirk }, /* 9 Series RAID */ + { PCI_VDEVICE(INTEL, 0x8c85), board_ahci_pcs_quirk }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x8c86), board_ahci_pcs_quirk }, /* 9 Series RAID */ + { PCI_VDEVICE(INTEL, 0x8c87), board_ahci_pcs_quirk }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x8c8e), board_ahci_pcs_quirk }, /* 9 Series RAID */ + { PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_pcs_quirk }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x9d03), board_ahci_pcs_quirk }, /* Sunrise LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9d05), board_ahci_pcs_quirk }, /* Sunrise LP RAID */ + { PCI_VDEVICE(INTEL, 0x9d07), board_ahci_pcs_quirk }, /* Sunrise LP RAID */ + { PCI_VDEVICE(INTEL, 0xa102), board_ahci_pcs_quirk }, /* Sunrise Point-H AHCI */ + { PCI_VDEVICE(INTEL, 0xa103), board_ahci_pcs_quirk }, /* Sunrise M AHCI */ + { PCI_VDEVICE(INTEL, 0xa105), board_ahci_pcs_quirk }, /* Sunrise Point-H RAID */ + { PCI_VDEVICE(INTEL, 0xa106), board_ahci_pcs_quirk }, /* Sunrise Point-H RAID */ + { PCI_VDEVICE(INTEL, 0xa107), board_ahci_pcs_quirk }, /* Sunrise M RAID */ + { PCI_VDEVICE(INTEL, 0xa10f), board_ahci_pcs_quirk }, /* Sunrise Point-H RAID */ + { PCI_VDEVICE(INTEL, 0xa182), board_ahci_pcs_quirk }, /* Lewisburg AHCI*/ + { PCI_VDEVICE(INTEL, 0xa186), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa1d2), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa1d6), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa202), board_ahci_pcs_quirk }, /* Lewisburg AHCI*/ + { PCI_VDEVICE(INTEL, 0xa206), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa252), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa256), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa356), board_ahci_pcs_quirk }, /* Cannon Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x06d7), board_ahci_pcs_quirk }, /* Comet Lake-H RAID */ + { PCI_VDEVICE(INTEL, 0xa386), board_ahci_pcs_quirk }, /* Comet Lake PCH-V RAID */ + { PCI_VDEVICE(INTEL, 0x0f22), board_ahci_pcs_quirk }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x0f23), board_ahci_pcs_quirk }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x22a3), board_ahci_pcs_quirk }, /* Cherry Tr. AHCI */ + { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_pcs_quirk }, /* ApolloLake AHCI */ + { PCI_VDEVICE(INTEL, 0x34d3), board_ahci_pcs_quirk }, /* Ice Lake LP AHCI */ + { PCI_VDEVICE(INTEL, 0x02d3), board_ahci_pcs_quirk }, /* Comet Lake PCH-U AHCI */ + { PCI_VDEVICE(INTEL, 0x02d7), board_ahci_pcs_quirk }, /* Comet Lake PCH RAID */ /* Elkhart Lake IDs 0x4b60 & 0x4b62 https://sata-io.org/product/8803 not tested yet */ - { PCI_VDEVICE(INTEL, 0x4b63), board_ahci }, /* Elkhart Lake AHCI */ - { PCI_VDEVICE(INTEL, 0x7ae2), board_ahci }, /* Alder Lake-P AHCI */ + { PCI_VDEVICE(INTEL, 0x4b63), board_ahci_pcs_quirk }, /* Elkhart Lake AHCI */ + { PCI_VDEVICE(INTEL, 0x7ae2), board_ahci_pcs_quirk }, /* Alder Lake-P AHCI */ /* JMicron 360/1/3/5/6, match class to avoid IDE function */ { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, @@ -458,7 +463,7 @@ static const struct pci_device_id ahci_pci_tbl[] = { /* Dell S140/S150 */ { PCI_VENDOR_ID_INTEL, PCI_ANY_ID, PCI_SUBVENDOR_ID_DELL, PCI_ANY_ID, - PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci }, + PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci_pcs_quirk }, /* VIA */ { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */ @@ -1681,17 +1686,9 @@ update_policy: static void ahci_intel_pcs_quirk(struct pci_dev *pdev, struct ahci_host_priv *hpriv) { - const struct pci_device_id *id = pci_match_id(ahci_pci_tbl, pdev); u16 tmp16; - /* - * Only apply the 6-port PCS quirk for known legacy platforms. - */ - if (!id || id->vendor != PCI_VENDOR_ID_INTEL) - return; - - /* Skip applying the quirk on Denverton and beyond */ - if (((enum board_ids) id->driver_data) >= board_ahci_pcs7) + if (!(hpriv->flags & AHCI_HFLAG_INTEL_PCS_QUIRK)) return; /* diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index 4a0a602c6b16..344c87210d8f 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -245,6 +245,7 @@ enum { suspend/resume */ AHCI_HFLAG_NO_SXS = BIT(26), /* SXS not supported */ AHCI_HFLAG_43BIT_ONLY = BIT(27), /* 43bit DMA addr limit */ + AHCI_HFLAG_INTEL_PCS_QUIRK = BIT(28), /* apply Intel PCS quirk */ /* ap->flags bits */ From 05d103950352d35c99142fe52ab50a8280dff7aa Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sat, 10 Feb 2024 17:38:56 +0100 Subject: [PATCH 0290/1038] dt-bindings: power: rpmpd: Add MSM8974 power domains Add the compatibles and indexes for the rpmpd in MSM8974, both with the standard PM8841+PM8941 PMICs but also devices found with PMA8084. Signed-off-by: Luca Weiss Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240210-msm8974-rpmpd-v2-1-595e2ff80ea1@z3ntu.xyz Signed-off-by: Ulf Hansson --- Documentation/devicetree/bindings/power/qcom,rpmpd.yaml | 2 ++ include/dt-bindings/power/qcom-rpmpd.h | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml index 2ff246cf8b81..929b7ef9c1bc 100644 --- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml +++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml @@ -24,6 +24,8 @@ properties: - qcom,msm8917-rpmpd - qcom,msm8939-rpmpd - qcom,msm8953-rpmpd + - qcom,msm8974-rpmpd + - qcom,msm8974pro-pma8084-rpmpd - qcom,msm8976-rpmpd - qcom,msm8994-rpmpd - qcom,msm8996-rpmpd diff --git a/include/dt-bindings/power/qcom-rpmpd.h b/include/dt-bindings/power/qcom-rpmpd.h index 7f4e2983a4c5..608087fb9a3d 100644 --- a/include/dt-bindings/power/qcom-rpmpd.h +++ b/include/dt-bindings/power/qcom-rpmpd.h @@ -308,6 +308,13 @@ #define MSM8953_VDDMX 5 #define MSM8953_VDDMX_AO 6 +/* MSM8974 Power Domain Indexes */ +#define MSM8974_VDDCX 0 +#define MSM8974_VDDCX_AO 1 +#define MSM8974_VDDCX_VFC 2 +#define MSM8974_VDDGFX 3 +#define MSM8974_VDDGFX_VFC 4 + /* MSM8976 Power Domain Indexes */ #define MSM8976_VDDCX 0 #define MSM8976_VDDCX_AO 1 From 18cbe81639f2c8999f9ad0505a08ddfc01cd05f7 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 14 Jan 2024 16:01:57 +0100 Subject: [PATCH 0291/1038] mmc: core: Remove usage of the deprecated ida_simple_xx() API ida_alloc() and ida_free() should be preferred to the deprecated ida_simple_get() and ida_simple_remove(). Note that the upper limit of ida_simple_get() is exclusive, but the one of ida_alloc_range()/ida_alloc_max() is inclusive. So a -1 has been added when needed. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/583c57d0ae09f9d3a1e1a7b80c1e39ada17954b7.1705244502.git.christophe.jaillet@wanadoo.fr Signed-off-by: Ulf Hansson --- drivers/mmc/core/block.c | 12 ++++++------ drivers/mmc/core/host.c | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 32d49100dff5..a9b60b91e32f 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -206,7 +206,7 @@ static void mmc_blk_kref_release(struct kref *ref) int devidx; devidx = mmc_get_devidx(md->disk); - ida_simple_remove(&mmc_blk_ida, devidx); + ida_free(&mmc_blk_ida, devidx); mutex_lock(&open_lock); md->disk->private_data = NULL; @@ -2467,7 +2467,7 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, bool cache_enabled = false; bool fua_enabled = false; - devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL); + devidx = ida_alloc_max(&mmc_blk_ida, max_devices - 1, GFP_KERNEL); if (devidx < 0) { /* * We get -ENOSPC because there are no more any available @@ -2577,7 +2577,7 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, err_kfree: kfree(md); out: - ida_simple_remove(&mmc_blk_ida, devidx); + ida_free(&mmc_blk_ida, devidx); return ERR_PTR(ret); } @@ -2703,7 +2703,7 @@ static void mmc_blk_rpmb_device_release(struct device *dev) { struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev); - ida_simple_remove(&mmc_rpmb_ida, rpmb->id); + ida_free(&mmc_rpmb_ida, rpmb->id); kfree(rpmb); } @@ -2719,13 +2719,13 @@ static int mmc_blk_alloc_rpmb_part(struct mmc_card *card, struct mmc_rpmb_data *rpmb; /* This creates the minor number for the RPMB char device */ - devidx = ida_simple_get(&mmc_rpmb_ida, 0, max_devices, GFP_KERNEL); + devidx = ida_alloc_max(&mmc_rpmb_ida, max_devices - 1, GFP_KERNEL); if (devidx < 0) return devidx; rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL); if (!rpmb) { - ida_simple_remove(&mmc_rpmb_ida, devidx); + ida_free(&mmc_rpmb_ida, devidx); return -ENOMEM; } diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index cf396e8f34e9..7cc9a33d28ca 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -76,7 +76,7 @@ static void mmc_host_classdev_release(struct device *dev) struct mmc_host *host = cls_dev_to_mmc_host(dev); wakeup_source_unregister(host->ws); if (of_alias_get_id(host->parent->of_node, "mmc") < 0) - ida_simple_remove(&mmc_host_ida, host->index); + ida_free(&mmc_host_ida, host->index); kfree(host); } @@ -538,7 +538,8 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev) min_idx = mmc_first_nonreserved_index(); max_idx = 0; - index = ida_simple_get(&mmc_host_ida, min_idx, max_idx, GFP_KERNEL); + index = ida_alloc_range(&mmc_host_ida, min_idx, max_idx - 1, + GFP_KERNEL); if (index < 0) { kfree(host); return NULL; From c64ec3a7d9e93e2e0e06bdc4b1ce760b54676c97 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 22 Jan 2024 17:16:23 +0800 Subject: [PATCH 0292/1038] dt-bindings: mmc: fsl-imx-esdhc: add i.MX95 compatible string Same as i.MX93, add i.MX95 SDHC which is compatible with i.MX8MM USDHC. Signed-off-by: Peng Fan Link: https://lore.kernel.org/r/20240122091623.2078089-1-peng.fan@oss.nxp.com Signed-off-by: Ulf Hansson --- Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml index 82eb7a24c857..f3c5aa64affc 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml @@ -55,8 +55,9 @@ properties: - enum: - fsl,imx8mn-usdhc - fsl,imx8mp-usdhc - - fsl,imx93-usdhc - fsl,imx8ulp-usdhc + - fsl,imx93-usdhc + - fsl,imx95-usdhc - const: fsl,imx8mm-usdhc - items: - enum: From 069279d6fef57b06d5489d6aa8775630903c0da2 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Thu, 25 Jan 2024 09:50:23 +0100 Subject: [PATCH 0293/1038] mmc: core Drop BLK_BOUNCE_HIGH The MMC core sets BLK_BOUNCE_HIGH for devices where dma_mask is unassigned. For the majority of MMC hosts this path is never taken: the OF core will unconditionally assign a 32-bit mask to any OF device, and most MMC hosts are probed from device tree, see drivers/of/platform.c: of_platform_device_create_pdata() dev->dev.coherent_dma_mask = DMA_BIT_MASK(32); if (!dev->dev.dma_mask) dev->dev.dma_mask = &dev->dev.coherent_dma_mask; of_amba_device_create() dev->dev.coherent_dma_mask = DMA_BIT_MASK(32); dev->dev.dma_mask = &dev->dev.coherent_dma_mask; MMC devices that are probed from ACPI or PCI will likewise have a proper dma_mask assigned. The only remaining devices that could have a blank dma_mask are platform devices instantiated from board files. These are mostly used on systems without CONFIG_HIGHMEM enabled which means the block layer will not bounce, and in the few cases where it is enabled it is not used anyway: for example some OMAP2 systems such as Nokia n800/n810 will create a platform_device and not assign a dma_mask, however they do not have any highmem, so no bouncing will happen anyway: the block core checks if max_low_pfn >= max_pfn and this will always be false. Should it turn out there is a platform_device with blank DMA mask actually using CONFIG_HIGHMEM somewhere out there we should set dma_mask for it, not do this trickery. Signed-off-by: Linus Walleij Acked-by: Arnd Bergmann Acked-by: Christoph Hellwig Link: https://lore.kernel.org/r/20240125-mmc-no-blk-bounce-high-v1-1-d0f92a30e085@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/core/queue.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index a0a2412f62a7..316415588a77 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -351,8 +351,6 @@ static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card) if (mmc_can_erase(card)) mmc_queue_setup_discard(mq->queue, card); - if (!mmc_dev(host)->dma_mask || !*mmc_dev(host)->dma_mask) - blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_HIGH); blk_queue_max_hw_sectors(mq->queue, min(host->max_blk_count, host->max_req_size / 512)); if (host->can_dma_map_merge) From 6bf26a0e5e9b11f7f4a614e539b9ed511e2fc0cd Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 16:02:00 -0300 Subject: [PATCH 0294/1038] mmc: core: make mmc_rpmb_bus_type const Now that the driver core can properly handle constant struct bus_type, move the mmc_rpmb_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-mmc-v1-1-ad054dce8dc3@marliere.net Signed-off-by: Ulf Hansson --- drivers/mmc/core/block.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index a9b60b91e32f..bd165cef2bbc 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -144,7 +144,7 @@ struct mmc_blk_data { static dev_t mmc_rpmb_devt; /* Bus type for RPMB character devices */ -static struct bus_type mmc_rpmb_bus_type = { +static const struct bus_type mmc_rpmb_bus_type = { .name = "mmc_rpmb", }; From 1ba28dee4b88f14887d0a6981bce80c5885f34c2 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 16:02:01 -0300 Subject: [PATCH 0295/1038] mmc: core: make mmc_bus_type const Now that the driver core can properly handle constant struct bus_type, move the mmc_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-mmc-v1-2-ad054dce8dc3@marliere.net Signed-off-by: Ulf Hansson --- drivers/mmc/core/bus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c index 0af96548e7da..48daeafdff7a 100644 --- a/drivers/mmc/core/bus.c +++ b/drivers/mmc/core/bus.c @@ -214,7 +214,7 @@ static const struct dev_pm_ops mmc_bus_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(mmc_bus_suspend, mmc_bus_resume) }; -static struct bus_type mmc_bus_type = { +static const struct bus_type mmc_bus_type = { .name = "mmc", .dev_groups = mmc_dev_groups, .uevent = mmc_bus_uevent, From 13dbc21bf80218acc51ab257d1976e3695b9d01a Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 16:02:02 -0300 Subject: [PATCH 0296/1038] mmc: core: make sdio_bus_type const Now that the driver core can properly handle constant struct bus_type, move the sdio_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-mmc-v1-3-ad054dce8dc3@marliere.net Signed-off-by: Ulf Hansson --- drivers/mmc/core/sdio_bus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/core/sdio_bus.c b/drivers/mmc/core/sdio_bus.c index 47a48e902a24..71d885fbc228 100644 --- a/drivers/mmc/core/sdio_bus.c +++ b/drivers/mmc/core/sdio_bus.c @@ -244,7 +244,7 @@ static const struct dev_pm_ops sdio_bus_pm_ops = { ) }; -static struct bus_type sdio_bus_type = { +static const struct bus_type sdio_bus_type = { .name = "sdio", .dev_groups = sdio_dev_groups, .match = sdio_bus_match, From cb8e8570d5c99e99715466d5616ad1bc3a4ad2f8 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 17:05:58 -0300 Subject: [PATCH 0297/1038] memstick: core: make memstick_bus_type const Now that the driver core can properly handle constant struct bus_type, move the memstick_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240204-bus_cleanup-memstick-v1-1-14809d4405d8@marliere.net Signed-off-by: Ulf Hansson --- drivers/memstick/core/memstick.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c index bbfaf6536903..23fea51ecbdd 100644 --- a/drivers/memstick/core/memstick.c +++ b/drivers/memstick/core/memstick.c @@ -164,7 +164,7 @@ static struct attribute *memstick_dev_attrs[] = { }; ATTRIBUTE_GROUPS(memstick_dev); -static struct bus_type memstick_bus_type = { +static const struct bus_type memstick_bus_type = { .name = "memstick", .dev_groups = memstick_dev_groups, .match = memstick_bus_match, From 3a999e57cf4bebe46070378013a3c3e7ac00e4d3 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 1 Feb 2024 15:22:41 -0500 Subject: [PATCH 0298/1038] dt-bindings: mmc: fsl-imx-esdhc: add iommus property iMX95 and iMX8QM have smmu. Add property "iommus". Signed-off-by: Frank Li Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240201-8qm_smmu-v2-1-3d12a80201a3@nxp.com Signed-off-by: Ulf Hansson --- Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml index f3c5aa64affc..3f6dcc773092 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml @@ -163,6 +163,9 @@ properties: - const: ahb - const: per + iommus: + maxItems: 1 + power-domains: maxItems: 1 From ed01d210fd910f7fa7933638df14ffb8d4aac2a9 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:48 +0100 Subject: [PATCH 0299/1038] mmc: davinci_mmc: Use sg_miter for PIO Use the scatterlist memory iterator instead of just dereferencing virtual memory using sg_virt(). This make highmem references work properly. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-1-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/davinci_mmc.c | 61 ++++++++++++++++------------------ 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c index ee3b1a4e0848..c46577305138 100644 --- a/drivers/mmc/host/davinci_mmc.c +++ b/drivers/mmc/host/davinci_mmc.c @@ -180,12 +180,6 @@ struct mmc_davinci_host { #define DAVINCI_MMC_DATADIR_WRITE 2 unsigned char data_dir; - /* buffer is used during PIO of one scatterlist segment, and - * is updated along with buffer_bytes_left. bytes_left applies - * to all N blocks of the PIO transfer. - */ - u8 *buffer; - u32 buffer_bytes_left; u32 bytes_left; struct dma_chan *dma_tx; @@ -196,8 +190,8 @@ struct mmc_davinci_host { bool active_request; /* For PIO we walk scatterlists one segment at a time. */ + struct sg_mapping_iter sg_miter; unsigned int sg_len; - struct scatterlist *sg; /* Version of the MMC/SD controller */ u8 version; @@ -213,30 +207,24 @@ struct mmc_davinci_host { static irqreturn_t mmc_davinci_irq(int irq, void *dev_id); /* PIO only */ -static void mmc_davinci_sg_to_buf(struct mmc_davinci_host *host) -{ - host->buffer_bytes_left = sg_dma_len(host->sg); - host->buffer = sg_virt(host->sg); - if (host->buffer_bytes_left > host->bytes_left) - host->buffer_bytes_left = host->bytes_left; -} - static void davinci_fifo_data_trans(struct mmc_davinci_host *host, unsigned int n) { + struct sg_mapping_iter *sgm = &host->sg_miter; + size_t sglen; u8 *p; unsigned int i; - if (host->buffer_bytes_left == 0) { - host->sg = sg_next(host->data->sg); - mmc_davinci_sg_to_buf(host); + /* + * By adjusting sgm->consumed this will give a pointer to the + * current index into the sgm. + */ + if (!sg_miter_next(sgm)) { + dev_err(mmc_dev(host->mmc), "ran out of sglist prematurely\n"); + return; } - - p = host->buffer; - if (n > host->buffer_bytes_left) - n = host->buffer_bytes_left; - host->buffer_bytes_left -= n; - host->bytes_left -= n; + p = sgm->addr; + sglen = sgm->length; /* NOTE: we never transfer more than rw_threshold bytes * to/from the fifo here; there's no I/O overlap. @@ -261,7 +249,9 @@ static void davinci_fifo_data_trans(struct mmc_davinci_host *host, p = p + (n & 3); } } - host->buffer = p; + + sgm->consumed = n; + host->bytes_left -= n; } static void mmc_davinci_start_command(struct mmc_davinci_host *host, @@ -517,6 +507,7 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) int fifo_lev = (rw_threshold == 32) ? MMCFIFOCTL_FIFOLEV : 0; int timeout; struct mmc_data *data = req->data; + unsigned int flags = SG_MITER_ATOMIC; /* Used from IRQ */ if (host->version == MMC_CTLR_VERSION_2) fifo_lev = (rw_threshold == 64) ? MMCFIFOCTL_FIFOLEV : 0; @@ -545,12 +536,14 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) /* Configure the FIFO */ if (data->flags & MMC_DATA_WRITE) { + flags |= SG_MITER_FROM_SG; host->data_dir = DAVINCI_MMC_DATADIR_WRITE; writel(fifo_lev | MMCFIFOCTL_FIFODIR_WR | MMCFIFOCTL_FIFORST, host->base + DAVINCI_MMCFIFOCTL); writel(fifo_lev | MMCFIFOCTL_FIFODIR_WR, host->base + DAVINCI_MMCFIFOCTL); } else { + flags |= SG_MITER_TO_SG; host->data_dir = DAVINCI_MMC_DATADIR_READ; writel(fifo_lev | MMCFIFOCTL_FIFODIR_RD | MMCFIFOCTL_FIFORST, host->base + DAVINCI_MMCFIFOCTL); @@ -558,7 +551,6 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) host->base + DAVINCI_MMCFIFOCTL); } - host->buffer = NULL; host->bytes_left = data->blocks * data->blksz; /* For now we try to use DMA whenever we won't need partial FIFO @@ -576,8 +568,7 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) } else { /* Revert to CPU Copy */ host->sg_len = data->sg_len; - host->sg = host->data->sg; - mmc_davinci_sg_to_buf(host); + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); } } @@ -843,6 +834,8 @@ davinci_abort_data(struct mmc_davinci_host *host, struct mmc_data *data) { mmc_davinci_reset_ctrl(host, 1); mmc_davinci_reset_ctrl(host, 0); + if (!host->do_dma) + sg_miter_stop(&host->sg_miter); } static irqreturn_t mmc_davinci_sdio_irq(int irq, void *dev_id) @@ -919,11 +912,13 @@ static irqreturn_t mmc_davinci_irq(int irq, void *dev_id) if (qstatus & MMCST0_DATDNE) { /* All blocks sent/received, and CRC checks passed */ if (data != NULL) { - if ((host->do_dma == 0) && (host->bytes_left > 0)) { - /* if datasize < rw_threshold - * no RX ints are generated - */ - davinci_fifo_data_trans(host, host->bytes_left); + if (!host->do_dma) { + if (host->bytes_left > 0) + /* if datasize < rw_threshold + * no RX ints are generated + */ + davinci_fifo_data_trans(host, host->bytes_left); + sg_miter_stop(&host->sg_miter); } end_transfer = 1; data->bytes_xfered = data->blocks * data->blksz; From bc169ad24308ff3304966af10bea2df9219a0189 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:49 +0100 Subject: [PATCH 0300/1038] mmc: moxart-mmc: Factor out moxart_use_dma() helper The same code is in two places and we will add a third place. Break this out into its own function. Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-2-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/moxart-mmc.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c index 5cfdd3a86e54..d12d7d79b19c 100644 --- a/drivers/mmc/host/moxart-mmc.c +++ b/drivers/mmc/host/moxart-mmc.c @@ -254,6 +254,11 @@ static void moxart_dma_complete(void *param) complete(&host->dma_complete); } +static bool moxart_use_dma(struct moxart_host *host) +{ + return (host->data_len > host->fifo_width) && host->have_dma; +} + static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host) { u32 len, dir_slave; @@ -375,7 +380,7 @@ static void moxart_prepare_data(struct moxart_host *host) if (data->flags & MMC_DATA_WRITE) datactrl |= DCR_DATA_WRITE; - if ((host->data_len > host->fifo_width) && host->have_dma) + if (moxart_use_dma(host)) datactrl |= DCR_DMA_EN; writel(DCR_DATA_FIFO_RESET, host->base + REG_DATA_CONTROL); @@ -407,7 +412,7 @@ static void moxart_request(struct mmc_host *mmc, struct mmc_request *mrq) moxart_send_command(host, host->mrq->cmd); if (mrq->cmd->data) { - if ((host->data_len > host->fifo_width) && host->have_dma) { + if (moxart_use_dma(host)) { writel(CARD_CHANGE, host->base + REG_INTERRUPT_MASK); From 54fd8cd61799673622a0c683e870ee584c713aa7 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:50 +0100 Subject: [PATCH 0301/1038] mmc: moxart-mmc: Fix accounting in DMA transfer The whole scatterlist chain is submitted to the DMA engine, but the code is written to just account for the length of the first sg entry. When the DMA transfer is finished, all the data in the request has been transferred, account for this instead. This only works because the moxart_request() function isn't checking that all data was transferred and will unconditionally issue mmc_request_done() after returning successfully from moxart_transfer_dma(). Keep the assignment of accounted bytes in .bytes_xfered but move it after the completion where we know it has actually happened. Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-3-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/moxart-mmc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c index d12d7d79b19c..8ede4ce93271 100644 --- a/drivers/mmc/host/moxart-mmc.c +++ b/drivers/mmc/host/moxart-mmc.c @@ -296,11 +296,11 @@ static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host) dma_async_issue_pending(dma_chan); } - data->bytes_xfered += host->data_remain; - wait_for_completion_interruptible_timeout(&host->dma_complete, host->timeout); + data->bytes_xfered = host->data_len; + dma_unmap_sg(dma_chan->device->dev, data->sg, data->sg_len, mmc_get_dma_dir(data)); From 3ee0e7c3e67cab83ffbbe7707b43df8d41c9fe47 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:51 +0100 Subject: [PATCH 0302/1038] mmc: moxart-mmc: Use sg_miter for PIO Use the scatterlist memory iterator instead of just dereferencing virtual memory using sg_virt(). This make highmem references work properly. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-4-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/moxart-mmc.c | 77 ++++++++++++++++------------------- 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c index 8ede4ce93271..b88d6dec209f 100644 --- a/drivers/mmc/host/moxart-mmc.c +++ b/drivers/mmc/host/moxart-mmc.c @@ -131,12 +131,10 @@ struct moxart_host { struct dma_async_tx_descriptor *tx_desc; struct mmc_host *mmc; struct mmc_request *mrq; - struct scatterlist *cur_sg; struct completion dma_complete; struct completion pio_complete; - u32 num_sg; - u32 data_remain; + struct sg_mapping_iter sg_miter; u32 data_len; u32 fifo_width; u32 timeout; @@ -148,35 +146,6 @@ struct moxart_host { bool is_removed; }; -static inline void moxart_init_sg(struct moxart_host *host, - struct mmc_data *data) -{ - host->cur_sg = data->sg; - host->num_sg = data->sg_len; - host->data_remain = host->cur_sg->length; - - if (host->data_remain > host->data_len) - host->data_remain = host->data_len; -} - -static inline int moxart_next_sg(struct moxart_host *host) -{ - int remain; - struct mmc_data *data = host->mrq->cmd->data; - - host->cur_sg++; - host->num_sg--; - - if (host->num_sg > 0) { - host->data_remain = host->cur_sg->length; - remain = host->data_len - data->bytes_xfered; - if (remain > 0 && remain < host->data_remain) - host->data_remain = remain; - } - - return host->num_sg; -} - static int moxart_wait_for_status(struct moxart_host *host, u32 mask, u32 *status) { @@ -309,14 +278,28 @@ static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host) static void moxart_transfer_pio(struct moxart_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct mmc_data *data = host->mrq->cmd->data; u32 *sgp, len = 0, remain, status; if (host->data_len == data->bytes_xfered) return; - sgp = sg_virt(host->cur_sg); - remain = host->data_remain; + /* + * By updating sgm->consumes this will get a proper pointer into the + * buffer at any time. + */ + if (!sg_miter_next(sgm)) { + /* This shold not happen */ + dev_err(mmc_dev(host->mmc), "ran out of scatterlist prematurely\n"); + data->error = -EINVAL; + complete(&host->pio_complete); + return; + } + sgp = sgm->addr; + remain = sgm->length; + if (remain > host->data_len) + remain = host->data_len; if (data->flags & MMC_DATA_WRITE) { while (remain > 0) { @@ -331,6 +314,7 @@ static void moxart_transfer_pio(struct moxart_host *host) sgp++; len += 4; } + sgm->consumed += len; remain -= len; } @@ -347,22 +331,22 @@ static void moxart_transfer_pio(struct moxart_host *host) sgp++; len += 4; } + sgm->consumed += len; remain -= len; } } - data->bytes_xfered += host->data_remain - remain; - host->data_remain = remain; - - if (host->data_len != data->bytes_xfered) - moxart_next_sg(host); - else + data->bytes_xfered += sgm->consumed; + if (host->data_len == data->bytes_xfered) { complete(&host->pio_complete); + return; + } } static void moxart_prepare_data(struct moxart_host *host) { struct mmc_data *data = host->mrq->cmd->data; + unsigned int flags = SG_MITER_ATOMIC; /* Used from IRQ */ u32 datactrl; int blksz_bits; @@ -373,15 +357,19 @@ static void moxart_prepare_data(struct moxart_host *host) blksz_bits = ffs(data->blksz) - 1; BUG_ON(1 << blksz_bits != data->blksz); - moxart_init_sg(host, data); - datactrl = DCR_DATA_EN | (blksz_bits & DCR_BLK_SIZE); - if (data->flags & MMC_DATA_WRITE) + if (data->flags & MMC_DATA_WRITE) { + flags |= SG_MITER_FROM_SG; datactrl |= DCR_DATA_WRITE; + } else { + flags |= SG_MITER_TO_SG; + } if (moxart_use_dma(host)) datactrl |= DCR_DMA_EN; + else + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); writel(DCR_DATA_FIFO_RESET, host->base + REG_DATA_CONTROL); writel(MASK_DATA | FIFO_URUN | FIFO_ORUN, host->base + REG_CLEAR); @@ -454,6 +442,9 @@ static void moxart_request(struct mmc_host *mmc, struct mmc_request *mrq) } request_done: + if (!moxart_use_dma(host)) + sg_miter_stop(&host->sg_miter); + spin_unlock_irqrestore(&host->lock, flags); mmc_request_done(host->mmc, mrq); } From 2761822c00e8c271f10a10affdbd4917d900d7ea Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:52 +0100 Subject: [PATCH 0303/1038] mmc: mvsdio: Use sg_miter for PIO Use the scatterlist memory iterator instead of just dereferencing virtual memory using sg_virt(). This make highmem references work properly. This driver also has a bug in the PIO sglist handling that is fixed as part of the patch: it does not travers the list of scatterbuffers: it will just process the first item in the list. This is fixed by augmenting the logic such that we do not process more than one sgitem per IRQ instead of counting down potentially the whole length of the request. We can suspect that the PIO path is quite untested. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-5-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/mvsdio.c | 71 +++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/drivers/mmc/host/mvsdio.c b/drivers/mmc/host/mvsdio.c index ca01b7d204ba..af7f21888e27 100644 --- a/drivers/mmc/host/mvsdio.c +++ b/drivers/mmc/host/mvsdio.c @@ -38,8 +38,9 @@ struct mvsd_host { unsigned int xfer_mode; unsigned int intr_en; unsigned int ctrl; + bool use_pio; + struct sg_mapping_iter sg_miter; unsigned int pio_size; - void *pio_ptr; unsigned int sg_frags; unsigned int ns_per_clk; unsigned int clock; @@ -114,11 +115,18 @@ static int mvsd_setup_data(struct mvsd_host *host, struct mmc_data *data) * data when the buffer is not aligned on a 64 byte * boundary. */ + unsigned int miter_flags = SG_MITER_ATOMIC; /* Used from IRQ */ + + if (data->flags & MMC_DATA_READ) + miter_flags |= SG_MITER_TO_SG; + else + miter_flags |= SG_MITER_FROM_SG; + host->pio_size = data->blocks * data->blksz; - host->pio_ptr = sg_virt(data->sg); + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, miter_flags); if (!nodma) - dev_dbg(host->dev, "fallback to PIO for data at 0x%p size %d\n", - host->pio_ptr, host->pio_size); + dev_dbg(host->dev, "fallback to PIO for data\n"); + host->use_pio = true; return 1; } else { dma_addr_t phys_addr; @@ -129,6 +137,7 @@ static int mvsd_setup_data(struct mvsd_host *host, struct mmc_data *data) phys_addr = sg_dma_address(data->sg); mvsd_write(MVSD_SYS_ADDR_LOW, (u32)phys_addr & 0xffff); mvsd_write(MVSD_SYS_ADDR_HI, (u32)phys_addr >> 16); + host->use_pio = false; return 0; } } @@ -288,8 +297,8 @@ static u32 mvsd_finish_data(struct mvsd_host *host, struct mmc_data *data, { void __iomem *iobase = host->base; - if (host->pio_ptr) { - host->pio_ptr = NULL; + if (host->use_pio) { + sg_miter_stop(&host->sg_miter); host->pio_size = 0; } else { dma_unmap_sg(mmc_dev(host->mmc), data->sg, host->sg_frags, @@ -344,9 +353,12 @@ static u32 mvsd_finish_data(struct mvsd_host *host, struct mmc_data *data, static irqreturn_t mvsd_irq(int irq, void *dev) { struct mvsd_host *host = dev; + struct sg_mapping_iter *sgm = &host->sg_miter; void __iomem *iobase = host->base; u32 intr_status, intr_done_mask; int irq_handled = 0; + u16 *p; + int s; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); dev_dbg(host->dev, "intr 0x%04x intr_en 0x%04x hw_state 0x%04x\n", @@ -370,15 +382,36 @@ static irqreturn_t mvsd_irq(int irq, void *dev) spin_lock(&host->lock); /* PIO handling, if needed. Messy business... */ - if (host->pio_size && + if (host->use_pio) { + /* + * As we set sgm->consumed this always gives a valid buffer + * position. + */ + if (!sg_miter_next(sgm)) { + /* This should not happen */ + dev_err(host->dev, "ran out of scatter segments\n"); + spin_unlock(&host->lock); + host->intr_en &= + ~(MVSD_NOR_RX_READY | MVSD_NOR_RX_FIFO_8W | + MVSD_NOR_TX_AVAIL | MVSD_NOR_TX_FIFO_8W); + mvsd_write(MVSD_NOR_INTR_EN, host->intr_en); + return IRQ_HANDLED; + } + p = sgm->addr; + s = sgm->length; + if (s > host->pio_size) + s = host->pio_size; + } + + if (host->use_pio && (intr_status & host->intr_en & (MVSD_NOR_RX_READY | MVSD_NOR_RX_FIFO_8W))) { - u16 *p = host->pio_ptr; - int s = host->pio_size; + while (s >= 32 && (intr_status & MVSD_NOR_RX_FIFO_8W)) { readsw(iobase + MVSD_FIFO, p, 16); p += 16; s -= 32; + sgm->consumed += 32; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } /* @@ -391,6 +424,7 @@ static irqreturn_t mvsd_irq(int irq, void *dev) put_unaligned(mvsd_read(MVSD_FIFO), p++); put_unaligned(mvsd_read(MVSD_FIFO), p++); s -= 4; + sgm->consumed += 4; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } if (s && s < 4 && (intr_status & MVSD_NOR_RX_READY)) { @@ -398,10 +432,13 @@ static irqreturn_t mvsd_irq(int irq, void *dev) val[0] = mvsd_read(MVSD_FIFO); val[1] = mvsd_read(MVSD_FIFO); memcpy(p, ((void *)&val) + 4 - s, s); + sgm->consumed += s; s = 0; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } - if (s == 0) { + /* PIO transfer done */ + host->pio_size -= sgm->consumed; + if (host->pio_size == 0) { host->intr_en &= ~(MVSD_NOR_RX_READY | MVSD_NOR_RX_FIFO_8W); mvsd_write(MVSD_NOR_INTR_EN, host->intr_en); @@ -413,14 +450,10 @@ static irqreturn_t mvsd_irq(int irq, void *dev) } dev_dbg(host->dev, "pio %d intr 0x%04x hw_state 0x%04x\n", s, intr_status, mvsd_read(MVSD_HW_STATE)); - host->pio_ptr = p; - host->pio_size = s; irq_handled = 1; - } else if (host->pio_size && + } else if (host->use_pio && (intr_status & host->intr_en & (MVSD_NOR_TX_AVAIL | MVSD_NOR_TX_FIFO_8W))) { - u16 *p = host->pio_ptr; - int s = host->pio_size; /* * The TX_FIFO_8W bit is unreliable. When set, bursting * 16 halfwords all at once in the FIFO drops data. Actually @@ -431,6 +464,7 @@ static irqreturn_t mvsd_irq(int irq, void *dev) mvsd_write(MVSD_FIFO, get_unaligned(p++)); mvsd_write(MVSD_FIFO, get_unaligned(p++)); s -= 4; + sgm->consumed += 4; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } if (s < 4) { @@ -439,10 +473,13 @@ static irqreturn_t mvsd_irq(int irq, void *dev) memcpy(((void *)&val) + 4 - s, p, s); mvsd_write(MVSD_FIFO, val[0]); mvsd_write(MVSD_FIFO, val[1]); + sgm->consumed += s; s = 0; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } - if (s == 0) { + /* PIO transfer done */ + host->pio_size -= sgm->consumed; + if (host->pio_size == 0) { host->intr_en &= ~(MVSD_NOR_TX_AVAIL | MVSD_NOR_TX_FIFO_8W); mvsd_write(MVSD_NOR_INTR_EN, host->intr_en); @@ -450,8 +487,6 @@ static irqreturn_t mvsd_irq(int irq, void *dev) } dev_dbg(host->dev, "pio %d intr 0x%04x hw_state 0x%04x\n", s, intr_status, mvsd_read(MVSD_HW_STATE)); - host->pio_ptr = p; - host->pio_size = s; irq_handled = 1; } From 12658af54f2ed10d812dcd0a769d62cb9e9291e5 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:53 +0100 Subject: [PATCH 0304/1038] mmc: mxcmmc: Use sg_miter for PIO Use the scatterlist memory iterator instead of just dereferencing virtual memory using sg_virt(). This make highmem references work properly. Since this driver is using a worker, no atomic trickery is needed. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-6-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/mxcmmc.c | 53 ++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/drivers/mmc/host/mxcmmc.c b/drivers/mmc/host/mxcmmc.c index 5b3ab0e20505..1edf65291354 100644 --- a/drivers/mmc/host/mxcmmc.c +++ b/drivers/mmc/host/mxcmmc.c @@ -266,11 +266,18 @@ static inline void buffer_swap32(u32 *buf, int len) static void mxcmci_swap_buffers(struct mmc_data *data) { - struct scatterlist *sg; - int i; + struct sg_mapping_iter sgm; + u32 *buf; - for_each_sg(data->sg, sg, data->sg_len, i) - buffer_swap32(sg_virt(sg), sg->length); + sg_miter_start(&sgm, data->sg, data->sg_len, + SG_MITER_TO_SG | SG_MITER_FROM_SG); + + while (sg_miter_next(&sgm)) { + buf = sgm.addr; + buffer_swap32(buf, sgm.length); + } + + sg_miter_stop(&sgm); } #else static inline void mxcmci_swap_buffers(struct mmc_data *data) {} @@ -526,10 +533,9 @@ static int mxcmci_poll_status(struct mxcmci_host *host, u32 mask) } while (1); } -static int mxcmci_pull(struct mxcmci_host *host, void *_buf, int bytes) +static int mxcmci_pull(struct mxcmci_host *host, u32 *buf, int bytes) { unsigned int stat; - u32 *buf = _buf; while (bytes > 3) { stat = mxcmci_poll_status(host, @@ -555,10 +561,9 @@ static int mxcmci_pull(struct mxcmci_host *host, void *_buf, int bytes) return 0; } -static int mxcmci_push(struct mxcmci_host *host, void *_buf, int bytes) +static int mxcmci_push(struct mxcmci_host *host, u32 *buf, int bytes) { unsigned int stat; - u32 *buf = _buf; while (bytes > 3) { stat = mxcmci_poll_status(host, STATUS_BUF_WRITE_RDY); @@ -586,31 +591,39 @@ static int mxcmci_push(struct mxcmci_host *host, void *_buf, int bytes) static int mxcmci_transfer_data(struct mxcmci_host *host) { struct mmc_data *data = host->req->data; - struct scatterlist *sg; - int stat, i; + struct sg_mapping_iter sgm; + int stat; + u32 *buf; host->data = data; host->datasize = 0; + sg_miter_start(&sgm, data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? SG_MITER_TO_SG : SG_MITER_FROM_SG); if (data->flags & MMC_DATA_READ) { - for_each_sg(data->sg, sg, data->sg_len, i) { - stat = mxcmci_pull(host, sg_virt(sg), sg->length); + while (sg_miter_next(&sgm)) { + buf = sgm.addr; + stat = mxcmci_pull(host, buf, sgm.length); if (stat) - return stat; - host->datasize += sg->length; + goto transfer_error; + host->datasize += sgm.length; } } else { - for_each_sg(data->sg, sg, data->sg_len, i) { - stat = mxcmci_push(host, sg_virt(sg), sg->length); + while (sg_miter_next(&sgm)) { + buf = sgm.addr; + stat = mxcmci_push(host, buf, sgm.length); if (stat) - return stat; - host->datasize += sg->length; + goto transfer_error; + host->datasize += sgm.length; } stat = mxcmci_poll_status(host, STATUS_WRITE_OP_DONE); if (stat) - return stat; + goto transfer_error; } - return 0; + +transfer_error: + sg_miter_stop(&sgm); + return stat; } static void mxcmci_datawork(struct work_struct *work) From 68cfdeb4064e049647395271eec3c0d987141674 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:54 +0100 Subject: [PATCH 0305/1038] mmc: omap: Use sg_miter for PIO Use the scatterlist memory iterator instead of just dereferencing virtual memory using sg_virt(). This make highmem references work properly. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-7-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/omap.c | 53 +++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/drivers/mmc/host/omap.c b/drivers/mmc/host/omap.c index 9fb8995b43a1..088f8ed4fdc4 100644 --- a/drivers/mmc/host/omap.c +++ b/drivers/mmc/host/omap.c @@ -148,10 +148,8 @@ struct mmc_omap_host { struct work_struct send_stop_work; struct mmc_data *stop_data; + struct sg_mapping_iter sg_miter; unsigned int sg_len; - int sg_idx; - u16 * buffer; - u32 buffer_bytes_left; u32 total_bytes_left; unsigned features; @@ -456,6 +454,8 @@ mmc_omap_xfer_done(struct mmc_omap_host *host, struct mmc_data *data) { if (host->dma_in_use) mmc_omap_release_dma(host, data, data->error); + else + sg_miter_stop(&host->sg_miter); host->data = NULL; host->sg_len = 0; @@ -651,19 +651,6 @@ mmc_omap_cmd_timer(struct timer_list *t) spin_unlock_irqrestore(&host->slot_lock, flags); } -/* PIO only */ -static void -mmc_omap_sg_to_buf(struct mmc_omap_host *host) -{ - struct scatterlist *sg; - - sg = host->data->sg + host->sg_idx; - host->buffer_bytes_left = sg->length; - host->buffer = sg_virt(sg); - if (host->buffer_bytes_left > host->total_bytes_left) - host->buffer_bytes_left = host->total_bytes_left; -} - static void mmc_omap_clk_timer(struct timer_list *t) { @@ -676,33 +663,37 @@ mmc_omap_clk_timer(struct timer_list *t) static void mmc_omap_xfer_data(struct mmc_omap_host *host, int write) { + struct sg_mapping_iter *sgm = &host->sg_miter; int n, nwords; + u16 *buffer; - if (host->buffer_bytes_left == 0) { - host->sg_idx++; - BUG_ON(host->sg_idx == host->sg_len); - mmc_omap_sg_to_buf(host); + if (!sg_miter_next(sgm)) { + /* This should not happen */ + dev_err(mmc_dev(host->mmc), "ran out of scatterlist prematurely\n"); + return; } + buffer = sgm->addr; + n = 64; - if (n > host->buffer_bytes_left) - n = host->buffer_bytes_left; + if (n > sgm->length) + n = sgm->length; + if (n > host->total_bytes_left) + n = host->total_bytes_left; /* Round up to handle odd number of bytes to transfer */ nwords = DIV_ROUND_UP(n, 2); - host->buffer_bytes_left -= n; + sgm->consumed = n; host->total_bytes_left -= n; host->data->bytes_xfered += n; if (write) { __raw_writesw(host->virt_base + OMAP_MMC_REG(host, DATA), - host->buffer, nwords); + buffer, nwords); } else { __raw_readsw(host->virt_base + OMAP_MMC_REG(host, DATA), - host->buffer, nwords); + buffer, nwords); } - - host->buffer += nwords; } #ifdef CONFIG_MMC_DEBUG @@ -956,6 +947,7 @@ static inline void set_data_timeout(struct mmc_omap_host *host, struct mmc_reque static void mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req) { + unsigned int miter_flags = SG_MITER_ATOMIC; /* Used from IRQ */ struct mmc_data *data = req->data; int i, use_dma = 1, block_size; struct scatterlist *sg; @@ -990,7 +982,6 @@ mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req) } } - host->sg_idx = 0; if (use_dma) { enum dma_data_direction dma_data_dir; struct dma_async_tx_descriptor *tx; @@ -1071,7 +1062,11 @@ mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req) OMAP_MMC_WRITE(host, BUF, 0x1f1f); host->total_bytes_left = data->blocks * block_size; host->sg_len = sg_len; - mmc_omap_sg_to_buf(host); + if (data->flags & MMC_DATA_READ) + miter_flags |= SG_MITER_TO_SG; + else + miter_flags |= SG_MITER_FROM_SG; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, miter_flags); host->dma_in_use = 0; } From e8a167b84886ccca21ca4d0d0396f7e172876c92 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:55 +0100 Subject: [PATCH 0306/1038] mmc: sdhci-esdhc-mcf: Use sg_miter for swapping Use sg_miter iterator instead of sg_virt() and custom code to loop over the scatterlist. The memory iterator will do bounce buffering if the page happens to be located in high memory, which the driver may or may not be using. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-8-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-esdhc-mcf.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/mmc/host/sdhci-esdhc-mcf.c b/drivers/mmc/host/sdhci-esdhc-mcf.c index a07f8333cd6b..1909a11fd065 100644 --- a/drivers/mmc/host/sdhci-esdhc-mcf.c +++ b/drivers/mmc/host/sdhci-esdhc-mcf.c @@ -299,9 +299,8 @@ static void esdhc_mcf_pltfm_set_bus_width(struct sdhci_host *host, int width) static void esdhc_mcf_request_done(struct sdhci_host *host, struct mmc_request *mrq) { - struct scatterlist *sg; + struct sg_mapping_iter sgm; u32 *buffer; - int i; if (!mrq->data || !mrq->data->bytes_xfered) goto exit_done; @@ -313,10 +312,13 @@ static void esdhc_mcf_request_done(struct sdhci_host *host, * On mcf5441x there is no hw sdma option/flag to select the dma * transfer endiannes. A swap after the transfer is needed. */ - for_each_sg(mrq->data->sg, sg, mrq->data->sg_len, i) { - buffer = (u32 *)sg_virt(sg); - esdhc_mcf_buffer_swap32(buffer, sg->length); + sg_miter_start(&sgm, mrq->data->sg, mrq->data->sg_len, + SG_MITER_TO_SG | SG_MITER_FROM_SG); + while (sg_miter_next(&sgm)) { + buffer = sgm.addr; + esdhc_mcf_buffer_swap32(buffer, sgm.length); } + sg_miter_stop(&sgm); exit_done: mmc_request_done(host->mmc, mrq); From 27b57277d9ba3a900daffdb91856ba8a5f46b3a7 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sat, 27 Jan 2024 01:19:56 +0100 Subject: [PATCH 0307/1038] mmc: sh_mmcif: Use sg_miter for PIO Use sg_miter iterator instead of sg_virt() and custom code to loop over the scatterlist. The memory iterator will do bounce buffering if the page happens to be located in high memory, which the driver may or may not be using. Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/linux-mmc/20240122073423.GA25859@lst.de/ Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240127-mmc-proper-kmap-v2-9-d8e732aa97d1@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/sh_mmcif.c | 102 ++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c index 077d711e964e..1ef6e153e5a3 100644 --- a/drivers/mmc/host/sh_mmcif.c +++ b/drivers/mmc/host/sh_mmcif.c @@ -227,14 +227,12 @@ struct sh_mmcif_host { bool dying; long timeout; void __iomem *addr; - u32 *pio_ptr; spinlock_t lock; /* protect sh_mmcif_host::state */ enum sh_mmcif_state state; enum sh_mmcif_wait_for wait_for; struct delayed_work timeout_work; size_t blocksize; - int sg_idx; - int sg_blkidx; + struct sg_mapping_iter sg_miter; bool power; bool ccs_enable; /* Command Completion Signal support */ bool clk_ctrl2_enable; @@ -600,32 +598,17 @@ static int sh_mmcif_error_manage(struct sh_mmcif_host *host) return ret; } -static bool sh_mmcif_next_block(struct sh_mmcif_host *host, u32 *p) -{ - struct mmc_data *data = host->mrq->data; - - host->sg_blkidx += host->blocksize; - - /* data->sg->length must be a multiple of host->blocksize? */ - BUG_ON(host->sg_blkidx > data->sg->length); - - if (host->sg_blkidx == data->sg->length) { - host->sg_blkidx = 0; - if (++host->sg_idx < data->sg_len) - host->pio_ptr = sg_virt(++data->sg); - } else { - host->pio_ptr = p; - } - - return host->sg_idx != data->sg_len; -} - static void sh_mmcif_single_read(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct mmc_data *data = mrq->data; + host->blocksize = (sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK) + 3; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + SG_MITER_ATOMIC | SG_MITER_TO_SG); + host->wait_for = MMCIF_WAIT_FOR_READ; /* buf read enable */ @@ -634,20 +617,32 @@ static void sh_mmcif_single_read(struct sh_mmcif_host *host, static bool sh_mmcif_read_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = sg_virt(data->sg); + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } + if (!sg_miter_next(sgm)) { + /* This should not happen on single blocks */ + sg_miter_stop(sgm); + return false; + } + + p = sgm->addr; + for (i = 0; i < host->blocksize / 4; i++) *p++ = sh_mmcif_readl(host->addr, MMCIF_CE_DATA); + sg_miter_stop(&host->sg_miter); + /* buffer read end */ sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFRE); host->wait_for = MMCIF_WAIT_FOR_READ_END; @@ -666,34 +661,40 @@ static void sh_mmcif_multi_read(struct sh_mmcif_host *host, host->blocksize = sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + SG_MITER_ATOMIC | SG_MITER_TO_SG); + host->wait_for = MMCIF_WAIT_FOR_MREAD; - host->sg_idx = 0; - host->sg_blkidx = 0; - host->pio_ptr = sg_virt(data->sg); sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFREN); } static bool sh_mmcif_mread_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = host->pio_ptr; + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } - BUG_ON(!data->sg->length); + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return false; + } + + p = sgm->addr; for (i = 0; i < host->blocksize / 4; i++) *p++ = sh_mmcif_readl(host->addr, MMCIF_CE_DATA); - if (!sh_mmcif_next_block(host, p)) - return false; + sgm->consumed = host->blocksize; sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFREN); @@ -703,9 +704,14 @@ static bool sh_mmcif_mread_block(struct sh_mmcif_host *host) static void sh_mmcif_single_write(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct mmc_data *data = mrq->data; + host->blocksize = (sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK) + 3; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + SG_MITER_ATOMIC | SG_MITER_FROM_SG); + host->wait_for = MMCIF_WAIT_FOR_WRITE; /* buf write enable */ @@ -714,20 +720,32 @@ static void sh_mmcif_single_write(struct sh_mmcif_host *host, static bool sh_mmcif_write_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = sg_virt(data->sg); + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } + if (!sg_miter_next(sgm)) { + /* This should not happen on single blocks */ + sg_miter_stop(sgm); + return false; + } + + p = sgm->addr; + for (i = 0; i < host->blocksize / 4; i++) sh_mmcif_writel(host->addr, MMCIF_CE_DATA, *p++); + sg_miter_stop(&host->sg_miter); + /* buffer write end */ sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MDTRANE); host->wait_for = MMCIF_WAIT_FOR_WRITE_END; @@ -746,34 +764,40 @@ static void sh_mmcif_multi_write(struct sh_mmcif_host *host, host->blocksize = sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + SG_MITER_ATOMIC | SG_MITER_FROM_SG); + host->wait_for = MMCIF_WAIT_FOR_MWRITE; - host->sg_idx = 0; - host->sg_blkidx = 0; - host->pio_ptr = sg_virt(data->sg); sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFWEN); } static bool sh_mmcif_mwrite_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = host->pio_ptr; + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } - BUG_ON(!data->sg->length); + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return false; + } + + p = sgm->addr; for (i = 0; i < host->blocksize / 4; i++) sh_mmcif_writel(host->addr, MMCIF_CE_DATA, *p++); - if (!sh_mmcif_next_block(host, p)) - return false; + sgm->consumed = host->blocksize; sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFWEN); From 4615ac9010be84d676f9e893e5a7ea4b5febd1e8 Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Tue, 13 Feb 2024 12:16:00 +0100 Subject: [PATCH 0308/1038] cpufreq: intel_pstate: remove cpudata::prev_cummulative_iowait Commit 09c448d3c61f ("cpufreq: intel_pstate: Use IOWAIT flag in Atom algorithm") removed the last user of cpudata::prev_cummulative_iowait. Remove the member too. Found by https://github.com/jirislaby/clang-struct. Signed-off-by: Jiri Slaby (SUSE) Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ca94e60e705a..5ad3542c0e1e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -201,8 +201,6 @@ struct global_params { * @prev_aperf: Last APERF value read from APERF MSR * @prev_mperf: Last MPERF value read from MPERF MSR * @prev_tsc: Last timestamp counter (TSC) value - * @prev_cummulative_iowait: IO Wait time difference from last and - * current sample * @sample: Storage for storing last Sample data * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios @@ -241,7 +239,6 @@ struct cpudata { u64 prev_aperf; u64 prev_mperf; u64 prev_tsc; - u64 prev_cummulative_iowait; struct sample sample; int32_t min_perf_ratio; int32_t max_perf_ratio; From 59fbb55a297d0cf6e10c0a27fb172aada9902bc2 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 5 Feb 2024 19:13:10 +0000 Subject: [PATCH 0309/1038] mmc: wbsd: remove redundant assignment to variable id The variable id is being initialized with a value that is never read, it is being re-assigned later on. The initialization is redundant and can be removed. Cleans up clang scan build warning: drivers/mmc/host/wbsd.c:1287:4: warning: Value stored to 'id' is never read [deadcode.DeadStores] Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20240205191310.1848561-1-colin.i.king@gmail.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/wbsd.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/mmc/host/wbsd.c b/drivers/mmc/host/wbsd.c index 001a468bc149..f0562f712d98 100644 --- a/drivers/mmc/host/wbsd.c +++ b/drivers/mmc/host/wbsd.c @@ -1284,8 +1284,6 @@ static int wbsd_scan(struct wbsd_host *host) continue; for (j = 0; j < ARRAY_SIZE(unlock_codes); j++) { - id = 0xFFFF; - host->config = config_ports[i]; host->unlock_code = unlock_codes[j]; From 2d1f5006ff95770da502f8cee2a224a1ff83866e Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 31 Jan 2024 19:37:08 +0800 Subject: [PATCH 0310/1038] powercap: intel_rapl: Fix a NULL pointer dereference A NULL pointer dereference is triggered when probing the MMIO RAPL driver on platforms with CPU ID not listed in intel_rapl_common CPU model list. This is because the intel_rapl_common module still probes on such platforms even if 'defaults_msr' is not set after commit 1488ac990ac8 ("powercap: intel_rapl: Allow probing without CPUID match"). Thus the MMIO RAPL rp->priv->defaults is NULL when registering to RAPL framework. Fix the problem by adding sanity check to ensure rp->priv->rapl_defaults is always valid. Fixes: 1488ac990ac8 ("powercap: intel_rapl: Allow probing without CPUID match") Signed-off-by: Zhang Rui Cc: 6.5+ # 6.5+ Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 2feed036c1cd..1a739afd47d9 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -759,6 +759,11 @@ static int rapl_config(struct rapl_package *rp) default: return -EINVAL; } + + /* defaults_msr can be NULL on unsupported platforms */ + if (!rp->priv->defaults || !rp->priv->rpi) + return -ENODEV; + return 0; } From 1aa09b9379a7a644cd2f75ae0bac82b8783df600 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 31 Jan 2024 19:37:09 +0800 Subject: [PATCH 0311/1038] powercap: intel_rapl: Fix locking in TPMI RAPL The RAPL framework uses CPU hotplug locking to protect the rapl_packages list and rp->lead_cpu to guarantee that 1. the RAPL package device is not unprobed and freed 2. the cached rp->lead_cpu is always valid for operations like powercap sysfs accesses. Current RAPL APIs assume being called from CPU hotplug callbacks which hold the CPU hotplug lock, but TPMI RAPL driver invokes the APIs in the driver's .probe() function without acquiring the CPU hotplug lock. Fix the problem by providing both locked and lockless versions of RAPL APIs. Fixes: 9eef7f9da928 ("powercap: intel_rapl: Introduce RAPL TPMI interface driver") Signed-off-by: Zhang Rui Cc: 6.5+ # 6.5+ Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 29 +++++++++++++++++-- drivers/powercap/intel_rapl_msr.c | 8 ++--- .../int340x_thermal/processor_thermal_rapl.c | 8 ++--- include/linux/intel_rapl.h | 6 ++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 1a739afd47d9..9d3e102f1a76 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -5,6 +5,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -1504,7 +1505,7 @@ static int rapl_detect_domains(struct rapl_package *rp) } /* called from CPU hotplug notifier, hotplug lock held */ -void rapl_remove_package(struct rapl_package *rp) +void rapl_remove_package_cpuslocked(struct rapl_package *rp) { struct rapl_domain *rd, *rd_package = NULL; @@ -1533,10 +1534,18 @@ void rapl_remove_package(struct rapl_package *rp) list_del(&rp->plist); kfree(rp); } +EXPORT_SYMBOL_GPL(rapl_remove_package_cpuslocked); + +void rapl_remove_package(struct rapl_package *rp) +{ + guard(cpus_read_lock)(); + rapl_remove_package_cpuslocked(rp); +} EXPORT_SYMBOL_GPL(rapl_remove_package); /* caller to ensure CPU hotplug lock is held */ -struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) +struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu) { struct rapl_package *rp; int uid; @@ -1554,10 +1563,17 @@ struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, return NULL; } +EXPORT_SYMBOL_GPL(rapl_find_package_domain_cpuslocked); + +struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) +{ + guard(cpus_read_lock)(); + return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu); +} EXPORT_SYMBOL_GPL(rapl_find_package_domain); /* called from CPU hotplug notifier, hotplug lock held */ -struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) +struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu) { struct rapl_package *rp; int ret; @@ -1603,6 +1619,13 @@ err_free_package: kfree(rp); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(rapl_add_package_cpuslocked); + +struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) +{ + guard(cpus_read_lock)(); + return rapl_add_package_cpuslocked(id, priv, id_is_cpu); +} EXPORT_SYMBOL_GPL(rapl_add_package); static void power_limit_state_save(void) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 250bd41a588c..b4b6930cacb0 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -73,9 +73,9 @@ static int rapl_cpu_online(unsigned int cpu) { struct rapl_package *rp; - rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, rapl_msr_priv, true); if (!rp) { - rp = rapl_add_package(cpu, rapl_msr_priv, true); + rp = rapl_add_package_cpuslocked(cpu, rapl_msr_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -88,14 +88,14 @@ static int rapl_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, rapl_msr_priv, true); if (!rp) return 0; cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); + rapl_remove_package_cpuslocked(rp); else if (rp->lead_cpu == cpu) rp->lead_cpu = lead_cpu; return 0; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index 2f00fc3bf274..e964a9375722 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -27,9 +27,9 @@ static int rapl_mmio_cpu_online(unsigned int cpu) if (topology_physical_package_id(cpu)) return 0; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, &rapl_mmio_priv, true); if (!rp) { - rp = rapl_add_package(cpu, &rapl_mmio_priv, true); + rp = rapl_add_package_cpuslocked(cpu, &rapl_mmio_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -42,14 +42,14 @@ static int rapl_mmio_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, &rapl_mmio_priv, true); if (!rp) return 0; cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); + rapl_remove_package_cpuslocked(rp); else if (rp->lead_cpu == cpu) rp->lead_cpu = lead_cpu; return 0; diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 33f21bd85dbf..f3196f82fd8a 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -178,6 +178,12 @@ struct rapl_package { struct rapl_if_priv *priv; }; +struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu); +struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu); +void rapl_remove_package_cpuslocked(struct rapl_package *rp); + struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu); struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu); void rapl_remove_package(struct rapl_package *rp); From faa9130ce716b286d786d59032bacfd9052c2094 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 31 Jan 2024 19:37:10 +0800 Subject: [PATCH 0312/1038] powercap: intel_rapl_tpmi: Fix a register bug Add the missing Domain Info register. This also fixes the bogus definition of the Interrupt register. Neither of these two registers was used previously. Fixes: 9eef7f9da928 ("powercap: intel_rapl: Introduce RAPL TPMI interface driver") Signed-off-by: Zhang Rui Cc: 6.5+ # 6.5+ Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_tpmi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 891c90fefd8b..f1c734ac3c34 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -40,6 +40,7 @@ enum tpmi_rapl_register { TPMI_RAPL_REG_ENERGY_STATUS, TPMI_RAPL_REG_PERF_STATUS, TPMI_RAPL_REG_POWER_INFO, + TPMI_RAPL_REG_DOMAIN_INFO, TPMI_RAPL_REG_INTERRUPT, TPMI_RAPL_REG_MAX = 15, }; From 903eb9fb85e32810f376a2858aad77c9298f9488 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 31 Jan 2024 19:37:11 +0800 Subject: [PATCH 0313/1038] powercap: intel_rapl_tpmi: Fix System Domain probing Only domain root packages can enumerate System (Psys) domain. Whether a package is domain root or not is described in the Bit 0 of the Domain Info register. Add support for Domain Info register and fix the System domain probing accordingly. Fixes: 9eef7f9da928 ("powercap: intel_rapl: Introduce RAPL TPMI interface driver") Signed-off-by: Zhang Rui Cc: 6.5+ # 6.5+ Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_tpmi.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index f1c734ac3c34..f6b7f085977c 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -131,6 +131,12 @@ static void trp_release(struct tpmi_rapl_package *trp) mutex_unlock(&tpmi_rapl_lock); } +/* + * Bit 0 of TPMI_RAPL_REG_DOMAIN_INFO indicates if the current package is a domain + * root or not. Only domain root packages can enumerate System (Psys) Domain. + */ +#define TPMI_RAPL_DOMAIN_ROOT BIT(0) + static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) { u8 tpmi_domain_version; @@ -140,6 +146,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) enum rapl_domain_reg_id reg_id; int tpmi_domain_size, tpmi_domain_flags; u64 tpmi_domain_header = readq(trp->base + offset); + u64 tpmi_domain_info; /* Domain Parent bits are ignored for now */ tpmi_domain_version = tpmi_domain_header & 0xff; @@ -170,6 +177,13 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) domain_type = RAPL_DOMAIN_PACKAGE; break; case TPMI_RAPL_DOMAIN_SYSTEM: + if (!(tpmi_domain_flags & BIT(TPMI_RAPL_REG_DOMAIN_INFO))) { + pr_warn(FW_BUG "System domain must support Domain Info register\n"); + return -ENODEV; + } + tpmi_domain_info = readq(trp->base + offset + TPMI_RAPL_REG_DOMAIN_INFO); + if (!(tpmi_domain_info & TPMI_RAPL_DOMAIN_ROOT)) + return 0; domain_type = RAPL_DOMAIN_PLATFORM; break; case TPMI_RAPL_DOMAIN_MEMORY: From 876ed77fbed4450a0006ce9a953433a599848c9c Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 31 Jan 2024 19:37:12 +0800 Subject: [PATCH 0314/1038] powercap: intel_rapl: Add support for Lunar Lake-M paltform Add support for Lunar Lake-M platform to the RAPL common driver. Signed-off-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 9d3e102f1a76..a3094cb9f296 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1262,6 +1262,7 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &rapl_defaults_spr_server), + X86_MATCH_INTEL_FAM6_MODEL(LUNARLAKE_M, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &rapl_defaults_byt), From 4add6e841a3e079c8cb18fab6eaf7385c28c25c7 Mon Sep 17 00:00:00 2001 From: Sumeet Pawnikar Date: Wed, 31 Jan 2024 19:37:13 +0800 Subject: [PATCH 0315/1038] powercap: intel_rapl: Add support for Arrow Lake Add support for Arrow Lake platform to the RAPL common driver. Signed-off-by: Sumeet Pawnikar Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index a3094cb9f296..aa627a6b12a4 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1263,6 +1263,7 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(LUNARLAKE_M, &rapl_defaults_core), + X86_MATCH_INTEL_FAM6_MODEL(ARROWLAKE, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &rapl_defaults_byt), From d6e5288c8bf4f9bfbb9d3630d8605f45f15a525f Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:21:37 +0100 Subject: [PATCH 0316/1038] mmc: renesas_sdhi: use typedef for dma_filter_fn Use existing typedef for dma_filter_fn to avoid duplicating type definition. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Wolfram Sang Link: https://lore.kernel.org/r/20240208202137.630281-1-krzysztof.kozlowski@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/renesas_sdhi.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/renesas_sdhi.h b/drivers/mmc/host/renesas_sdhi.h index c1fb9740eab0..586f94d4dbfd 100644 --- a/drivers/mmc/host/renesas_sdhi.h +++ b/drivers/mmc/host/renesas_sdhi.h @@ -9,6 +9,7 @@ #ifndef RENESAS_SDHI_H #define RENESAS_SDHI_H +#include #include #include "tmio_mmc.h" @@ -63,7 +64,7 @@ struct renesas_sdhi_of_data_with_quirks { struct renesas_sdhi_dma { unsigned long end_flags; enum dma_slave_buswidth dma_buswidth; - bool (*filter)(struct dma_chan *chan, void *arg); + dma_filter_fn filter; void (*enable)(struct tmio_mmc_host *host, bool enable); struct completion dma_dataend; struct tasklet_struct dma_complete; From 02c801376bdd3dc9bcfae46ea9db382611f7612a Mon Sep 17 00:00:00 2001 From: Jeffrey Hugo Date: Fri, 9 Feb 2024 09:09:34 -0700 Subject: [PATCH 0317/1038] MAINTAINERS: Update bouncing @codeaurora addresses for EMMC CMDQ The @codeaurora email domain's servers have been decommissioned for a long while now, and any emails addressed there will bounce. Asutosh has an entry in .mailmap pointing to a new address, but MAINTAINERS still lists an old @codeaurora address. Update MAINTAINERS to match .mailmap for anyone reading the file directly. Ritesh appears to have changed jobs, but looks to be still active in the community. Update Ritesh's address to the one used in recient community postings. Also Ritesh has indicated their entry should be changed from Maintainer (M:) to Reviewer (R:) so make that update while we are making changes to the entry. Signed-off-by: Jeffrey Hugo Acked-by: Ritesh Harjani Link: https://lore.kernel.org/r/20240209160934.3866475-1-quic_jhugo@quicinc.com Signed-off-by: Ulf Hansson --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 73d898383e51..137582fa1012 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7785,8 +7785,8 @@ F: drivers/media/usb/em28xx/ EMMC CMDQ HOST CONTROLLER INTERFACE (CQHCI) DRIVER M: Adrian Hunter -M: Ritesh Harjani -M: Asutosh Das +M: Asutosh Das +R: Ritesh Harjani L: linux-mmc@vger.kernel.org S: Supported F: drivers/mmc/host/cqhci* From df20385302eb01cb610b9edc71611a63d1683923 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 13 Feb 2024 21:07:45 +0300 Subject: [PATCH 0318/1038] spi: mchp-pci1xxxx: release resources on error in probe() Call pci_release_regions(pdev) before returning on this error path. Fixes: 3e7cfd6ad29a ("spi: mchp-pci1xxxx: Add support for DMA in SPI") Signed-off-by: Dan Carpenter Link: https://msgid.link/r/efc92197-4023-4bfe-bc63-452e7ed112e8@moroto.mountain Signed-off-by: Mark Brown --- drivers/spi/spi-pci1xxxx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-pci1xxxx.c b/drivers/spi/spi-pci1xxxx.c index a99db6163aec..969965d7bc98 100644 --- a/drivers/spi/spi-pci1xxxx.c +++ b/drivers/spi/spi-pci1xxxx.c @@ -776,7 +776,7 @@ static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id * ret = pci1xxxx_spi_dma_init(spi_bus, spi_sub_ptr->irq); if (ret && ret != -EOPNOTSUPP) - return ret; + goto error; /* This register is only applicable for 1st instance */ regval = readl(spi_bus->reg_base + SPI_PCI_CTRL_REG_OFFSET(0)); From ba5c5effe02c4ae06291c7c5e67de8fedad3cf9a Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 12 Feb 2024 22:39:20 +0100 Subject: [PATCH 0319/1038] gpio: initialize descriptor SRCU structure before adding OF-based chips In certain situations we may end up taking the GPIO descriptor SRCU read lock in of_gpiochip_add() before the SRCU struct is initialized. Move the initialization before the call to of_gpiochip_add(). Fixes: be711caa87c5 ("gpio: add SRCU infrastructure to struct gpio_desc") Fixes: 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202402122228.e607a080-lkp@intel.com Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 82811d9a4477..f5434e559382 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -991,10 +991,6 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_cleanup_gdev_srcu; - ret = of_gpiochip_add(gc); - if (ret) - goto err_free_gpiochip_mask; - for (i = 0; i < gc->ngpio; i++) { struct gpio_desc *desc = &gdev->descs[i]; @@ -1002,7 +998,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) { for (j = 0; j < i; j++) cleanup_srcu_struct(&gdev->descs[j].srcu); - goto err_remove_of_chip; + goto err_free_gpiochip_mask; } if (gc->get_direction && gpiochip_line_is_valid(gc, i)) { @@ -1014,10 +1010,14 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, } } - ret = gpiochip_add_pin_ranges(gc); + ret = of_gpiochip_add(gc); if (ret) goto err_cleanup_desc_srcu; + ret = gpiochip_add_pin_ranges(gc); + if (ret) + goto err_remove_of_chip; + acpi_gpiochip_add(gc); machine_gpiochip_add(gc); @@ -1055,12 +1055,12 @@ err_remove_irqchip_mask: gpiochip_irqchip_free_valid_mask(gc); err_remove_acpi_chip: acpi_gpiochip_remove(gc); -err_cleanup_desc_srcu: - for (i = 0; i < gdev->ngpio; i++) - cleanup_srcu_struct(&gdev->descs[i].srcu); err_remove_of_chip: gpiochip_free_hogs(gc); of_gpiochip_remove(gc); +err_cleanup_desc_srcu: + for (i = 0; i < gdev->ngpio; i++) + cleanup_srcu_struct(&gdev->descs[i].srcu); err_free_gpiochip_mask: gpiochip_remove_pin_ranges(gc); gpiochip_free_valid_mask(gc); From 2b391c4ca735d0633bcdca5f83939c9791405225 Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Wed, 3 Jan 2024 22:10:05 +0100 Subject: [PATCH 0320/1038] pmdomain: core: Scale down parent/child performance states in reverse order Power domains might have parent domains assigned that are automatically managed by the PM domain core. In particular, parent domains are automatically powered on/off and setting performance states on child domains are propagated to parent domains (e.g. using an OPP table from the device tree). Currently the parent performance state is always adjusted before the performance state of the child domain, which is a problem for some cases when scaling down the performance state. More exactly, it may lead to that the parent domain could run in a lower performance state, than what is required by the child domain. To fix the behaviour, let's differentiate between scaling up/down and adjust the order of operations: - When scaling up, parent domains should be adjusted before the child domain. In case of an error, the rollback happens in reverse order. - When scaling down, parent domains should be adjusted after the child domain, in reverse order, just as if we would rollback scaling up. In case of an error, the rollback happens in normal order (just as if we would normally scale up). Signed-off-by: Stephan Gerhold Link: https://lore.kernel.org/r/20240103-genpd-perf-order-v2-1-eeecfc55624b@gerhold.net Tested-by: Ulf Hansson Signed-off-by: Ulf Hansson --- drivers/pmdomain/core.c | 132 ++++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 51 deletions(-) diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 18e232b5ed53..8c28d5e50500 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -310,73 +310,103 @@ static int genpd_xlate_performance_state(struct generic_pm_domain *genpd, pstate); } +static int _genpd_set_performance_state(struct generic_pm_domain *genpd, + unsigned int state, int depth); + +static void _genpd_rollback_parent_state(struct gpd_link *link, int depth) +{ + struct generic_pm_domain *parent = link->parent; + int parent_state; + + genpd_lock_nested(parent, depth + 1); + + parent_state = link->prev_performance_state; + link->performance_state = parent_state; + + parent_state = _genpd_reeval_performance_state(parent, parent_state); + if (_genpd_set_performance_state(parent, parent_state, depth + 1)) { + pr_err("%s: Failed to roll back to %d performance state\n", + parent->name, parent_state); + } + + genpd_unlock(parent); +} + +static int _genpd_set_parent_state(struct generic_pm_domain *genpd, + struct gpd_link *link, + unsigned int state, int depth) +{ + struct generic_pm_domain *parent = link->parent; + int parent_state, ret; + + /* Find parent's performance state */ + ret = genpd_xlate_performance_state(genpd, parent, state); + if (unlikely(ret < 0)) + return ret; + + parent_state = ret; + + genpd_lock_nested(parent, depth + 1); + + link->prev_performance_state = link->performance_state; + link->performance_state = parent_state; + + parent_state = _genpd_reeval_performance_state(parent, parent_state); + ret = _genpd_set_performance_state(parent, parent_state, depth + 1); + if (ret) + link->performance_state = link->prev_performance_state; + + genpd_unlock(parent); + + return ret; +} + static int _genpd_set_performance_state(struct generic_pm_domain *genpd, unsigned int state, int depth) { - struct generic_pm_domain *parent; - struct gpd_link *link; - int parent_state, ret; + struct gpd_link *link = NULL; + int ret; if (state == genpd->performance_state) return 0; - /* Propagate to parents of genpd */ - list_for_each_entry(link, &genpd->child_links, child_node) { - parent = link->parent; - - /* Find parent's performance state */ - ret = genpd_xlate_performance_state(genpd, parent, state); - if (unlikely(ret < 0)) - goto err; - - parent_state = ret; - - genpd_lock_nested(parent, depth + 1); - - link->prev_performance_state = link->performance_state; - link->performance_state = parent_state; - parent_state = _genpd_reeval_performance_state(parent, - parent_state); - ret = _genpd_set_performance_state(parent, parent_state, depth + 1); - if (ret) - link->performance_state = link->prev_performance_state; - - genpd_unlock(parent); - - if (ret) - goto err; + /* When scaling up, propagate to parents first in normal order */ + if (state > genpd->performance_state) { + list_for_each_entry(link, &genpd->child_links, child_node) { + ret = _genpd_set_parent_state(genpd, link, state, depth); + if (ret) + goto rollback_parents_up; + } } if (genpd->set_performance_state) { ret = genpd->set_performance_state(genpd, state); - if (ret) - goto err; + if (ret) { + if (link) + goto rollback_parents_up; + return ret; + } + } + + /* When scaling down, propagate to parents last in reverse order */ + if (state < genpd->performance_state) { + list_for_each_entry_reverse(link, &genpd->child_links, child_node) { + ret = _genpd_set_parent_state(genpd, link, state, depth); + if (ret) + goto rollback_parents_down; + } } genpd->performance_state = state; return 0; -err: - /* Encountered an error, lets rollback */ - list_for_each_entry_continue_reverse(link, &genpd->child_links, - child_node) { - parent = link->parent; - - genpd_lock_nested(parent, depth + 1); - - parent_state = link->prev_performance_state; - link->performance_state = parent_state; - - parent_state = _genpd_reeval_performance_state(parent, - parent_state); - if (_genpd_set_performance_state(parent, parent_state, depth + 1)) { - pr_err("%s: Failed to roll back to %d performance state\n", - parent->name, parent_state); - } - - genpd_unlock(parent); - } - +rollback_parents_up: + list_for_each_entry_continue_reverse(link, &genpd->child_links, child_node) + _genpd_rollback_parent_state(link, depth); + return ret; +rollback_parents_down: + list_for_each_entry_continue(link, &genpd->child_links, child_node) + _genpd_rollback_parent_state(link, depth); return ret; } From c358eb461a51b7d94b403db471ac9ff2077c48a6 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Wed, 27 Dec 2023 02:20:32 +0100 Subject: [PATCH 0321/1038] pmdomain: qcom: rpmpd: Keep one RPM handle for all RPMPDs For no apparent reason (as there's just one RPM per SoC), all RPMPDs currently store a copy of a pointer to smd_rpm. Introduce a single, global one to save up on space in each definition. bloat-o-meter reports: Total: Before=92010, After=91062, chg -1.03% Signed-off-by: Konrad Dybcio Link: https://lore.kernel.org/r/20231227-topic-rpmpd_cleanup-v1-1-860ab141b076@linaro.org Signed-off-by: Ulf Hansson --- drivers/pmdomain/qcom/rpmpd.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/qcom/rpmpd.c b/drivers/pmdomain/qcom/rpmpd.c index 7796d65f96e8..90b62767f9d0 100644 --- a/drivers/pmdomain/qcom/rpmpd.c +++ b/drivers/pmdomain/qcom/rpmpd.c @@ -16,6 +16,8 @@ #define domain_to_rpmpd(domain) container_of(domain, struct rpmpd, pd) +static struct qcom_smd_rpm *rpmpd_smd_rpm; + /* Resource types: * RPMPD_X is X encoded as a little-endian, lower-case, ASCII string */ #define RPMPD_SMPA 0x61706d73 @@ -54,7 +56,6 @@ struct rpmpd { bool enabled; const int res_type; const int res_id; - struct qcom_smd_rpm *rpm; unsigned int max_state; __le32 key; bool state_synced; @@ -879,7 +880,7 @@ static int rpmpd_send_enable(struct rpmpd *pd, bool enable) .value = cpu_to_le32(enable), }; - return qcom_rpm_smd_write(pd->rpm, QCOM_SMD_RPM_ACTIVE_STATE, + return qcom_rpm_smd_write(rpmpd_smd_rpm, QCOM_SMD_RPM_ACTIVE_STATE, pd->res_type, pd->res_id, &req, sizeof(req)); } @@ -891,7 +892,7 @@ static int rpmpd_send_corner(struct rpmpd *pd, int state, unsigned int corner) .value = cpu_to_le32(corner), }; - return qcom_rpm_smd_write(pd->rpm, state, pd->res_type, pd->res_id, + return qcom_rpm_smd_write(rpmpd_smd_rpm, state, pd->res_type, pd->res_id, &req, sizeof(req)); }; @@ -1004,12 +1005,11 @@ static int rpmpd_probe(struct platform_device *pdev) int i; size_t num; struct genpd_onecell_data *data; - struct qcom_smd_rpm *rpm; struct rpmpd **rpmpds; const struct rpmpd_desc *desc; - rpm = dev_get_drvdata(pdev->dev.parent); - if (!rpm) { + rpmpd_smd_rpm = dev_get_drvdata(pdev->dev.parent); + if (!rpmpd_smd_rpm) { dev_err(&pdev->dev, "Unable to retrieve handle to RPM\n"); return -ENODEV; } @@ -1039,7 +1039,6 @@ static int rpmpd_probe(struct platform_device *pdev) continue; } - rpmpds[i]->rpm = rpm; rpmpds[i]->max_state = desc->max_state; rpmpds[i]->pd.power_off = rpmpd_power_off; rpmpds[i]->pd.power_on = rpmpd_power_on; From 745fe55bc4c07a22f0d979eaeae44ec4d85f18be Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Wed, 27 Dec 2023 16:18:54 +0100 Subject: [PATCH 0322/1038] pmdomain: core: Print a message when unused power domains are disabled In a similar spirit to commit 12ca59b91d04 ("clk: Print an info line before disabling unused clocks"), print the message in both ignore AND cleanup cases to better inform the user (and more importantly, the developer) when it happens. Signed-off-by: Konrad Dybcio Link: https://lore.kernel.org/r/20231227-topic-pmdomain_spam-v1-1-ff0410086b36@linaro.org Signed-off-by: Ulf Hansson --- drivers/pmdomain/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 8c28d5e50500..46331e71108a 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -1130,6 +1130,7 @@ static int __init genpd_power_off_unused(void) return 0; } + pr_info("genpd: Disabling unused power domains\n"); mutex_lock(&gpd_list_lock); list_for_each_entry(genpd, &gpd_list, gpd_list_node) From e990f3fb926367f67ab3c2310997864f82a0dcd5 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 22 Jan 2024 14:44:41 +0100 Subject: [PATCH 0323/1038] pmdomain: renesas: rcar-gen4-sysc: Remove unneeded includes The R-Car V3U System Controller (SYSC) driver no longer needs these includes since the factoring out of the common R-Car Gen4 SYSC driver in commit e62906d6315f652b ("soc: renesas: rcar-gen4-sysc: Introduce R-Car Gen4 SYSC driver"). The R-Car S4-8 and V4H SYSC drivers never needed these includes, as these drivers always used the common R-Car Gen4 SYSC driver. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/5b440f84ab8b52499ab307c84154dcbc0f41d1d7.1705931035.git.geert+renesas@glider.be Signed-off-by: Ulf Hansson --- drivers/pmdomain/renesas/r8a779a0-sysc.c | 12 ------------ drivers/pmdomain/renesas/r8a779f0-sysc.c | 12 ------------ drivers/pmdomain/renesas/r8a779g0-sysc.c | 12 ------------ 3 files changed, 36 deletions(-) diff --git a/drivers/pmdomain/renesas/r8a779a0-sysc.c b/drivers/pmdomain/renesas/r8a779a0-sysc.c index 04f1bc322ae7..54cdf250f7c2 100644 --- a/drivers/pmdomain/renesas/r8a779a0-sysc.c +++ b/drivers/pmdomain/renesas/r8a779a0-sysc.c @@ -5,19 +5,7 @@ * Copyright (C) 2020 Renesas Electronics Corp. */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include diff --git a/drivers/pmdomain/renesas/r8a779f0-sysc.c b/drivers/pmdomain/renesas/r8a779f0-sysc.c index 5602aa6bd7ed..6ed13cd1cb24 100644 --- a/drivers/pmdomain/renesas/r8a779f0-sysc.c +++ b/drivers/pmdomain/renesas/r8a779f0-sysc.c @@ -5,19 +5,7 @@ * Copyright (C) 2021 Renesas Electronics Corp. */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include diff --git a/drivers/pmdomain/renesas/r8a779g0-sysc.c b/drivers/pmdomain/renesas/r8a779g0-sysc.c index b932eba1b804..249cf43af45b 100644 --- a/drivers/pmdomain/renesas/r8a779g0-sysc.c +++ b/drivers/pmdomain/renesas/r8a779g0-sysc.c @@ -5,19 +5,7 @@ * Copyright (C) 2022 Renesas Electronics Corp. */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include From 5d7f58ee08434a33340f75ac7ac5071eea9673b3 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Thu, 18 Jan 2024 13:42:57 +0800 Subject: [PATCH 0324/1038] pmdomain: ti: Add a null pointer check to the omap_prm_domain_init devm_kasprintf() returns a pointer to dynamically allocated memory which can be NULL upon failure. Ensure the allocation was successful by checking the pointer validity. Signed-off-by: Kunwu Chan Link: https://lore.kernel.org/r/20240118054257.200814-1-chentao@kylinos.cn Signed-off-by: Ulf Hansson --- drivers/pmdomain/ti/omap_prm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pmdomain/ti/omap_prm.c b/drivers/pmdomain/ti/omap_prm.c index c2feae3a634c..b8ceb3c2b81c 100644 --- a/drivers/pmdomain/ti/omap_prm.c +++ b/drivers/pmdomain/ti/omap_prm.c @@ -695,6 +695,8 @@ static int omap_prm_domain_init(struct device *dev, struct omap_prm *prm) data = prm->data; name = devm_kasprintf(dev, GFP_KERNEL, "prm_%s", data->name); + if (!name) + return -ENOMEM; prmd->dev = dev; prmd->prm = prm; From d9e473513930836e617f36c066f205a9a0c4450a Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Fri, 19 Jan 2024 02:47:41 +0100 Subject: [PATCH 0325/1038] pmdomain: imx8mp-blk-ctrl: Error out if domains are missing in DT This driver assumes that domain->power_dev is non-NULL in its suspend/resume path. The assumption is valid, since all the devices that are being looked up here should be described in DT. In case they are not described in DT, because the DT is faulty, suspend/resume attempt would trigger NULL pointer dereference. To avoid this failure, check whether the power_dev assignment is not NULL right away in probe callback and fail early if it is. Signed-off-by: Marek Vasut Reviewed-by: Peng Fan Link: https://lore.kernel.org/r/20240119014807.268694-1-marex@denx.de Signed-off-by: Ulf Hansson --- drivers/pmdomain/imx/imx8m-blk-ctrl.c | 9 ++++++--- drivers/pmdomain/imx/imx8mp-blk-ctrl.c | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c index 1341a707f61b..ca942d7929c2 100644 --- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c @@ -258,11 +258,14 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev) domain->power_dev = dev_pm_domain_attach_by_name(dev, data->gpc_name); - if (IS_ERR(domain->power_dev)) { - dev_err_probe(dev, PTR_ERR(domain->power_dev), + if (IS_ERR_OR_NULL(domain->power_dev)) { + if (!domain->power_dev) + ret = -ENODEV; + else + ret = PTR_ERR(domain->power_dev); + dev_err_probe(dev, ret, "failed to attach power domain \"%s\"\n", data->gpc_name); - ret = PTR_ERR(domain->power_dev); goto cleanup_pds; } diff --git a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c index e3203eb6a022..e488cf79b800 100644 --- a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c @@ -687,11 +687,14 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) domain->power_dev = dev_pm_domain_attach_by_name(dev, data->gpc_name); - if (IS_ERR(domain->power_dev)) { - dev_err_probe(dev, PTR_ERR(domain->power_dev), + if (IS_ERR_OR_NULL(domain->power_dev)) { + if (!domain->power_dev) + ret = -ENODEV; + else + ret = PTR_ERR(domain->power_dev); + dev_err_probe(dev, ret, "failed to attach power domain %s\n", data->gpc_name); - ret = PTR_ERR(domain->power_dev); goto cleanup_pds; } From ce816e0baceaa3a734a54f47e9423cb6c41fe8ec Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Thu, 25 Jan 2024 16:34:37 +0100 Subject: [PATCH 0326/1038] pmdomain: renesas: r8a779h0-sysc: Add r8a779h0 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for R-Car V4M (R8A779H0) SoC power areas to the R-Car SYSC driver. Signed-off-by: Duy Nguyen Signed-off-by: Geert Uytterhoeven Acked-by: Niklas Söderlund Link: https://lore.kernel.org/r/eed6faa02c628d32676ab8ea0eee636b4ffd6c47.1706194617.git.geert+renesas@glider.be Signed-off-by: Ulf Hansson --- drivers/pmdomain/renesas/Kconfig | 4 ++ drivers/pmdomain/renesas/Makefile | 1 + drivers/pmdomain/renesas/r8a779h0-sysc.c | 54 +++++++++++++++++++++++ drivers/pmdomain/renesas/rcar-gen4-sysc.c | 3 ++ drivers/pmdomain/renesas/rcar-gen4-sysc.h | 1 + 5 files changed, 63 insertions(+) create mode 100644 drivers/pmdomain/renesas/r8a779h0-sysc.c diff --git a/drivers/pmdomain/renesas/Kconfig b/drivers/pmdomain/renesas/Kconfig index 80bf2cf8b60e..54acb4b1ec7c 100644 --- a/drivers/pmdomain/renesas/Kconfig +++ b/drivers/pmdomain/renesas/Kconfig @@ -71,6 +71,10 @@ config SYSC_R8A779G0 bool "System Controller support for R-Car V4H" if COMPILE_TEST select SYSC_RCAR_GEN4 +config SYSC_R8A779H0 + bool "System Controller support for R-Car V4M" if COMPILE_TEST + select SYSC_RCAR_GEN4 + config SYSC_RMOBILE bool "System Controller support for R-Mobile" if COMPILE_TEST diff --git a/drivers/pmdomain/renesas/Makefile b/drivers/pmdomain/renesas/Makefile index e306e396fc8c..89180f19c23b 100644 --- a/drivers/pmdomain/renesas/Makefile +++ b/drivers/pmdomain/renesas/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_SYSC_R8A77995) += r8a77995-sysc.o obj-$(CONFIG_SYSC_R8A779A0) += r8a779a0-sysc.o obj-$(CONFIG_SYSC_R8A779F0) += r8a779f0-sysc.o obj-$(CONFIG_SYSC_R8A779G0) += r8a779g0-sysc.o +obj-$(CONFIG_SYSC_R8A779H0) += r8a779h0-sysc.o # Family obj-$(CONFIG_SYSC_RCAR) += rcar-sysc.o obj-$(CONFIG_SYSC_RCAR_GEN4) += rcar-gen4-sysc.o diff --git a/drivers/pmdomain/renesas/r8a779h0-sysc.c b/drivers/pmdomain/renesas/r8a779h0-sysc.c new file mode 100644 index 000000000000..e13372cb80ee --- /dev/null +++ b/drivers/pmdomain/renesas/r8a779h0-sysc.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas R-Car V4M System Controller + * + * Copyright (C) 2023 Renesas Electronics Corp + */ + +#include + +#include + +#include "rcar-gen4-sysc.h" + +static struct rcar_gen4_sysc_area r8a779h0_areas[] __initdata = { + { "always-on", R8A779H0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON }, + { "c4", R8A779H0_PD_C4, R8A779H0_PD_ALWAYS_ON }, + { "a2e0d0", R8A779H0_PD_A2E0D0, R8A779H0_PD_C4, PD_SCU }, + { "a1e0d0c0", R8A779H0_PD_A1E0D0C0, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c1", R8A779H0_PD_A1E0D0C1, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c2", R8A779H0_PD_A1E0D0C2, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c3", R8A779H0_PD_A1E0D0C3, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a3cr0", R8A779H0_PD_A3CR0, R8A779H0_PD_ALWAYS_ON, PD_CPU_NOCR }, + { "a3cr1", R8A779H0_PD_A3CR1, R8A779H0_PD_ALWAYS_ON, PD_CPU_NOCR }, + { "a3cr2", R8A779H0_PD_A3CR2, R8A779H0_PD_ALWAYS_ON, PD_CPU_NOCR }, + { "a33dga", R8A779H0_PD_A33DGA, R8A779H0_PD_C4 }, + { "a23dgb", R8A779H0_PD_A23DGB, R8A779H0_PD_A33DGA }, + { "a3vip0", R8A779H0_PD_A3VIP0, R8A779H0_PD_C4 }, + { "a3vip2", R8A779H0_PD_A3VIP2, R8A779H0_PD_C4 }, + { "a3dul", R8A779H0_PD_A3DUL, R8A779H0_PD_C4 }, + { "a3isp0", R8A779H0_PD_A3ISP0, R8A779H0_PD_C4 }, + { "a2cn0", R8A779H0_PD_A2CN0, R8A779H0_PD_C4 }, + { "a1cn0", R8A779H0_PD_A1CN0, R8A779H0_PD_A2CN0 }, + { "a1dsp0", R8A779H0_PD_A1DSP0, R8A779H0_PD_A2CN0 }, + { "a1dsp1", R8A779H0_PD_A1DSP1, R8A779H0_PD_A2CN0 }, + { "a2imp01", R8A779H0_PD_A2IMP01, R8A779H0_PD_C4 }, + { "a2psc", R8A779H0_PD_A2PSC, R8A779H0_PD_C4 }, + { "a2dma", R8A779H0_PD_A2DMA, R8A779H0_PD_C4 }, + { "a2cv0", R8A779H0_PD_A2CV0, R8A779H0_PD_C4 }, + { "a2cv1", R8A779H0_PD_A2CV1, R8A779H0_PD_C4 }, + { "a2cv2", R8A779H0_PD_A2CV2, R8A779H0_PD_C4 }, + { "a2cv3", R8A779H0_PD_A2CV3, R8A779H0_PD_C4 }, + { "a3imr0", R8A779H0_PD_A3IMR0, R8A779H0_PD_C4 }, + { "a3imr1", R8A779H0_PD_A3IMR1, R8A779H0_PD_C4 }, + { "a3imr2", R8A779H0_PD_A3IMR2, R8A779H0_PD_C4 }, + { "a3imr3", R8A779H0_PD_A3IMR3, R8A779H0_PD_C4 }, + { "a3vc", R8A779H0_PD_A3VC, R8A779H0_PD_C4 }, + { "a3pci", R8A779H0_PD_A3PCI, R8A779H0_PD_C4 }, + { "a2pciphy", R8A779H0_PD_A2PCIPHY, R8A779H0_PD_A3PCI }, +}; + +const struct rcar_gen4_sysc_info r8a779h0_sysc_info __initconst = { + .areas = r8a779h0_areas, + .num_areas = ARRAY_SIZE(r8a779h0_areas), +}; diff --git a/drivers/pmdomain/renesas/rcar-gen4-sysc.c b/drivers/pmdomain/renesas/rcar-gen4-sysc.c index 9e5e6e077abc..728248659a97 100644 --- a/drivers/pmdomain/renesas/rcar-gen4-sysc.c +++ b/drivers/pmdomain/renesas/rcar-gen4-sysc.c @@ -284,6 +284,9 @@ static const struct of_device_id rcar_gen4_sysc_matches[] __initconst = { #endif #ifdef CONFIG_SYSC_R8A779G0 { .compatible = "renesas,r8a779g0-sysc", .data = &r8a779g0_sysc_info }, +#endif +#ifdef CONFIG_SYSC_R8A779H0 + { .compatible = "renesas,r8a779h0-sysc", .data = &r8a779h0_sysc_info }, #endif { /* sentinel */ } }; diff --git a/drivers/pmdomain/renesas/rcar-gen4-sysc.h b/drivers/pmdomain/renesas/rcar-gen4-sysc.h index 388cfa8f8f9f..fdf843aa5113 100644 --- a/drivers/pmdomain/renesas/rcar-gen4-sysc.h +++ b/drivers/pmdomain/renesas/rcar-gen4-sysc.h @@ -40,5 +40,6 @@ struct rcar_gen4_sysc_info { extern const struct rcar_gen4_sysc_info r8a779a0_sysc_info; extern const struct rcar_gen4_sysc_info r8a779f0_sysc_info; extern const struct rcar_gen4_sysc_info r8a779g0_sysc_info; +extern const struct rcar_gen4_sysc_info r8a779h0_sysc_info; #endif /* __SOC_RENESAS_RCAR_GEN4_SYSC_H__ */ From a0691f280b3240dc4aeca7d0f0c824d0277c4856 Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Tue, 6 Feb 2024 10:05:34 +0100 Subject: [PATCH 0327/1038] pmdomain: mediatek: Use devm_platform_ioremap_resource() in init_scp() A wrapper function is available since the commit 7945f929f1a77a1c8887a97ca07f87626858ff42 ("drivers: provide devm_platform_ioremap_resource()"). * Thus reuse existing functionality instead of keeping duplicate source code. * Delete a local variable which became unnecessary with this refactoring. This issue was transformed by using the Coccinelle software. Signed-off-by: Markus Elfring Reviewed-by: Geert Uytterhoeven Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/6e397bf2-1d45-434f-8619-58a737a138e3@web.de Signed-off-by: Ulf Hansson --- drivers/pmdomain/mediatek/mtk-scpsys.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/pmdomain/mediatek/mtk-scpsys.c b/drivers/pmdomain/mediatek/mtk-scpsys.c index b374d01fdac7..59a7a8c261ed 100644 --- a/drivers/pmdomain/mediatek/mtk-scpsys.c +++ b/drivers/pmdomain/mediatek/mtk-scpsys.c @@ -425,7 +425,6 @@ static struct scp *init_scp(struct platform_device *pdev, bool bus_prot_reg_update) { struct genpd_onecell_data *pd_data; - struct resource *res; int i, j; struct scp *scp; struct clk *clk[CLK_MAX]; @@ -441,8 +440,7 @@ static struct scp *init_scp(struct platform_device *pdev, scp->dev = &pdev->dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - scp->base = devm_ioremap_resource(&pdev->dev, res); + scp->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(scp->base)) return ERR_CAST(scp->base); From 697624ee8ad557ab5417f985d2c804241a7ad30d Mon Sep 17 00:00:00 2001 From: Adam Ford Date: Sat, 3 Feb 2024 10:52:44 -0600 Subject: [PATCH 0328/1038] pmdomain: imx8mp-blk-ctrl: imx8mp_blk: Add fdcc clock to hdmimix domain According to i.MX8MP RM and HDMI ADD, the fdcc clock is part of hdmi rx verification IP that should not enable for HDMI TX. But actually if the clock is disabled before HDMI/LCDIF probe, LCDIF will not get pixel clock from HDMI PHY and print the error logs: [CRTC:39:crtc-2] vblank wait timed out WARNING: CPU: 2 PID: 9 at drivers/gpu/drm/drm_atomic_helper.c:1634 drm_atomic_helper_wait_for_vblanks.part.0+0x23c/0x260 Add fdcc clock to LCDIF and HDMI TX power domains to fix the issue. Signed-off-by: Adam Ford Reviewed-by: Jacky Bai Signed-off-by: Sandor Yu Link: https://lore.kernel.org/r/20240203165307.7806-5-aford173@gmail.com Signed-off-by: Ulf Hansson --- drivers/pmdomain/imx/imx8mp-blk-ctrl.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c index e488cf79b800..77e889165eed 100644 --- a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c @@ -55,7 +55,7 @@ struct imx8mp_blk_ctrl_domain_data { const char *gpc_name; }; -#define DOMAIN_MAX_CLKS 2 +#define DOMAIN_MAX_CLKS 3 #define DOMAIN_MAX_PATHS 3 struct imx8mp_blk_ctrl_domain { @@ -457,8 +457,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = { }, [IMX8MP_HDMIBLK_PD_LCDIF] = { .name = "hdmiblk-lcdif", - .clk_names = (const char *[]){ "axi", "apb" }, - .num_clks = 2, + .clk_names = (const char *[]){ "axi", "apb", "fdcc" }, + .num_clks = 3, .gpc_name = "lcdif", .path_names = (const char *[]){"lcdif-hdmi"}, .num_paths = 1, @@ -483,8 +483,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = { }, [IMX8MP_HDMIBLK_PD_HDMI_TX] = { .name = "hdmiblk-hdmi-tx", - .clk_names = (const char *[]){ "apb", "ref_266m" }, - .num_clks = 2, + .clk_names = (const char *[]){ "apb", "ref_266m", "fdcc" }, + .num_clks = 3, .gpc_name = "hdmi-tx", }, [IMX8MP_HDMIBLK_PD_HDMI_TX_PHY] = { From 161e16a5e50a82d219b3df3ce32286b0a2ae08bd Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 Jan 2024 13:39:47 +0100 Subject: [PATCH 0329/1038] PM: domains: Add helper functions to attach/detach multiple PM domains Attaching/detaching of a device to multiple PM domains has started to become a common operation for many drivers, typically during ->probe() and ->remove(). In most cases, this has lead to lots of boilerplate code in the drivers. To fixup up the situation, let's introduce a pair of helper functions, dev_pm_domain_attach|detach_list(), that driver can use instead of the open-coding. Note that, it seems reasonable to limit the support for these helpers to DT based platforms, at it's the only valid use case for now. Suggested-by: Daniel Baluta Tested-by: Bryan O'Donoghue Tested-by: Iuliana Prodan Signed-off-by: Ulf Hansson Link: https://lore.kernel.org/r/20240130123951.236243-2-ulf.hansson@linaro.org --- drivers/base/power/common.c | 134 ++++++++++++++++++++++++++++++++++++ include/linux/pm_domain.h | 38 ++++++++++ 2 files changed, 172 insertions(+) diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index 44ec20918a4d..327d168dd37a 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -167,6 +167,115 @@ struct device *dev_pm_domain_attach_by_name(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_name); +/** + * dev_pm_domain_attach_list - Associate a device with its PM domains. + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * This function helps to attach a device to its multiple PM domains. The + * caller, which is typically a driver's probe function, may provide a list of + * names for the PM domains that we should try to attach the device to, but it + * may also provide an empty list, in case the attach should be done for all of + * the available PM domains. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. Note that, to detach the list of PM domains, the driver shall call + * dev_pm_domain_detach_list(), typically during the remove phase. + */ +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + struct device_node *np = dev->of_node; + struct dev_pm_domain_list *pds; + struct device *pd_dev = NULL; + int ret, i, num_pds = 0; + bool by_id = true; + u32 pd_flags = data ? data->pd_flags : 0; + u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 : + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME; + + if (dev->pm_domain) + return -EEXIST; + + /* For now this is limited to OF based platforms. */ + if (!np) + return 0; + + if (data && data->pd_names) { + num_pds = data->num_pd_names; + by_id = false; + } else { + num_pds = of_count_phandle_with_args(np, "power-domains", + "#power-domain-cells"); + } + + if (num_pds <= 0) + return 0; + + pds = devm_kzalloc(dev, sizeof(*pds), GFP_KERNEL); + if (!pds) + return -ENOMEM; + + pds->pd_devs = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_devs), + GFP_KERNEL); + if (!pds->pd_devs) + return -ENOMEM; + + pds->pd_links = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_links), + GFP_KERNEL); + if (!pds->pd_links) + return -ENOMEM; + + if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON) + link_flags |= DL_FLAG_RPM_ACTIVE; + + for (i = 0; i < num_pds; i++) { + if (by_id) + pd_dev = dev_pm_domain_attach_by_id(dev, i); + else + pd_dev = dev_pm_domain_attach_by_name(dev, + data->pd_names[i]); + if (IS_ERR_OR_NULL(pd_dev)) { + ret = pd_dev ? PTR_ERR(pd_dev) : -ENODEV; + goto err_attach; + } + + if (link_flags) { + struct device_link *link; + + link = device_link_add(dev, pd_dev, link_flags); + if (!link) { + ret = -ENODEV; + goto err_link; + } + + pds->pd_links[i] = link; + } + + pds->pd_devs[i] = pd_dev; + } + + pds->num_pds = num_pds; + *list = pds; + return num_pds; + +err_link: + dev_pm_domain_detach(pd_dev, true); +err_attach: + while (--i >= 0) { + if (pds->pd_links[i]) + device_link_del(pds->pd_links[i]); + dev_pm_domain_detach(pds->pd_devs[i], true); + } + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list); + /** * dev_pm_domain_detach - Detach a device from its PM domain. * @dev: Device to detach. @@ -187,6 +296,31 @@ void dev_pm_domain_detach(struct device *dev, bool power_off) } EXPORT_SYMBOL_GPL(dev_pm_domain_detach); +/** + * dev_pm_domain_detach_list - Detach a list of PM domains. + * @list: The list of PM domains to detach. + * + * This function reverse the actions from dev_pm_domain_attach_list(). + * Typically it should be invoked during the remove phase from drivers. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + */ +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) +{ + int i; + + if (!list) + return; + + for (i = 0; i < list->num_pds; i++) { + if (list->pd_links[i]) + device_link_del(list->pd_links[i]); + dev_pm_domain_detach(list->pd_devs[i], true); + } +} +EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list); + /** * dev_pm_domain_start - Start the device through its PM domain. * @dev: Device to start. diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index b97c5e9820f9..fb91770ba4ba 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -19,6 +19,33 @@ #include #include +/* + * Flags to control the behaviour when attaching a device to its PM domains. + * + * PD_FLAG_NO_DEV_LINK: As the default behaviour creates a device-link + * for every PM domain that gets attached, this + * flag can be used to skip that. + * + * PD_FLAG_DEV_LINK_ON: Add the DL_FLAG_RPM_ACTIVE to power-on the + * supplier and its PM domain when creating the + * device-links. + * + */ +#define PD_FLAG_NO_DEV_LINK BIT(0) +#define PD_FLAG_DEV_LINK_ON BIT(1) + +struct dev_pm_domain_attach_data { + const char * const *pd_names; + const u32 num_pd_names; + const u32 pd_flags; +}; + +struct dev_pm_domain_list { + struct device **pd_devs; + struct device_link **pd_links; + u32 num_pds; +}; + /* * Flags to control the behaviour of a genpd. * @@ -420,7 +447,11 @@ struct device *dev_pm_domain_attach_by_id(struct device *dev, unsigned int index); struct device *dev_pm_domain_attach_by_name(struct device *dev, const char *name); +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list); void dev_pm_domain_detach(struct device *dev, bool power_off); +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list); int dev_pm_domain_start(struct device *dev); void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd); int dev_pm_domain_set_performance_state(struct device *dev, unsigned int state); @@ -439,7 +470,14 @@ static inline struct device *dev_pm_domain_attach_by_name(struct device *dev, { return NULL; } +static inline int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + return 0; +} static inline void dev_pm_domain_detach(struct device *dev, bool power_off) {} +static inline void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) {} static inline int dev_pm_domain_start(struct device *dev) { return 0; From ccd946889a3754332297e00f1535f6d7bf48f946 Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 Jan 2024 13:39:48 +0100 Subject: [PATCH 0330/1038] remoteproc: imx_dsp_rproc: Convert to dev_pm_domain_attach|detach_list() Let's avoid the boilerplate code to manage the multiple PM domain case, by converting into using dev_pm_domain_attach|detach_list(). Cc: Bjorn Andersson Cc: Shawn Guo Cc: Sascha Hauer Cc: Daniel Baluta Cc: Tested-by: Iuliana Prodan Reviewed-by: Iuliana Prodan Reviewed-by: Mathieu Poirier Signed-off-by: Ulf Hansson Link: https://lore.kernel.org/r/20240130123951.236243-3-ulf.hansson@linaro.org --- drivers/remoteproc/imx_dsp_rproc.c | 82 ++++-------------------------- 1 file changed, 9 insertions(+), 73 deletions(-) diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index a1c62d15f16c..d73727a5828a 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -103,12 +103,10 @@ enum imx_dsp_rp_mbox_messages { * @tx_ch: mailbox tx channel handle * @rx_ch: mailbox rx channel handle * @rxdb_ch: mailbox rx doorbell channel handle - * @pd_dev: power domain device - * @pd_dev_link: power domain device link + * @pd_list: power domain list * @ipc_handle: System Control Unit ipc handle * @rproc_work: work for processing virtio interrupts * @pm_comp: completion primitive to sync for suspend response - * @num_domains: power domain number * @flags: control flags */ struct imx_dsp_rproc { @@ -121,12 +119,10 @@ struct imx_dsp_rproc { struct mbox_chan *tx_ch; struct mbox_chan *rx_ch; struct mbox_chan *rxdb_ch; - struct device **pd_dev; - struct device_link **pd_dev_link; + struct dev_pm_domain_list *pd_list; struct imx_sc_ipc *ipc_handle; struct work_struct rproc_work; struct completion pm_comp; - int num_domains; u32 flags; }; @@ -955,74 +951,14 @@ static const struct rproc_ops imx_dsp_rproc_ops = { static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv) { struct device *dev = priv->rproc->dev.parent; - int ret, i; + int ret; - priv->num_domains = of_count_phandle_with_args(dev->of_node, - "power-domains", - "#power-domain-cells"); - - /* If only one domain, then no need to link the device */ - if (priv->num_domains <= 1) + /* A single PM domain is already attached. */ + if (dev->pm_domain) return 0; - priv->pd_dev = devm_kmalloc_array(dev, priv->num_domains, - sizeof(*priv->pd_dev), - GFP_KERNEL); - if (!priv->pd_dev) - return -ENOMEM; - - priv->pd_dev_link = devm_kmalloc_array(dev, priv->num_domains, - sizeof(*priv->pd_dev_link), - GFP_KERNEL); - if (!priv->pd_dev_link) - return -ENOMEM; - - for (i = 0; i < priv->num_domains; i++) { - priv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); - if (IS_ERR(priv->pd_dev[i])) { - ret = PTR_ERR(priv->pd_dev[i]); - goto detach_pm; - } - - /* - * device_link_add will check priv->pd_dev[i], if it is - * NULL, then will break. - */ - priv->pd_dev_link[i] = device_link_add(dev, - priv->pd_dev[i], - DL_FLAG_STATELESS | - DL_FLAG_PM_RUNTIME); - if (!priv->pd_dev_link[i]) { - dev_pm_domain_detach(priv->pd_dev[i], false); - ret = -EINVAL; - goto detach_pm; - } - } - - return 0; - -detach_pm: - while (--i >= 0) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return ret; -} - -static int imx_dsp_detach_pm_domains(struct imx_dsp_rproc *priv) -{ - int i; - - if (priv->num_domains <= 1) - return 0; - - for (i = 0; i < priv->num_domains; i++) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return 0; + ret = dev_pm_domain_attach_list(dev, NULL, &priv->pd_list); + return ret < 0 ? ret : 0; } /** @@ -1154,7 +1090,7 @@ static int imx_dsp_rproc_probe(struct platform_device *pdev) return 0; err_detach_domains: - imx_dsp_detach_pm_domains(priv); + dev_pm_domain_detach_list(priv->pd_list); err_put_rproc: rproc_free(rproc); @@ -1168,7 +1104,7 @@ static void imx_dsp_rproc_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); rproc_del(rproc); - imx_dsp_detach_pm_domains(priv); + dev_pm_domain_detach_list(priv->pd_list); rproc_free(rproc); } From 3f6905fb2fdeaf5faaecfc6b80fe30c5a32b612b Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 Jan 2024 13:39:49 +0100 Subject: [PATCH 0331/1038] remoteproc: imx_rproc: Convert to dev_pm_domain_attach|detach_list() Let's avoid the boilerplate code to manage the multiple PM domain case, by converting into using dev_pm_domain_attach|detach_list(). Cc: Bjorn Andersson Cc: Shawn Guo Cc: Sascha Hauer Cc: Daniel Baluta Cc: Tested-by: Iuliana Prodan Reviewed-by: Iuliana Prodan Reviewed-by: Mathieu Poirier Signed-off-by: Ulf Hansson Link: https://lore.kernel.org/r/20240130123951.236243-4-ulf.hansson@linaro.org --- drivers/remoteproc/imx_rproc.c | 73 +++++----------------------------- 1 file changed, 9 insertions(+), 64 deletions(-) diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 8bb293b9f327..3161f14442bc 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -92,7 +92,6 @@ struct imx_rproc_mem { static int imx_rproc_xtr_mbox_init(struct rproc *rproc); static void imx_rproc_free_mbox(struct rproc *rproc); -static int imx_rproc_detach_pd(struct rproc *rproc); struct imx_rproc { struct device *dev; @@ -113,10 +112,8 @@ struct imx_rproc { u32 rproc_pt; /* partition id */ u32 rsrc_id; /* resource id */ u32 entry; /* cpu start address */ - int num_pd; u32 core_index; - struct device **pd_dev; - struct device_link **pd_dev_link; + struct dev_pm_domain_list *pd_list; }; static const struct imx_rproc_att imx_rproc_att_imx93[] = { @@ -853,7 +850,7 @@ static void imx_rproc_put_scu(struct rproc *rproc) return; if (imx_sc_rm_is_resource_owned(priv->ipc_handle, priv->rsrc_id)) { - imx_rproc_detach_pd(rproc); + dev_pm_domain_detach_list(priv->pd_list); return; } @@ -880,72 +877,20 @@ static int imx_rproc_partition_notify(struct notifier_block *nb, static int imx_rproc_attach_pd(struct imx_rproc *priv) { struct device *dev = priv->dev; - int ret, i; + int ret; + struct dev_pm_domain_attach_data pd_data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; /* * If there is only one power-domain entry, the platform driver framework * will handle it, no need handle it in this driver. */ - priv->num_pd = of_count_phandle_with_args(dev->of_node, "power-domains", - "#power-domain-cells"); - if (priv->num_pd <= 1) + if (dev->pm_domain) return 0; - priv->pd_dev = devm_kmalloc_array(dev, priv->num_pd, sizeof(*priv->pd_dev), GFP_KERNEL); - if (!priv->pd_dev) - return -ENOMEM; - - priv->pd_dev_link = devm_kmalloc_array(dev, priv->num_pd, sizeof(*priv->pd_dev_link), - GFP_KERNEL); - - if (!priv->pd_dev_link) - return -ENOMEM; - - for (i = 0; i < priv->num_pd; i++) { - priv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); - if (IS_ERR(priv->pd_dev[i])) { - ret = PTR_ERR(priv->pd_dev[i]); - goto detach_pd; - } - - priv->pd_dev_link[i] = device_link_add(dev, priv->pd_dev[i], DL_FLAG_STATELESS | - DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); - if (!priv->pd_dev_link[i]) { - dev_pm_domain_detach(priv->pd_dev[i], false); - ret = -EINVAL; - goto detach_pd; - } - } - - return 0; - -detach_pd: - while (--i >= 0) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return ret; -} - -static int imx_rproc_detach_pd(struct rproc *rproc) -{ - struct imx_rproc *priv = rproc->priv; - int i; - - /* - * If there is only one power-domain entry, the platform driver framework - * will handle it, no need handle it in this driver. - */ - if (priv->num_pd <= 1) - return 0; - - for (i = 0; i < priv->num_pd; i++) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return 0; + ret = dev_pm_domain_attach_list(dev, &pd_data, &priv->pd_list); + return ret < 0 ? ret : 0; } static int imx_rproc_detect_mode(struct imx_rproc *priv) From db34c47d56d5ad91f61978c367d55069a81255e6 Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 Jan 2024 13:39:50 +0100 Subject: [PATCH 0332/1038] remoteproc: qcom_q6v5_adsp: Convert to dev_pm_domain_attach|detach_list() Let's avoid some of the boilerplate code to manage the various PM domain cases, by converting into using dev_pm_domain_attach|detach_list(). As a part of the conversion, we are moving over to use device_links, which simplifies the runtime PM support too. Moreover, while attaching let's trust that an already attached single PM domain is the correct one. Cc: Mathieu Poirier Cc: Bjorn Andersson Cc: Konrad Dybcio Cc: Signed-off-by: Ulf Hansson Link: https://lore.kernel.org/r/20240130123951.236243-5-ulf.hansson@linaro.org --- drivers/remoteproc/qcom_q6v5_adsp.c | 160 +++++++++++++--------------- 1 file changed, 73 insertions(+), 87 deletions(-) diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index 6c67514cc493..93f9a1537ec6 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -55,8 +55,6 @@ #define QDSP6SS_CORE_CBCR 0x20 #define QDSP6SS_SLEEP_CBCR 0x3c -#define QCOM_Q6V5_RPROC_PROXY_PD_MAX 3 - #define LPASS_BOOT_CORE_START BIT(0) #define LPASS_BOOT_CMD_START BIT(0) #define LPASS_EFUSE_Q6SS_EVB_SEL 0x0 @@ -74,7 +72,8 @@ struct adsp_pil_data { const char **clk_ids; int num_clks; - const char **proxy_pd_names; + const char **pd_names; + unsigned int num_pds; const char *load_state; }; @@ -110,8 +109,7 @@ struct qcom_adsp { size_t mem_size; bool has_iommu; - struct device *proxy_pds[QCOM_Q6V5_RPROC_PROXY_PD_MAX]; - size_t proxy_pd_count; + struct dev_pm_domain_list *pd_list; struct qcom_rproc_glink glink_subdev; struct qcom_rproc_ssr ssr_subdev; @@ -120,98 +118,92 @@ struct qcom_adsp { int (*shutdown)(struct qcom_adsp *adsp); }; -static int qcom_rproc_pds_attach(struct device *dev, struct qcom_adsp *adsp, - const char **pd_names) +static int qcom_rproc_pds_attach(struct qcom_adsp *adsp, const char **pd_names, + unsigned int num_pds) { - struct device **devs = adsp->proxy_pds; - size_t num_pds = 0; + struct device *dev = adsp->dev; + struct dev_pm_domain_attach_data pd_data = { + .pd_names = pd_names, + .num_pd_names = num_pds, + }; int ret; - int i; + + /* Handle single power domain */ + if (dev->pm_domain) + goto out; if (!pd_names) return 0; - /* Handle single power domain */ - if (dev->pm_domain) { - devs[0] = dev; - pm_runtime_enable(dev); - return 1; - } + ret = dev_pm_domain_attach_list(dev, &pd_data, &adsp->pd_list); + if (ret < 0) + return ret; - while (pd_names[num_pds]) - num_pds++; - - if (num_pds > ARRAY_SIZE(adsp->proxy_pds)) - return -E2BIG; - - for (i = 0; i < num_pds; i++) { - devs[i] = dev_pm_domain_attach_by_name(dev, pd_names[i]); - if (IS_ERR_OR_NULL(devs[i])) { - ret = PTR_ERR(devs[i]) ? : -ENODATA; - goto unroll_attach; - } - } - - return num_pds; - -unroll_attach: - for (i--; i >= 0; i--) - dev_pm_domain_detach(devs[i], false); - - return ret; +out: + pm_runtime_enable(dev); + return 0; } -static void qcom_rproc_pds_detach(struct qcom_adsp *adsp, struct device **pds, - size_t pd_count) +static void qcom_rproc_pds_detach(struct qcom_adsp *adsp) { struct device *dev = adsp->dev; - int i; + struct dev_pm_domain_list *pds = adsp->pd_list; - /* Handle single power domain */ - if (dev->pm_domain && pd_count) { - pm_runtime_disable(dev); - return; - } + dev_pm_domain_detach_list(pds); - for (i = 0; i < pd_count; i++) - dev_pm_domain_detach(pds[i], false); + if (dev->pm_domain || pds) + pm_runtime_disable(adsp->dev); } -static int qcom_rproc_pds_enable(struct qcom_adsp *adsp, struct device **pds, - size_t pd_count) +static int qcom_rproc_pds_enable(struct qcom_adsp *adsp) { - int ret; - int i; + struct device *dev = adsp->dev; + struct dev_pm_domain_list *pds = adsp->pd_list; + int ret, i = 0; - for (i = 0; i < pd_count; i++) { - dev_pm_genpd_set_performance_state(pds[i], INT_MAX); - ret = pm_runtime_resume_and_get(pds[i]); - if (ret < 0) { - dev_pm_genpd_set_performance_state(pds[i], 0); - goto unroll_pd_votes; - } + if (!dev->pm_domain && !pds) + return 0; + + if (dev->pm_domain) + dev_pm_genpd_set_performance_state(dev, INT_MAX); + + while (pds && i < pds->num_pds) { + dev_pm_genpd_set_performance_state(pds->pd_devs[i], INT_MAX); + i++; } - return 0; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + while (pds && i > 0) { + i--; + dev_pm_genpd_set_performance_state(pds->pd_devs[i], 0); + } -unroll_pd_votes: - for (i--; i >= 0; i--) { - dev_pm_genpd_set_performance_state(pds[i], 0); - pm_runtime_put(pds[i]); + if (dev->pm_domain) + dev_pm_genpd_set_performance_state(dev, 0); } return ret; } -static void qcom_rproc_pds_disable(struct qcom_adsp *adsp, struct device **pds, - size_t pd_count) +static void qcom_rproc_pds_disable(struct qcom_adsp *adsp) { - int i; + struct device *dev = adsp->dev; + struct dev_pm_domain_list *pds = adsp->pd_list; + int i = 0; - for (i = 0; i < pd_count; i++) { - dev_pm_genpd_set_performance_state(pds[i], 0); - pm_runtime_put(pds[i]); + if (!dev->pm_domain && !pds) + return; + + if (dev->pm_domain) + dev_pm_genpd_set_performance_state(dev, 0); + + while (pds && i < pds->num_pds) { + dev_pm_genpd_set_performance_state(pds->pd_devs[i], 0); + i++; } + + pm_runtime_put(dev); } static int qcom_wpss_shutdown(struct qcom_adsp *adsp) @@ -397,8 +389,7 @@ static int adsp_start(struct rproc *rproc) if (ret) goto adsp_smmu_unmap; - ret = qcom_rproc_pds_enable(adsp, adsp->proxy_pds, - adsp->proxy_pd_count); + ret = qcom_rproc_pds_enable(adsp); if (ret < 0) goto disable_xo_clk; @@ -448,7 +439,7 @@ static int adsp_start(struct rproc *rproc) disable_adsp_clks: clk_bulk_disable_unprepare(adsp->num_clks, adsp->clks); disable_power_domain: - qcom_rproc_pds_disable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_disable(adsp); disable_xo_clk: clk_disable_unprepare(adsp->xo); adsp_smmu_unmap: @@ -464,7 +455,7 @@ static void qcom_adsp_pil_handover(struct qcom_q6v5 *q6v5) struct qcom_adsp *adsp = container_of(q6v5, struct qcom_adsp, q6v5); clk_disable_unprepare(adsp->xo); - qcom_rproc_pds_disable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_disable(adsp); } static int adsp_stop(struct rproc *rproc) @@ -715,13 +706,11 @@ static int adsp_probe(struct platform_device *pdev) if (ret) goto free_rproc; - ret = qcom_rproc_pds_attach(adsp->dev, adsp, - desc->proxy_pd_names); + ret = qcom_rproc_pds_attach(adsp, desc->pd_names, desc->num_pds); if (ret < 0) { dev_err(&pdev->dev, "Failed to attach proxy power domains\n"); goto free_rproc; } - adsp->proxy_pd_count = ret; ret = adsp_init_reset(adsp); if (ret) @@ -753,7 +742,7 @@ static int adsp_probe(struct platform_device *pdev) return 0; disable_pm: - qcom_rproc_pds_detach(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_detach(adsp); free_rproc: rproc_free(rproc); @@ -771,7 +760,7 @@ static void adsp_remove(struct platform_device *pdev) qcom_remove_glink_subdev(adsp->rproc, &adsp->glink_subdev); qcom_remove_sysmon_subdev(adsp->sysmon); qcom_remove_ssr_subdev(adsp->rproc, &adsp->ssr_subdev); - qcom_rproc_pds_detach(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_detach(adsp); rproc_free(adsp->rproc); } @@ -788,9 +777,8 @@ static const struct adsp_pil_data adsp_resource_init = { "qdsp6ss_xo", "qdsp6ss_sleep", "qdsp6ss_core", NULL }, .num_clks = 7, - .proxy_pd_names = (const char*[]) { - "cx", NULL - }, + .pd_names = (const char*[]) { "cx" }, + .num_pds = 1, }; static const struct adsp_pil_data adsp_sc7280_resource_init = { @@ -821,9 +809,8 @@ static const struct adsp_pil_data cdsp_resource_init = { "q6_axim", NULL }, .num_clks = 7, - .proxy_pd_names = (const char*[]) { - "cx", NULL - }, + .pd_names = (const char*[]) { "cx" }, + .num_pds = 1, }; static const struct adsp_pil_data wpss_resource_init = { @@ -839,9 +826,8 @@ static const struct adsp_pil_data wpss_resource_init = { "ahb_bdg", "ahb", "rscp", NULL }, .num_clks = 3, - .proxy_pd_names = (const char*[]) { - "cx", "mx", NULL - }, + .pd_names = (const char*[]) { "cx", "mx" }, + .num_pds = 2, }; static const struct of_device_id adsp_of_match[] = { From 693c301a3aab4f7001f26f1006437932fa19a53b Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Tue, 30 Jan 2024 13:39:51 +0100 Subject: [PATCH 0333/1038] media: venus: Convert to dev_pm_domain_attach|detach_list() for vcodec Let's avoid some of the boilerplate code to manage the vcodec PM domains, by converting into using dev_pm_domain_attach|detach_list(). Cc: Mauro Carvalho Chehab Cc: Stanimir Varbanov Cc: Vikash Garodia Cc: Bjorn Andersson Cc: Konrad Dybcio Cc: Tested-by: Bryan O'Donoghue Reviewed-by: Bryan O'Donoghue Signed-off-by: Ulf Hansson Link: https://lore.kernel.org/r/20240130123951.236243-6-ulf.hansson@linaro.org --- drivers/media/platform/qcom/venus/core.c | 12 +++-- drivers/media/platform/qcom/venus/core.h | 7 ++- .../media/platform/qcom/venus/pm_helpers.c | 48 +++++++------------ 3 files changed, 26 insertions(+), 41 deletions(-) diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index a712dd4f02a5..ce206b709754 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -114,7 +115,8 @@ static void venus_sys_error_handler(struct work_struct *work) pm_runtime_put_sync(core->dev); for (i = 0; i < max_attempts; i++) { - if (!core->pmdomains[0] || !pm_runtime_active(core->pmdomains[0])) + if (!core->pmdomains || + !pm_runtime_active(core->pmdomains->pd_devs[0])) break; usleep_range(1000, 1500); } @@ -705,7 +707,7 @@ static const struct venus_resources sdm845_res_v2 = { .vcodec0_clks = { "vcodec0_core", "vcodec0_bus" }, .vcodec1_clks = { "vcodec1_core", "vcodec1_bus" }, .vcodec_clks_num = 2, - .vcodec_pmdomains = { "venus", "vcodec0", "vcodec1" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0", "vcodec1" }, .vcodec_pmdomains_num = 3, .opp_pmdomain = (const char *[]) { "cx", NULL }, .vcodec_num = 2, @@ -754,7 +756,7 @@ static const struct venus_resources sc7180_res = { .clks_num = 3, .vcodec0_clks = { "vcodec0_core", "vcodec0_bus" }, .vcodec_clks_num = 2, - .vcodec_pmdomains = { "venus", "vcodec0" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx", NULL }, .vcodec_num = 1, @@ -811,7 +813,7 @@ static const struct venus_resources sm8250_res = { .resets_num = 2, .vcodec0_clks = { "vcodec0_core" }, .vcodec_clks_num = 1, - .vcodec_pmdomains = { "venus", "vcodec0" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "mx", NULL }, .vcodec_num = 1, @@ -870,7 +872,7 @@ static const struct venus_resources sc7280_res = { .clks_num = 3, .vcodec0_clks = {"vcodec_core", "vcodec_bus"}, .vcodec_clks_num = 2, - .vcodec_pmdomains = { "venus", "vcodec0" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx", NULL }, .vcodec_num = 1, diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h index 4a633261ece4..7ef341bf21cc 100644 --- a/drivers/media/platform/qcom/venus/core.h +++ b/drivers/media/platform/qcom/venus/core.h @@ -25,7 +25,6 @@ #define VIDC_CLKS_NUM_MAX 4 #define VIDC_VCODEC_CLKS_NUM_MAX 2 -#define VIDC_PMDOMAINS_NUM_MAX 3 #define VIDC_RESETS_NUM_MAX 2 extern int venus_fw_debug; @@ -72,7 +71,7 @@ struct venus_resources { const char * const vcodec0_clks[VIDC_VCODEC_CLKS_NUM_MAX]; const char * const vcodec1_clks[VIDC_VCODEC_CLKS_NUM_MAX]; unsigned int vcodec_clks_num; - const char * const vcodec_pmdomains[VIDC_PMDOMAINS_NUM_MAX]; + const char **vcodec_pmdomains; unsigned int vcodec_pmdomains_num; const char **opp_pmdomain; unsigned int vcodec_num; @@ -134,7 +133,7 @@ struct venus_format { * @video_path: an interconnect handle to video to/from memory path * @cpucfg_path: an interconnect handle to cpu configuration path * @has_opp_table: does OPP table exist - * @pmdomains: an array of pmdomains struct device pointers + * @pmdomains: a pointer to a list of pmdomains * @opp_dl_venus: an device-link for device OPP * @opp_pmdomain: an OPP power-domain * @resets: an array of reset signals @@ -187,7 +186,7 @@ struct venus_core { struct icc_path *video_path; struct icc_path *cpucfg_path; bool has_opp_table; - struct device *pmdomains[VIDC_PMDOMAINS_NUM_MAX]; + struct dev_pm_domain_list *pmdomains; struct device_link *opp_dl_venus; struct device *opp_pmdomain; struct reset_control *resets[VIDC_RESETS_NUM_MAX]; diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index a1b127caa90a..502822059498 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -455,7 +455,7 @@ static int poweroff_coreid(struct venus_core *core, unsigned int coreid_mask) if (ret) return ret; - ret = pm_runtime_put_sync(core->pmdomains[1]); + ret = pm_runtime_put_sync(core->pmdomains->pd_devs[1]); if (ret < 0) return ret; } @@ -471,7 +471,7 @@ static int poweroff_coreid(struct venus_core *core, unsigned int coreid_mask) if (ret) return ret; - ret = pm_runtime_put_sync(core->pmdomains[2]); + ret = pm_runtime_put_sync(core->pmdomains->pd_devs[2]); if (ret < 0) return ret; } @@ -484,7 +484,7 @@ static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask) int ret; if (coreid_mask & VIDC_CORE_ID_1) { - ret = pm_runtime_get_sync(core->pmdomains[1]); + ret = pm_runtime_get_sync(core->pmdomains->pd_devs[1]); if (ret < 0) return ret; @@ -502,7 +502,7 @@ static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask) } if (coreid_mask & VIDC_CORE_ID_2) { - ret = pm_runtime_get_sync(core->pmdomains[2]); + ret = pm_runtime_get_sync(core->pmdomains->pd_devs[2]); if (ret < 0) return ret; @@ -860,19 +860,18 @@ static int vcodec_domains_get(struct venus_core *core) struct device **opp_virt_dev; struct device *dev = core->dev; const struct venus_resources *res = core->res; - struct device *pd; - unsigned int i; + struct dev_pm_domain_attach_data vcodec_data = { + .pd_names = res->vcodec_pmdomains, + .num_pd_names = res->vcodec_pmdomains_num, + .pd_flags = PD_FLAG_NO_DEV_LINK, + }; if (!res->vcodec_pmdomains_num) goto skip_pmdomains; - for (i = 0; i < res->vcodec_pmdomains_num; i++) { - pd = dev_pm_domain_attach_by_name(dev, - res->vcodec_pmdomains[i]); - if (IS_ERR_OR_NULL(pd)) - return pd ? PTR_ERR(pd) : -ENODATA; - core->pmdomains[i] = pd; - } + ret = dev_pm_domain_attach_list(dev, &vcodec_data, &core->pmdomains); + if (ret < 0) + return ret; skip_pmdomains: if (!core->res->opp_pmdomain) @@ -896,30 +895,14 @@ skip_pmdomains: return 0; opp_attach_err: - for (i = 0; i < res->vcodec_pmdomains_num; i++) { - if (IS_ERR_OR_NULL(core->pmdomains[i])) - continue; - dev_pm_domain_detach(core->pmdomains[i], true); - } - + dev_pm_domain_detach_list(core->pmdomains); return ret; } static void vcodec_domains_put(struct venus_core *core) { - const struct venus_resources *res = core->res; - unsigned int i; + dev_pm_domain_detach_list(core->pmdomains); - if (!res->vcodec_pmdomains_num) - goto skip_pmdomains; - - for (i = 0; i < res->vcodec_pmdomains_num; i++) { - if (IS_ERR_OR_NULL(core->pmdomains[i])) - continue; - dev_pm_domain_detach(core->pmdomains[i], true); - } - -skip_pmdomains: if (!core->has_opp_table) return; @@ -1035,7 +1018,8 @@ static void core_put_v4(struct venus_core *core) static int core_power_v4(struct venus_core *core, int on) { struct device *dev = core->dev; - struct device *pmctrl = core->pmdomains[0]; + struct device *pmctrl = core->pmdomains ? + core->pmdomains->pd_devs[0] : NULL; int ret = 0; if (on == POWER_ON) { From 4d0824608a636b64373e52d3ef1516a86048e0e7 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:28:21 +0100 Subject: [PATCH 0334/1038] pmdomain: core: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202822.631449-1-krzysztof.kozlowski@linaro.org Signed-off-by: Ulf Hansson --- drivers/pmdomain/core.c | 4 ++-- drivers/pmdomain/imx/scu-pd.c | 2 +- drivers/pmdomain/tegra/powergate-bpmp.c | 2 +- drivers/pmdomain/ti/ti_sci_pm_domains.c | 2 +- drivers/pmdomain/xilinx/zynqmp-pm-domains.c | 2 +- include/linux/pm_domain.h | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 46331e71108a..ea4b29475206 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -2266,7 +2266,7 @@ static DEFINE_MUTEX(of_genpd_mutex); * to be a valid pointer to struct generic_pm_domain. */ static struct generic_pm_domain *genpd_xlate_simple( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { return data; @@ -2283,7 +2283,7 @@ static struct generic_pm_domain *genpd_xlate_simple( * the genpd_onecell_data struct when registering the provider. */ static struct generic_pm_domain *genpd_xlate_onecell( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; diff --git a/drivers/pmdomain/imx/scu-pd.c b/drivers/pmdomain/imx/scu-pd.c index 891c1d925a9d..05841b0bf7f3 100644 --- a/drivers/pmdomain/imx/scu-pd.c +++ b/drivers/pmdomain/imx/scu-pd.c @@ -393,7 +393,7 @@ static int imx_sc_pd_power_off(struct generic_pm_domain *domain) return imx_sc_pd_power(domain, false); } -static struct generic_pm_domain *imx_scu_pd_xlate(struct of_phandle_args *spec, +static struct generic_pm_domain *imx_scu_pd_xlate(const struct of_phandle_args *spec, void *data) { struct generic_pm_domain *domain = ERR_PTR(-ENOENT); diff --git a/drivers/pmdomain/tegra/powergate-bpmp.c b/drivers/pmdomain/tegra/powergate-bpmp.c index 179ed895c279..b0138ca9f851 100644 --- a/drivers/pmdomain/tegra/powergate-bpmp.c +++ b/drivers/pmdomain/tegra/powergate-bpmp.c @@ -305,7 +305,7 @@ static void tegra_bpmp_remove_powergates(struct tegra_bpmp *bpmp) } static struct generic_pm_domain * -tegra_powergate_xlate(struct of_phandle_args *spec, void *data) +tegra_powergate_xlate(const struct of_phandle_args *spec, void *data) { struct generic_pm_domain *domain = ERR_PTR(-ENOENT); struct genpd_onecell_data *genpd = data; diff --git a/drivers/pmdomain/ti/ti_sci_pm_domains.c b/drivers/pmdomain/ti/ti_sci_pm_domains.c index c091d569ecd5..9dddf227a3a6 100644 --- a/drivers/pmdomain/ti/ti_sci_pm_domains.c +++ b/drivers/pmdomain/ti/ti_sci_pm_domains.c @@ -85,7 +85,7 @@ static int ti_sci_pd_power_on(struct generic_pm_domain *domain) * @data: genpd core data for all the powerdomains on the device */ static struct generic_pm_domain *ti_sci_pd_xlate( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; diff --git a/drivers/pmdomain/xilinx/zynqmp-pm-domains.c b/drivers/pmdomain/xilinx/zynqmp-pm-domains.c index 6fd514286d82..0b5831e5ba1b 100644 --- a/drivers/pmdomain/xilinx/zynqmp-pm-domains.c +++ b/drivers/pmdomain/xilinx/zynqmp-pm-domains.c @@ -210,7 +210,7 @@ static void zynqmp_gpd_detach_dev(struct generic_pm_domain *domain, } static struct generic_pm_domain *zynqmp_gpd_xlate - (struct of_phandle_args *genpdspec, void *data) + (const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; unsigned int i, idx = genpdspec->args[0]; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index fb91770ba4ba..1a391ef1b6f8 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -349,7 +349,7 @@ static inline void dev_pm_genpd_resume(struct device *dev) {} /* OF PM domain providers */ struct of_device_id; -typedef struct generic_pm_domain *(*genpd_xlate_t)(struct of_phandle_args *args, +typedef struct generic_pm_domain *(*genpd_xlate_t)(const struct of_phandle_args *args, void *data); struct genpd_onecell_data { From 4af6bc163c4d841cf60bc9bf6e98603a232889b6 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 8 Feb 2024 21:28:22 +0100 Subject: [PATCH 0335/1038] pmdomain: core: constify of_phandle_args in add device and subdomain Pointer to of_phandle_args is not modified by of_genpd_add_device() and of_genpd_add_subdomain(), so it can be made pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240208202822.631449-2-krzysztof.kozlowski@linaro.org Signed-off-by: Ulf Hansson --- drivers/pmdomain/core.c | 12 ++++++------ include/linux/pm_domain.h | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index ea4b29475206..4215ffd9b11c 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -2526,7 +2526,7 @@ EXPORT_SYMBOL_GPL(of_genpd_del_provider); * on failure. */ static struct generic_pm_domain *genpd_get_from_provider( - struct of_phandle_args *genpdspec) + const struct of_phandle_args *genpdspec) { struct generic_pm_domain *genpd = ERR_PTR(-ENOENT); struct of_genpd_provider *provider; @@ -2557,7 +2557,7 @@ static struct generic_pm_domain *genpd_get_from_provider( * Looks-up an I/O PM domain based upon phandle args provided and adds * the device to the PM domain. Returns a negative error code on failure. */ -int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev) +int of_genpd_add_device(const struct of_phandle_args *genpdspec, struct device *dev) { struct generic_pm_domain *genpd; int ret; @@ -2591,8 +2591,8 @@ EXPORT_SYMBOL_GPL(of_genpd_add_device); * provided and adds the subdomain to the parent PM domain. Returns a * negative error code on failure. */ -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { struct generic_pm_domain *parent, *subdomain; int ret; @@ -2629,8 +2629,8 @@ EXPORT_SYMBOL_GPL(of_genpd_add_subdomain); * provided and removes the subdomain from the parent PM domain. Returns a * negative error code on failure. */ -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { struct generic_pm_domain *parent, *subdomain; int ret; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 1a391ef1b6f8..772d3280d35f 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -364,11 +364,11 @@ int of_genpd_add_provider_simple(struct device_node *np, int of_genpd_add_provider_onecell(struct device_node *np, struct genpd_onecell_data *data); void of_genpd_del_provider(struct device_node *np); -int of_genpd_add_device(struct of_phandle_args *args, struct device *dev); -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec); -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec); +int of_genpd_add_device(const struct of_phandle_args *args, struct device *dev); +int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec); +int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec); struct generic_pm_domain *of_genpd_remove_last(struct device_node *np); int of_genpd_parse_idle_states(struct device_node *dn, struct genpd_power_state **states, int *n); @@ -393,20 +393,20 @@ static inline int of_genpd_add_provider_onecell(struct device_node *np, static inline void of_genpd_del_provider(struct device_node *np) {} -static inline int of_genpd_add_device(struct of_phandle_args *args, +static inline int of_genpd_add_device(const struct of_phandle_args *args, struct device *dev) { return -ENODEV; } -static inline int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +static inline int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { return -ENODEV; } -static inline int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +static inline int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { return -ENODEV; } From 7547f9a027e37b7d5057492ac7907ac77c155618 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sat, 10 Feb 2024 17:38:57 +0100 Subject: [PATCH 0336/1038] pmdomain: qcom: rpmpd: Add MSM8974+PM8841 power domains Add the power domains CX & GFX found on devices with MSM8974 and PM8841. Signed-off-by: Luca Weiss Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240210-msm8974-rpmpd-v2-2-595e2ff80ea1@z3ntu.xyz Signed-off-by: Ulf Hansson --- drivers/pmdomain/qcom/rpmpd.c | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/drivers/pmdomain/qcom/rpmpd.c b/drivers/pmdomain/qcom/rpmpd.c index 90b62767f9d0..b7cb6a8d2331 100644 --- a/drivers/pmdomain/qcom/rpmpd.c +++ b/drivers/pmdomain/qcom/rpmpd.c @@ -227,6 +227,31 @@ static struct rpmpd cx_s3a_vfl = { .key = KEY_FLOOR_LEVEL, }; +static struct rpmpd cx_s2b_corner_ao; +static struct rpmpd cx_s2b_corner = { + .pd = { .name = "cx", }, + .peer = &cx_s2b_corner_ao, + .res_type = RPMPD_SMPB, + .res_id = 2, + .key = KEY_CORNER, +}; + +static struct rpmpd cx_s2b_corner_ao = { + .pd = { .name = "cx_ao", }, + .peer = &cx_s2b_corner, + .active_only = true, + .res_type = RPMPD_SMPB, + .res_id = 2, + .key = KEY_CORNER, +}; + +static struct rpmpd cx_s2b_vfc = { + .pd = { .name = "cx_vfc", }, + .res_type = RPMPD_SMPB, + .res_id = 2, + .key = KEY_FLOOR_CORNER, +}; + /* G(F)X */ static struct rpmpd gfx_s2b_corner = { .pd = { .name = "gfx", }, @@ -242,6 +267,20 @@ static struct rpmpd gfx_s2b_vfc = { .key = KEY_FLOOR_CORNER, }; +static struct rpmpd gfx_s4b_corner = { + .pd = { .name = "gfx", }, + .res_type = RPMPD_SMPB, + .res_id = 4, + .key = KEY_CORNER, +}; + +static struct rpmpd gfx_s4b_vfc = { + .pd = { .name = "gfx_vfc", }, + .res_type = RPMPD_SMPB, + .res_id = 4, + .key = KEY_FLOOR_CORNER, +}; + static struct rpmpd mx_rwmx0_lvl; static struct rpmpd gx_rwgx0_lvl_ao; static struct rpmpd gx_rwgx0_lvl = { @@ -664,6 +703,20 @@ static const struct rpmpd_desc msm8953_desc = { .max_state = RPM_SMD_LEVEL_TURBO, }; +static struct rpmpd *msm8974_rpmpds[] = { + [MSM8974_VDDCX] = &cx_s2b_corner, + [MSM8974_VDDCX_AO] = &cx_s2b_corner_ao, + [MSM8974_VDDCX_VFC] = &cx_s2b_vfc, + [MSM8974_VDDGFX] = &gfx_s4b_corner, + [MSM8974_VDDGFX_VFC] = &gfx_s4b_vfc, +}; + +static const struct rpmpd_desc msm8974_desc = { + .rpmpds = msm8974_rpmpds, + .num_pds = ARRAY_SIZE(msm8974_rpmpds), + .max_state = MAX_CORNER_RPMPD_STATE, +}; + static struct rpmpd *msm8976_rpmpds[] = { [MSM8976_VDDCX] = &cx_s2a_lvl, [MSM8976_VDDCX_AO] = &cx_s2a_lvl_ao, @@ -857,6 +910,7 @@ static const struct of_device_id rpmpd_match_table[] = { { .compatible = "qcom,msm8917-rpmpd", .data = &msm8917_desc }, { .compatible = "qcom,msm8939-rpmpd", .data = &msm8939_desc }, { .compatible = "qcom,msm8953-rpmpd", .data = &msm8953_desc }, + { .compatible = "qcom,msm8974-rpmpd", .data = &msm8974_desc }, { .compatible = "qcom,msm8976-rpmpd", .data = &msm8976_desc }, { .compatible = "qcom,msm8994-rpmpd", .data = &msm8994_desc }, { .compatible = "qcom,msm8996-rpmpd", .data = &msm8996_desc }, From 5bf85865eaaa21d453f75d479387743697858a26 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sat, 10 Feb 2024 17:38:58 +0100 Subject: [PATCH 0337/1038] pmdomain: qcom: rpmpd: Add MSM8974PRO+PMA8084 power domains Add the power domains CX & GFX found on MSM8974 devices that use PMA8084 instead of the standard PM8841+PM8941 combo. Signed-off-by: Luca Weiss Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240210-msm8974-rpmpd-v2-3-595e2ff80ea1@z3ntu.xyz Signed-off-by: Ulf Hansson --- drivers/pmdomain/qcom/rpmpd.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/pmdomain/qcom/rpmpd.c b/drivers/pmdomain/qcom/rpmpd.c index b7cb6a8d2331..5e6280b4cf70 100644 --- a/drivers/pmdomain/qcom/rpmpd.c +++ b/drivers/pmdomain/qcom/rpmpd.c @@ -253,6 +253,20 @@ static struct rpmpd cx_s2b_vfc = { }; /* G(F)X */ +static struct rpmpd gfx_s7a_corner = { + .pd = { .name = "gfx", }, + .res_type = RPMPD_SMPA, + .res_id = 7, + .key = KEY_CORNER, +}; + +static struct rpmpd gfx_s7a_vfc = { + .pd = { .name = "gfx_vfc", }, + .res_type = RPMPD_SMPA, + .res_id = 7, + .key = KEY_FLOOR_CORNER, +}; + static struct rpmpd gfx_s2b_corner = { .pd = { .name = "gfx", }, .res_type = RPMPD_SMPB, @@ -717,6 +731,20 @@ static const struct rpmpd_desc msm8974_desc = { .max_state = MAX_CORNER_RPMPD_STATE, }; +static struct rpmpd *msm8974pro_pma8084_rpmpds[] = { + [MSM8974_VDDCX] = &cx_s2a_corner, + [MSM8974_VDDCX_AO] = &cx_s2a_corner_ao, + [MSM8974_VDDCX_VFC] = &cx_s2a_vfc, + [MSM8974_VDDGFX] = &gfx_s7a_corner, + [MSM8974_VDDGFX_VFC] = &gfx_s7a_vfc, +}; + +static const struct rpmpd_desc msm8974pro_pma8084_desc = { + .rpmpds = msm8974pro_pma8084_rpmpds, + .num_pds = ARRAY_SIZE(msm8974pro_pma8084_rpmpds), + .max_state = MAX_CORNER_RPMPD_STATE, +}; + static struct rpmpd *msm8976_rpmpds[] = { [MSM8976_VDDCX] = &cx_s2a_lvl, [MSM8976_VDDCX_AO] = &cx_s2a_lvl_ao, @@ -911,6 +939,7 @@ static const struct of_device_id rpmpd_match_table[] = { { .compatible = "qcom,msm8939-rpmpd", .data = &msm8939_desc }, { .compatible = "qcom,msm8953-rpmpd", .data = &msm8953_desc }, { .compatible = "qcom,msm8974-rpmpd", .data = &msm8974_desc }, + { .compatible = "qcom,msm8974pro-pma8084-rpmpd", .data = &msm8974pro_pma8084_desc }, { .compatible = "qcom,msm8976-rpmpd", .data = &msm8976_desc }, { .compatible = "qcom,msm8994-rpmpd", .data = &msm8994_desc }, { .compatible = "qcom,msm8996-rpmpd", .data = &msm8996_desc }, From 713240877a26f3cc035d6531795bd819dfaa633c Mon Sep 17 00:00:00 2001 From: Dien Pham Date: Tue, 13 Feb 2024 00:26:11 +0000 Subject: [PATCH 0338/1038] pmdomain: renesas: Adjust the waiting time to cover the worst case Description in HWM rev0.51E, 9.4 Usage notes, page 455 tells "It takes several hundreds of microseconds to shutting off and resuming power domain. Because actual time required for shutting off and resuming depends on the status of on-board power line, shutoff/resume time is not guaranteed by electrical specification" Let's assume the safe value of waiting is about 1000us. Signed-off-by: Dien Pham Signed-off-by: Takeshi Kihara Signed-off-by: Tho Vu Signed-off-by: Kuninori Morimoto Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/8734tx8b18.wl-kuninori.morimoto.gx@renesas.com Signed-off-by: Ulf Hansson --- drivers/pmdomain/renesas/rcar-sysc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pmdomain/renesas/rcar-sysc.c b/drivers/pmdomain/renesas/rcar-sysc.c index eed47696e825..35d9aa0dfab8 100644 --- a/drivers/pmdomain/renesas/rcar-sysc.c +++ b/drivers/pmdomain/renesas/rcar-sysc.c @@ -45,10 +45,10 @@ #define PWRER_OFFS 0x14 /* Power Shutoff/Resume Error */ -#define SYSCSR_TIMEOUT 100 +#define SYSCSR_TIMEOUT 1000 #define SYSCSR_DELAY_US 1 -#define PWRER_RETRIES 100 +#define PWRER_RETRIES 1000 #define PWRER_DELAY_US 1 #define SYSCISR_TIMEOUT 1000 From 9e69d6d84e3841df073601440774a2221781fc1e Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Mon, 12 Feb 2024 17:34:20 +0800 Subject: [PATCH 0339/1038] Documentation: gpio: clarify sysfs line values are logical Clarify that line values are logical, not physical, by replacing high/low terminology with active/inactive. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- Documentation/userspace-api/gpio/sysfs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/userspace-api/gpio/sysfs.rst b/Documentation/userspace-api/gpio/sysfs.rst index e12037a0f2b4..116921048b18 100644 --- a/Documentation/userspace-api/gpio/sysfs.rst +++ b/Documentation/userspace-api/gpio/sysfs.rst @@ -87,9 +87,9 @@ and have the following read/write attributes: allow userspace to reconfigure this GPIO's direction. "value" ... - reads as either 0 (low) or 1 (high). If the GPIO + reads as either 0 (inactive) or 1 (active). If the GPIO is configured as an output, this value may be written; - any nonzero value is treated as high. + any nonzero value is treated as active. If the pin can be configured as interrupt-generating interrupt and if it has been configured to generate interrupts (see the From e3f927f2b0a23902923c768f07e274a1373216f3 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 13 Feb 2024 22:12:22 +0800 Subject: [PATCH 0340/1038] Documentation: gpio: consistently use logical line value terminology Consistently use active/inactive to describe logical line values, rather than high/low, which is used for physical values, or asserted/de-asserted which is awkward. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Reviewed-by: Linus Walleij [Bartosz: tweaked the commit subject to use imperative mood] Signed-off-by: Bartosz Golaszewski --- Documentation/driver-api/gpio/consumer.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/driver-api/gpio/consumer.rst b/Documentation/driver-api/gpio/consumer.rst index 3e588b9d678c..ab56ab0dd7a6 100644 --- a/Documentation/driver-api/gpio/consumer.rst +++ b/Documentation/driver-api/gpio/consumer.rst @@ -222,9 +222,9 @@ Use the following calls to access GPIOs from an atomic context:: int gpiod_get_value(const struct gpio_desc *desc); void gpiod_set_value(struct gpio_desc *desc, int value); -The values are boolean, zero for low, nonzero for high. When reading the value -of an output pin, the value returned should be what's seen on the pin. That -won't always match the specified output value, because of issues including +The values are boolean, zero for inactive, nonzero for active. When reading the +value of an output pin, the value returned should be what's seen on the pin. +That won't always match the specified output value, because of issues including open-drain signaling and output latencies. The get/set calls do not return errors because "invalid GPIO" should have been @@ -277,11 +277,11 @@ switch their output to a high impedance value. The consumer should not need to care. (For details read about open drain in driver.rst.) With this, all the gpiod_set_(array)_value_xxx() functions interpret the -parameter "value" as "asserted" ("1") or "de-asserted" ("0"). The physical line +parameter "value" as "active" ("1") or "inactive" ("0"). The physical line level will be driven accordingly. As an example, if the active low property for a dedicated GPIO is set, and the -gpiod_set_(array)_value_xxx() passes "asserted" ("1"), the physical line level +gpiod_set_(array)_value_xxx() passes "active" ("1"), the physical line level will be driven low. To summarize:: From 032b149bcc54759b90c1851b0e13dff5e8c6f349 Mon Sep 17 00:00:00 2001 From: Richard Acayan Date: Fri, 9 Feb 2024 18:19:15 -0500 Subject: [PATCH 0341/1038] cpufreq: dt-platdev: block SDM670 in cpufreq-dt-platdev The Snapdragon 670 uses the Qualcomm driver for CPU frequency scaling. Block this driver from loading on it so the driver does not pollute dmesg with an error. Signed-off-by: Richard Acayan Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq-dt-platdev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index bd1e1357cef8..b993a498084b 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -156,6 +156,7 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sc7280", }, { .compatible = "qcom,sc8180x", }, { .compatible = "qcom,sc8280xp", }, + { .compatible = "qcom,sdm670", }, { .compatible = "qcom,sdm845", }, { .compatible = "qcom,sdx75", }, { .compatible = "qcom,sm6115", }, From 815a1b5a6da4bedb29a1e15a94a042e525b0ba96 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 14 Feb 2024 09:44:16 +0100 Subject: [PATCH 0342/1038] gpio: take the SRCU read lock in gpiod_hog() gpiod_hog() may be called without the gpio_device SRCU read lock taken so we need to do it here as well. It's alright if someone else is already holding the lock as SRCU read critical sections can be nested. Fixes: d83cee3d2bb1 ("gpio: protect the pointer to gpio_chip in gpio_device with SRCU") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202402122234.d85cca9b-lkp@intel.com Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Paul E. McKenney --- drivers/gpio/gpiolib.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index f5434e559382..439d32d5aa38 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4492,24 +4492,27 @@ EXPORT_SYMBOL_GPL(gpiod_get_index_optional); int gpiod_hog(struct gpio_desc *desc, const char *name, unsigned long lflags, enum gpiod_flags dflags) { - struct gpio_chip *gc; + struct gpio_device *gdev = desc->gdev; struct gpio_desc *local_desc; int hwnum; int ret; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + if (test_and_set_bit(FLAG_IS_HOGGED, &desc->flags)) return 0; - gc = gpiod_to_chip(desc); hwnum = gpio_chip_hwgpio(desc); - local_desc = gpiochip_request_own_desc(gc, hwnum, name, + local_desc = gpiochip_request_own_desc(guard.gc, hwnum, name, lflags, dflags); if (IS_ERR(local_desc)) { clear_bit(FLAG_IS_HOGGED, &desc->flags); ret = PTR_ERR(local_desc); pr_err("requesting hog GPIO %s (chip %s, offset %d) failed, %d\n", - name, gc->label, hwnum, ret); + name, gdev->label, hwnum, ret); return ret; } From 8574b5b47610df22048adcdabf318ca983024f28 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 14 Feb 2024 09:44:17 +0100 Subject: [PATCH 0343/1038] gpio: cdev: use correct pointer accessors with SRCU We never dereference the chip pointer in character device code so we can use the lighter rcu_access_pointer() helper. This also makes lockep happier as it no longer complains about suspicious rcu_dereference() usage. Fixes: d83cee3d2bb1 ("gpio: protect the pointer to gpio_chip in gpio_device with SRCU") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202402122234.d85cca9b-lkp@intel.com Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij Acked-by: Paul E. McKenney --- drivers/gpio/gpiolib-cdev.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 9323b357df43..85037fa4925e 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -206,7 +206,7 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, guard(srcu)(&lh->gdev->srcu); - if (!rcu_dereference(lh->gdev->chip)) + if (!rcu_access_pointer(lh->gdev->chip)) return -ENODEV; switch (cmd) { @@ -1521,7 +1521,7 @@ static long linereq_ioctl(struct file *file, unsigned int cmd, guard(srcu)(&lr->gdev->srcu); - if (!rcu_dereference(lr->gdev->chip)) + if (!rcu_access_pointer(lr->gdev->chip)) return -ENODEV; switch (cmd) { @@ -1552,7 +1552,7 @@ static __poll_t linereq_poll(struct file *file, guard(srcu)(&lr->gdev->srcu); - if (!rcu_dereference(lr->gdev->chip)) + if (!rcu_access_pointer(lr->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &lr->wait, wait); @@ -1574,7 +1574,7 @@ static ssize_t linereq_read(struct file *file, char __user *buf, guard(srcu)(&lr->gdev->srcu); - if (!rcu_dereference(lr->gdev->chip)) + if (!rcu_access_pointer(lr->gdev->chip)) return -ENODEV; if (count < sizeof(le)) @@ -1875,7 +1875,7 @@ static __poll_t lineevent_poll(struct file *file, guard(srcu)(&le->gdev->srcu); - if (!rcu_dereference(le->gdev->chip)) + if (!rcu_access_pointer(le->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &le->wait, wait); @@ -1913,7 +1913,7 @@ static ssize_t lineevent_read(struct file *file, char __user *buf, guard(srcu)(&le->gdev->srcu); - if (!rcu_dereference(le->gdev->chip)) + if (!rcu_access_pointer(le->gdev->chip)) return -ENODEV; /* @@ -1996,7 +1996,7 @@ static long lineevent_ioctl(struct file *file, unsigned int cmd, guard(srcu)(&le->gdev->srcu); - if (!rcu_dereference(le->gdev->chip)) + if (!rcu_access_pointer(le->gdev->chip)) return -ENODEV; /* @@ -2510,7 +2510,7 @@ static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) guard(srcu)(&gdev->srcu); /* We fail any subsequent ioctl():s when the chip is gone */ - if (!rcu_dereference(gdev->chip)) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; /* Fill in the struct and pass to userspace */ @@ -2595,7 +2595,7 @@ static __poll_t lineinfo_watch_poll(struct file *file, guard(srcu)(&cdev->gdev->srcu); - if (!rcu_dereference(cdev->gdev->chip)) + if (!rcu_access_pointer(cdev->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &cdev->wait, pollt); @@ -2618,7 +2618,7 @@ static ssize_t lineinfo_watch_read(struct file *file, char __user *buf, guard(srcu)(&cdev->gdev->srcu); - if (!rcu_dereference(cdev->gdev->chip)) + if (!rcu_access_pointer(cdev->gdev->chip)) return -ENODEV; #ifndef CONFIG_GPIO_CDEV_V1 @@ -2696,7 +2696,7 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file) guard(srcu)(&gdev->srcu); /* Fail on open if the backing gpiochip is gone */ - if (!rcu_dereference(gdev->chip)) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); @@ -2796,8 +2796,7 @@ int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); - if (!gc) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; chip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); From d82b9e0887e69d9060c854b079a3a5024788f7cb Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 14 Feb 2024 09:44:18 +0100 Subject: [PATCH 0344/1038] gpio: use srcu_dereference() with SRCU-protected pointers Lockdep with CONFIG_PROVE_RCU enabled reports false positives about suspicious rcu_dereference() usage. Let's silence it by using srcu_dereference() which is the correct helper with SRCU. Fixes: d83cee3d2bb1 ("gpio: protect the pointer to gpio_chip in gpio_device with SRCU") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202402122234.d85cca9b-lkp@intel.com Signed-off-by: Bartosz Golaszewski Acked-by: Paul E. McKenney --- drivers/gpio/gpiolib-sysfs.c | 5 +++-- drivers/gpio/gpiolib.c | 16 ++++++++-------- drivers/gpio/gpiolib.h | 3 ++- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 6285fa5afbb1..71ba2a774197 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -756,7 +757,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) guard(srcu)(&gdev->srcu); - chip = rcu_dereference(gdev->chip); + chip = srcu_dereference(gdev->chip, &gdev->srcu); if (!chip) return -ENODEV; @@ -800,7 +801,7 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) guard(srcu)(&gdev->srcu); - chip = rcu_dereference(gdev->chip); + chip = srcu_dereference(gdev->chip, &gdev->srcu); if (chip) return; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 439d32d5aa38..b095f475805f 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -109,7 +109,7 @@ const char *gpiod_get_label(struct gpio_desc *desc) return "interrupt"; return test_bit(FLAG_REQUESTED, &flags) ? - rcu_dereference(desc->label) : NULL; + srcu_dereference(desc->label, &desc->srcu) : NULL; } static int desc_set_label(struct gpio_desc *desc, const char *label) @@ -447,7 +447,7 @@ static struct gpio_desc *gpio_name_to_desc(const char * const name) srcu_read_lock_held(&gpio_devices_srcu)) { guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (!gc) continue; @@ -1190,7 +1190,7 @@ struct gpio_device *gpio_device_find(void *data, srcu_read_lock_held(&gpio_devices_srcu)) { guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (gc && match(gc, data)) return gpio_device_get(gdev); @@ -2978,7 +2978,7 @@ static int gpiod_get_raw_value_commit(const struct gpio_desc *desc) guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (!gc) return -ENODEV; @@ -3012,7 +3012,7 @@ static bool gpio_device_chip_cmp(struct gpio_device *gdev, struct gpio_chip *gc) { guard(srcu)(&gdev->srcu); - return gc == rcu_dereference(gdev->chip); + return gc == srcu_dereference(gdev->chip, &gdev->srcu); } int gpiod_get_array_value_complex(bool raw, bool can_sleep, @@ -3593,7 +3593,7 @@ int gpiod_to_irq(const struct gpio_desc *desc) gdev = desc->gdev; /* FIXME Cannot use gpio_chip_guard due to const desc. */ guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (!gc) return -ENODEV; @@ -4787,7 +4787,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (!gc) { seq_puts(s, "Underlying GPIO chip is gone\n"); return; @@ -4872,7 +4872,7 @@ static int gpiolib_seq_show(struct seq_file *s, void *v) guard(srcu)(&gdev->srcu); - gc = rcu_dereference(gdev->chip); + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (!gc) { seq_printf(s, "%s%s: (dangling chip)", priv->newline ? "\n" : "", diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 07443d26cbca..ada36aa0f81a 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -202,7 +202,8 @@ DEFINE_CLASS(gpio_chip_guard, _guard.gdev = desc->gdev; _guard.idx = srcu_read_lock(&_guard.gdev->srcu); - _guard.gc = rcu_dereference(_guard.gdev->chip); + _guard.gc = srcu_dereference(_guard.gdev->chip, + &_guard.gdev->srcu); _guard; }), From 0d7fa0eda4e3ffe1b7ebc73b6ef81298bddda649 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 14 Feb 2024 09:44:19 +0100 Subject: [PATCH 0345/1038] gpio: don't let lockdep complain about inherently dangerous RCU usage There are two legacy, deprecated functions - gpiod_to_chip() and gpio_device_get_chip() - that still have users in tree. They return the address of the SRCU-protected chip outside of the read-only critical sections. They are inherently dangerous and the users should convert to safer alternatives. Let's explicitly silence lockdep warnings by using rcu_dereference_check(ptr, 1). While at it: reuse gpio_device_get_chip() in gpiod_to_chip(). Fixes: d83cee3d2bb1 ("gpio: protect the pointer to gpio_chip in gpio_device with SRCU") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202402122234.d85cca9b-lkp@intel.com Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index b095f475805f..02be0ba1a402 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -221,7 +221,8 @@ struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc) { if (!desc) return NULL; - return rcu_dereference(desc->gdev->chip); + + return gpio_device_get_chip(desc->gdev); } EXPORT_SYMBOL_GPL(gpiod_to_chip); @@ -291,7 +292,7 @@ EXPORT_SYMBOL(gpio_device_get_label); */ struct gpio_chip *gpio_device_get_chip(struct gpio_device *gdev) { - return rcu_dereference(gdev->chip); + return rcu_dereference_check(gdev->chip, 1); } EXPORT_SYMBOL_GPL(gpio_device_get_chip); From be91c19e47d1b9bf1ebd7ec4a859a50a53e54882 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 14 Feb 2024 09:52:48 +0100 Subject: [PATCH 0346/1038] gpio: sysfs: fix inverted pointer logic The logic is inverted, we want to return if the chip *IS* NULL. Fixes: d83cee3d2bb1 ("gpio: protect the pointer to gpio_chip in gpio_device with SRCU") Reported-by: Dan Carpenter Closes: https://lore.kernel.org/linux-gpio/15671341-0b29-40e0-b487-0a4cdc414d8e@moroto.mountain/ Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 71ba2a774197..67fc09a57f26 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -802,7 +802,7 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) guard(srcu)(&gdev->srcu); chip = srcu_dereference(gdev->chip, &gdev->srcu); - if (chip) + if (!chip) return; /* unregister gpiod class devices owned by sysfs */ From 8ff0d55ba3fb56b69e40c50ff030a77c27a072ca Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Sun, 11 Feb 2024 18:14:21 +0800 Subject: [PATCH 0347/1038] gpio: uapi: clarify default_values being logical The documentation for default_values mentions high/low which can be confusing, particularly when the ACTIVE_LOW flag is set. Replace high/low with active/inactive to clarify that the values are logical not physical. Similarly, clarify the interpretation of values in struct gpiohandle_data. Signed-off-by: Kent Gibson Reviewed-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- include/uapi/linux/gpio.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index 103cd3c6c81e..f7cb8ae87df7 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -399,8 +399,8 @@ struct gpioline_info_changed { * a batch of input or output lines, but they must all have the same * characteristics, i.e. all inputs or all outputs, all active low etc * @default_values: if the %GPIOHANDLE_REQUEST_OUTPUT is set for a requested - * line, this specifies the default output value, should be 0 (low) or - * 1 (high), anything else than 0 or 1 will be interpreted as 1 (high) + * line, this specifies the default output value, should be 0 (inactive) or + * 1 (active). Anything other than 0 or 1 will be interpreted as active. * @consumer_label: a desired consumer label for the selected GPIO line(s) * such as "my-bitbanged-relay" * @lines: number of lines requested in this request, i.e. the number of @@ -426,8 +426,8 @@ struct gpiohandle_request { * %GPIOHANDLE_REQUEST_OUTPUT, %GPIOHANDLE_REQUEST_ACTIVE_LOW etc, added * together * @default_values: if the %GPIOHANDLE_REQUEST_OUTPUT is set in flags, - * this specifies the default output value, should be 0 (low) or - * 1 (high), anything else than 0 or 1 will be interpreted as 1 (high) + * this specifies the default output value, should be 0 (inactive) or + * 1 (active). Anything other than 0 or 1 will be interpreted as active. * @padding: reserved for future use and should be zero filled * * Note: This struct is part of ABI v1 and is deprecated. @@ -443,7 +443,8 @@ struct gpiohandle_config { * struct gpiohandle_data - Information of values on a GPIO handle * @values: when getting the state of lines this contains the current * state of a line, when setting the state of lines these should contain - * the desired target state + * the desired target state. States are 0 (inactive) or 1 (active). + * When setting, anything other than 0 or 1 will be interpreted as active. * * Note: This struct is part of ABI v1 and is deprecated. * Use ABI v2 and &struct gpio_v2_line_values instead. From 3093fa33539b54db77171d2919352ad4f044a1c5 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 15 Feb 2024 09:33:14 +0100 Subject: [PATCH 0348/1038] cpufreq: qcom-hw: add CONFIG_COMMON_CLK dependency It is still possible to compile-test a kernel without CONFIG_COMMON_CLK for some ancient ARM boards or other architectures, but this causes a link failure in the qcom-cpufreq-hw driver: ERROR: modpost: "devm_clk_hw_register" [drivers/cpufreq/qcom-cpufreq-hw.ko] undefined! ERROR: modpost: "devm_of_clk_add_hw_provider" [drivers/cpufreq/qcom-cpufreq-hw.ko] undefined! ERROR: modpost: "of_clk_hw_onecell_get" [drivers/cpufreq/qcom-cpufreq-hw.ko] undefined! Add a Kconfig dependency here to make sure this always work. Apparently this bug has been in the kernel for a while without me running into it on randconfig builds as COMMON_CLK is almost always enabled. I have cross-checked by building an allmodconfig kernel with COMMON_CLK disabled, which showed no other driver having this problem. Fixes: 4370232c727b ("cpufreq: qcom-hw: Add CPU clock provider support") Signed-off-by: Arnd Bergmann Signed-off-by: Viresh Kumar --- drivers/cpufreq/Kconfig.arm | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index f911606897b8..a0ebad77666e 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -173,6 +173,7 @@ config ARM_QCOM_CPUFREQ_NVMEM config ARM_QCOM_CPUFREQ_HW tristate "QCOM CPUFreq HW driver" depends on ARCH_QCOM || COMPILE_TEST + depends on COMMON_CLK help Support for the CPUFreq HW driver. Some QCOM chipsets have a HW engine to offload the steps From 7cfce2b80d5ff7aa713a7710bfe3a562707cb226 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 8 Feb 2024 12:35:29 +0530 Subject: [PATCH 0349/1038] pwm: dwc: use pm_sleep_ptr() macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we don't have runtime PM handles here, we should be using pm_sleep_ptr() macro, so that the compiler can discard it in case CONFIG_PM_SLEEP=n. Fixes: 30b5b066fa83 ("pwm: dwc: Use DEFINE_SIMPLE_DEV_PM_OPS for PM functions") Signed-off-by: Raag Jadav Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240212130247.9985-2-raag.jadav@intel.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index 4929354f8cd9..a4a057ae03ea 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -120,7 +120,7 @@ static struct pci_driver dwc_pwm_driver = { .remove = dwc_pwm_remove, .id_table = dwc_pwm_id_table, .driver = { - .pm = pm_ptr(&dwc_pwm_pm_ops), + .pm = pm_sleep_ptr(&dwc_pwm_pm_ops), }, }; From 4e77431cda4973f03d063c47f6ea313dfceebf16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 9 Jan 2024 22:34:32 +0100 Subject: [PATCH 0350/1038] pwm: Drop useless member .of_pwm_n_cells of struct pwm_chip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apart from the two of_xlate implementations this member is write-only. In the of_xlate functions of_pwm_xlate_with_flags() and of_pwm_single_xlate() it's more sensible to check for args->args_count because this is what is actually used in the device tree. Acked-by: Douglas Anderson Link: https://lore.kernel.org/r/53d8c545aa8f79a920358be9e72e382b3981bdc4.1704835845.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/gpu/drm/bridge/ti-sn65dsi86.c | 1 - drivers/pwm/core.c | 22 +++------------------- drivers/pwm/pwm-clps711x.c | 1 - drivers/pwm/pwm-cros-ec.c | 1 - drivers/pwm/pwm-pxa.c | 4 +--- include/linux/pwm.h | 2 -- 6 files changed, 4 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index 62cc3893dca5..1f6e929c2f6a 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -1591,7 +1591,6 @@ static int ti_sn_pwm_probe(struct auxiliary_device *adev, pdata->pchip.ops = &ti_sn_pwm_ops; pdata->pchip.npwm = 1; pdata->pchip.of_xlate = of_pwm_single_xlate; - pdata->pchip.of_pwm_n_cells = 1; devm_pm_runtime_enable(&adev->dev); diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index f2728ee787d7..31f210872a07 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -107,9 +107,6 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg { struct pwm_device *pwm; - if (chip->of_pwm_n_cells < 2) - return ERR_PTR(-EINVAL); - /* flags in the third cell are optional */ if (args->args_count < 2) return ERR_PTR(-EINVAL); @@ -124,10 +121,8 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg pwm->args.period = args->args[1]; pwm->args.polarity = PWM_POLARITY_NORMAL; - if (chip->of_pwm_n_cells >= 3) { - if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) - pwm->args.polarity = PWM_POLARITY_INVERSED; - } + if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; return pwm; } @@ -138,9 +133,6 @@ of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) { struct pwm_device *pwm; - if (chip->of_pwm_n_cells < 1) - return ERR_PTR(-EINVAL); - /* validate that one cell is specified, optionally with flags */ if (args->args_count != 1 && args->args_count != 2) return ERR_PTR(-EINVAL); @@ -164,16 +156,8 @@ static void of_pwmchip_add(struct pwm_chip *chip) if (!chip->dev || !chip->dev->of_node) return; - if (!chip->of_xlate) { - u32 pwm_cells; - - if (of_property_read_u32(chip->dev->of_node, "#pwm-cells", - &pwm_cells)) - pwm_cells = 2; - + if (!chip->of_xlate) chip->of_xlate = of_pwm_xlate_with_flags; - chip->of_pwm_n_cells = pwm_cells; - } of_node_get(chip->dev->of_node); } diff --git a/drivers/pwm/pwm-clps711x.c b/drivers/pwm/pwm-clps711x.c index 42179b3f7ec3..06562d4bb963 100644 --- a/drivers/pwm/pwm-clps711x.c +++ b/drivers/pwm/pwm-clps711x.c @@ -103,7 +103,6 @@ static int clps711x_pwm_probe(struct platform_device *pdev) priv->chip.dev = &pdev->dev; priv->chip.npwm = 2; priv->chip.of_xlate = clps711x_pwm_xlate; - priv->chip.of_pwm_n_cells = 1; spin_lock_init(&priv->lock); diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 5fe303b8656d..339cedf3a7b1 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -279,7 +279,6 @@ static int cros_ec_pwm_probe(struct platform_device *pdev) chip->dev = dev; chip->ops = &cros_ec_pwm_ops; chip->of_xlate = cros_ec_pwm_xlate; - chip->of_pwm_n_cells = 1; if (ec_pwm->use_pwm_type) { chip->npwm = CROS_EC_PWM_DT_COUNT; diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c index 76685f926c75..61b74fa1d348 100644 --- a/drivers/pwm/pwm-pxa.c +++ b/drivers/pwm/pwm-pxa.c @@ -180,10 +180,8 @@ static int pwm_probe(struct platform_device *pdev) pc->chip.ops = &pxa_pwm_ops; pc->chip.npwm = (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1; - if (IS_ENABLED(CONFIG_OF)) { + if (IS_ENABLED(CONFIG_OF)) pc->chip.of_xlate = of_pwm_single_xlate; - pc->chip.of_pwm_n_cells = 1; - } pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) diff --git a/include/linux/pwm.h b/include/linux/pwm.h index fcc2c4496f73..8ffe9ae7a23a 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -271,7 +271,6 @@ struct pwm_ops { * @id: unique number of this PWM chip * @npwm: number of PWMs controlled by this chip * @of_xlate: request a PWM device given a device tree PWM specifier - * @of_pwm_n_cells: number of cells expected in the device tree PWM specifier * @atomic: can the driver's ->apply() be called in atomic context * @pwms: array of PWM devices allocated by the framework */ @@ -284,7 +283,6 @@ struct pwm_chip { struct pwm_device * (*of_xlate)(struct pwm_chip *chip, const struct of_phandle_args *args); - unsigned int of_pwm_n_cells; bool atomic; /* only used internally by the PWM framework */ From 40ade0c2e7940becad70a0643ba90488b905b468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 9 Jan 2024 22:34:33 +0100 Subject: [PATCH 0351/1038] pwm: Let the of_xlate callbacks accept references without period MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this extension of_pwm_xlate_with_flags() is suitable to replace the custom xlate function of the pwm-clps711x driver. While touching these very similar functions align their implementations. Link: https://lore.kernel.org/r/127622315d07d9d419ae8e6373c7e5be7fab7a62.1704835845.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/core.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 31f210872a07..606d9ef0c709 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -107,8 +107,8 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg { struct pwm_device *pwm; - /* flags in the third cell are optional */ - if (args->args_count < 2) + /* period in the second cell and flags in the third cell are optional */ + if (args->args_count < 1) return ERR_PTR(-EINVAL); if (args->args[0] >= chip->npwm) @@ -118,9 +118,10 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg if (IS_ERR(pwm)) return pwm; - pwm->args.period = args->args[1]; - pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 1) + pwm->args.period = args->args[1]; + pwm->args.polarity = PWM_POLARITY_NORMAL; if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) pwm->args.polarity = PWM_POLARITY_INVERSED; @@ -133,18 +134,15 @@ of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) { struct pwm_device *pwm; - /* validate that one cell is specified, optionally with flags */ - if (args->args_count != 1 && args->args_count != 2) - return ERR_PTR(-EINVAL); - pwm = pwm_request_from_chip(chip, 0, NULL); if (IS_ERR(pwm)) return pwm; - pwm->args.period = args->args[0]; - pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 1) + pwm->args.period = args->args[0]; - if (args->args_count == 2 && args->args[1] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 1 && args->args[1] & PWM_POLARITY_INVERTED) pwm->args.polarity = PWM_POLARITY_INVERSED; return pwm; From 0e253d4193cbc1c63b174f807a26e8b17693049d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 9 Jan 2024 22:34:34 +0100 Subject: [PATCH 0352/1038] pwm: clps711x: Drop custom .of_xlate() callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default of_xlate callback (of_pwm_xlate_with_flags()) does everything the drivers expects from its .of_xlate() callback. So drop the custom implementation. Link: https://lore.kernel.org/r/f58336c298d536107de5cab6a57e19f957ab326c.1704835845.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-clps711x.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drivers/pwm/pwm-clps711x.c b/drivers/pwm/pwm-clps711x.c index 06562d4bb963..f3b4af7963be 100644 --- a/drivers/pwm/pwm-clps711x.c +++ b/drivers/pwm/pwm-clps711x.c @@ -74,15 +74,6 @@ static const struct pwm_ops clps711x_pwm_ops = { .apply = clps711x_pwm_apply, }; -static struct pwm_device *clps711x_pwm_xlate(struct pwm_chip *chip, - const struct of_phandle_args *args) -{ - if (args->args[0] >= chip->npwm) - return ERR_PTR(-EINVAL); - - return pwm_request_from_chip(chip, args->args[0], NULL); -} - static int clps711x_pwm_probe(struct platform_device *pdev) { struct clps711x_chip *priv; @@ -102,7 +93,6 @@ static int clps711x_pwm_probe(struct platform_device *pdev) priv->chip.ops = &clps711x_pwm_ops; priv->chip.dev = &pdev->dev; priv->chip.npwm = 2; - priv->chip.of_xlate = clps711x_pwm_xlate; spin_lock_init(&priv->lock); From 91bb23493f8fd115e362d075ebaa3e6f12d1439d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 9 Jan 2024 22:34:35 +0100 Subject: [PATCH 0353/1038] pwm: Drop duplicate check against chip->npwm in of_pwm_xlate_with_flags() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit args->args[0] is passed as parameter "index" to pwm_request_from_chip(). The latter function also checks for index >= npwm, so of_pwm_xlate_with_flags() doesn't need to do that. Link: https://lore.kernel.org/r/b06e445a6ed62a339add727eccb969a33d678386.1704835845.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 606d9ef0c709..b025d90e201c 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -111,9 +111,6 @@ of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *arg if (args->args_count < 1) return ERR_PTR(-EINVAL); - if (args->args[0] >= chip->npwm) - return ERR_PTR(-EINVAL); - pwm = pwm_request_from_chip(chip, args->args[0], NULL); if (IS_ERR(pwm)) return pwm; From e92ce0e8f9c985815800975f6d76e1fedb58076a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 25 Jan 2024 09:56:49 +0100 Subject: [PATCH 0354/1038] pwm: mediatek: Update kernel doc for struct pwm_mediatek_of_data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The struct never had a member called clk_freq. This fixes the W=1 warning: drivers/pwm/pwm-mediatek.c:60: warning: Excess struct member 'clk_freq' description in 'pwm_mediatek_chip' Fixes: efecdeb82f21 ("pwm: mediatek: Allocate the clks array dynamically") Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20240125085649.1571268-2-u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mediatek.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index 17d290f847af..562102a47ac0 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -47,7 +47,6 @@ struct pwm_mediatek_of_data { * @clk_top: the top clock generator * @clk_main: the clock used by PWM core * @clk_pwms: the clock used by each PWM channel - * @clk_freq: the fix clock frequency of legacy MIPS SoC * @soc: pointer to chip's platform data */ struct pwm_mediatek_chip { From bdc585f987a0e18be9399abc609d564c83f74c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Duje=20Mihanovi=C4=87?= Date: Sun, 7 Jan 2024 12:46:59 +0100 Subject: [PATCH 0355/1038] dt-bindings: pxa-pwm: Convert to YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the PXA PWM binding file from TXT to YAML. The original binding does not mention any clocks, but the PWM controller will not probe without a clock. Reviewed-by: Uwe Kleine-König Signed-off-by: Duje Mihanović Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240107-pxa-pwm-yaml-v3-1-92ac90911c3f@skole.hr Signed-off-by: Uwe Kleine-König --- .../bindings/pwm/marvell,pxa-pwm.yaml | 51 +++++++++++++++++++ .../devicetree/bindings/pwm/pxa-pwm.txt | 30 ----------- 2 files changed, 51 insertions(+), 30 deletions(-) create mode 100644 Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml delete mode 100644 Documentation/devicetree/bindings/pwm/pxa-pwm.txt diff --git a/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml b/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml new file mode 100644 index 000000000000..ba6325575ea0 --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/marvell,pxa-pwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Marvell PXA PWM + +maintainers: + - Duje Mihanović + +allOf: + - $ref: pwm.yaml# + +properties: + compatible: + enum: + - marvell,pxa250-pwm + - marvell,pxa270-pwm + - marvell,pxa168-pwm + - marvell,pxa910-pwm + + reg: + # Length should be 0x10 + maxItems: 1 + + "#pwm-cells": + # Used for specifying the period length in nanoseconds + const: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - "#pwm-cells" + - clocks + +additionalProperties: false + +examples: + - | + #include + + pwm0: pwm@40b00000 { + compatible = "marvell,pxa250-pwm"; + reg = <0x40b00000 0x10>; + #pwm-cells = <1>; + clocks = <&clks CLK_PWM0>; + }; diff --git a/Documentation/devicetree/bindings/pwm/pxa-pwm.txt b/Documentation/devicetree/bindings/pwm/pxa-pwm.txt deleted file mode 100644 index 5ae9f1e3c338..000000000000 --- a/Documentation/devicetree/bindings/pwm/pxa-pwm.txt +++ /dev/null @@ -1,30 +0,0 @@ -Marvell PWM controller - -Required properties: -- compatible: should be one or more of: - - "marvell,pxa250-pwm" - - "marvell,pxa270-pwm" - - "marvell,pxa168-pwm" - - "marvell,pxa910-pwm" -- reg: Physical base address and length of the registers used by the PWM channel - Note that one device instance must be created for each PWM that is used, so the - length covers only the register window for one PWM output, not that of the - entire PWM controller. Currently length is 0x10 for all supported devices. -- #pwm-cells: Should be 1. This cell is used to specify the period in - nanoseconds. - -Example PWM device node: - -pwm0: pwm@40b00000 { - compatible = "marvell,pxa250-pwm"; - reg = <0x40b00000 0x10>; - #pwm-cells = <1>; -}; - -Example PWM client node: - -backlight { - compatible = "pwm-backlight"; - pwms = <&pwm0 5000000>; - ... -} From 62928315adfe32442b119cff42788891db89a575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 25 Jan 2024 13:08:23 +0100 Subject: [PATCH 0356/1038] pwm: Reorder symbols in core.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This moves the functions called by pwm consumers above the functions called by pwm providers. When character device support is added later this is hooked into the chip registration functions. As the needed callbacks are a kind of consumer and make use of the consumer functions, having this order is more natural and prevents having to add declarations for static functions. Also move the global variables for pwm tables to the respective functions to have them properly grouped. Link: https://lore.kernel.org/r/eed83de07bdfb69b5ceba0b9aed757ee612dea8f.1706182805.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/core.c | 619 +++++++++++++++++++++++---------------------- 1 file changed, 310 insertions(+), 309 deletions(-) diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index b025d90e201c..1b4c3d0caa82 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -24,289 +24,11 @@ #define CREATE_TRACE_POINTS #include -static DEFINE_MUTEX(pwm_lookup_lock); -static LIST_HEAD(pwm_lookup_list); - /* protects access to pwm_chips */ static DEFINE_MUTEX(pwm_lock); static DEFINE_IDR(pwm_chips); -static struct pwm_chip *pwmchip_find_by_name(const char *name) -{ - struct pwm_chip *chip; - unsigned long id, tmp; - - if (!name) - return NULL; - - mutex_lock(&pwm_lock); - - idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) { - const char *chip_name = dev_name(chip->dev); - - if (chip_name && strcmp(chip_name, name) == 0) { - mutex_unlock(&pwm_lock); - return chip; - } - } - - mutex_unlock(&pwm_lock); - - return NULL; -} - -static int pwm_device_request(struct pwm_device *pwm, const char *label) -{ - int err; - struct pwm_chip *chip = pwm->chip; - const struct pwm_ops *ops = chip->ops; - - if (test_bit(PWMF_REQUESTED, &pwm->flags)) - return -EBUSY; - - if (!try_module_get(chip->owner)) - return -ENODEV; - - if (ops->request) { - err = ops->request(chip, pwm); - if (err) { - module_put(chip->owner); - return err; - } - } - - if (ops->get_state) { - /* - * Zero-initialize state because most drivers are unaware of - * .usage_power. The other members of state are supposed to be - * set by lowlevel drivers. We still initialize the whole - * structure for simplicity even though this might paper over - * faulty implementations of .get_state(). - */ - struct pwm_state state = { 0, }; - - err = ops->get_state(chip, pwm, &state); - trace_pwm_get(pwm, &state, err); - - if (!err) - pwm->state = state; - - if (IS_ENABLED(CONFIG_PWM_DEBUG)) - pwm->last = pwm->state; - } - - set_bit(PWMF_REQUESTED, &pwm->flags); - pwm->label = label; - - return 0; -} - -struct pwm_device * -of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *args) -{ - struct pwm_device *pwm; - - /* period in the second cell and flags in the third cell are optional */ - if (args->args_count < 1) - return ERR_PTR(-EINVAL); - - pwm = pwm_request_from_chip(chip, args->args[0], NULL); - if (IS_ERR(pwm)) - return pwm; - - if (args->args_count > 1) - pwm->args.period = args->args[1]; - - pwm->args.polarity = PWM_POLARITY_NORMAL; - if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) - pwm->args.polarity = PWM_POLARITY_INVERSED; - - return pwm; -} -EXPORT_SYMBOL_GPL(of_pwm_xlate_with_flags); - -struct pwm_device * -of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) -{ - struct pwm_device *pwm; - - pwm = pwm_request_from_chip(chip, 0, NULL); - if (IS_ERR(pwm)) - return pwm; - - if (args->args_count > 1) - pwm->args.period = args->args[0]; - - pwm->args.polarity = PWM_POLARITY_NORMAL; - if (args->args_count > 1 && args->args[1] & PWM_POLARITY_INVERTED) - pwm->args.polarity = PWM_POLARITY_INVERSED; - - return pwm; -} -EXPORT_SYMBOL_GPL(of_pwm_single_xlate); - -static void of_pwmchip_add(struct pwm_chip *chip) -{ - if (!chip->dev || !chip->dev->of_node) - return; - - if (!chip->of_xlate) - chip->of_xlate = of_pwm_xlate_with_flags; - - of_node_get(chip->dev->of_node); -} - -static void of_pwmchip_remove(struct pwm_chip *chip) -{ - if (chip->dev) - of_node_put(chip->dev->of_node); -} - -static bool pwm_ops_check(const struct pwm_chip *chip) -{ - const struct pwm_ops *ops = chip->ops; - - if (!ops->apply) - return false; - - if (IS_ENABLED(CONFIG_PWM_DEBUG) && !ops->get_state) - dev_warn(chip->dev, - "Please implement the .get_state() callback\n"); - - return true; -} - -/** - * __pwmchip_add() - register a new PWM chip - * @chip: the PWM chip to add - * @owner: reference to the module providing the chip. - * - * Register a new PWM chip. @owner is supposed to be THIS_MODULE, use the - * pwmchip_add wrapper to do this right. - * - * Returns: 0 on success or a negative error code on failure. - */ -int __pwmchip_add(struct pwm_chip *chip, struct module *owner) -{ - unsigned int i; - int ret; - - if (!chip || !chip->dev || !chip->ops || !chip->npwm) - return -EINVAL; - - if (!pwm_ops_check(chip)) - return -EINVAL; - - chip->owner = owner; - - chip->pwms = kcalloc(chip->npwm, sizeof(*chip->pwms), GFP_KERNEL); - if (!chip->pwms) - return -ENOMEM; - - mutex_lock(&pwm_lock); - - ret = idr_alloc(&pwm_chips, chip, 0, 0, GFP_KERNEL); - if (ret < 0) { - mutex_unlock(&pwm_lock); - kfree(chip->pwms); - return ret; - } - - chip->id = ret; - - for (i = 0; i < chip->npwm; i++) { - struct pwm_device *pwm = &chip->pwms[i]; - - pwm->chip = chip; - pwm->hwpwm = i; - } - - mutex_unlock(&pwm_lock); - - if (IS_ENABLED(CONFIG_OF)) - of_pwmchip_add(chip); - - pwmchip_sysfs_export(chip); - - return 0; -} -EXPORT_SYMBOL_GPL(__pwmchip_add); - -/** - * pwmchip_remove() - remove a PWM chip - * @chip: the PWM chip to remove - * - * Removes a PWM chip. - */ -void pwmchip_remove(struct pwm_chip *chip) -{ - pwmchip_sysfs_unexport(chip); - - if (IS_ENABLED(CONFIG_OF)) - of_pwmchip_remove(chip); - - mutex_lock(&pwm_lock); - - idr_remove(&pwm_chips, chip->id); - - mutex_unlock(&pwm_lock); - - kfree(chip->pwms); -} -EXPORT_SYMBOL_GPL(pwmchip_remove); - -static void devm_pwmchip_remove(void *data) -{ - struct pwm_chip *chip = data; - - pwmchip_remove(chip); -} - -int __devm_pwmchip_add(struct device *dev, struct pwm_chip *chip, struct module *owner) -{ - int ret; - - ret = __pwmchip_add(chip, owner); - if (ret) - return ret; - - return devm_add_action_or_reset(dev, devm_pwmchip_remove, chip); -} -EXPORT_SYMBOL_GPL(__devm_pwmchip_add); - -/** - * pwm_request_from_chip() - request a PWM device relative to a PWM chip - * @chip: PWM chip - * @index: per-chip index of the PWM to request - * @label: a literal description string of this PWM - * - * Returns: A pointer to the PWM device at the given index of the given PWM - * chip. A negative error code is returned if the index is not valid for the - * specified PWM chip or if the PWM device cannot be requested. - */ -struct pwm_device *pwm_request_from_chip(struct pwm_chip *chip, - unsigned int index, - const char *label) -{ - struct pwm_device *pwm; - int err; - - if (!chip || index >= chip->npwm) - return ERR_PTR(-EINVAL); - - mutex_lock(&pwm_lock); - pwm = &chip->pwms[index]; - - err = pwm_device_request(pwm, label); - if (err < 0) - pwm = ERR_PTR(err); - - mutex_unlock(&pwm_lock); - return pwm; -} -EXPORT_SYMBOL_GPL(pwm_request_from_chip); - static void pwm_apply_debug(struct pwm_device *pwm, const struct pwm_state *state) { @@ -502,33 +224,6 @@ int pwm_apply_atomic(struct pwm_device *pwm, const struct pwm_state *state) } EXPORT_SYMBOL_GPL(pwm_apply_atomic); -/** - * pwm_capture() - capture and report a PWM signal - * @pwm: PWM device - * @result: structure to fill with capture result - * @timeout: time to wait, in milliseconds, before giving up on capture - * - * Returns: 0 on success or a negative error code on failure. - */ -int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, - unsigned long timeout) -{ - int err; - - if (!pwm || !pwm->chip->ops) - return -EINVAL; - - if (!pwm->chip->ops->capture) - return -ENOSYS; - - mutex_lock(&pwm_lock); - err = pwm->chip->ops->capture(pwm->chip, pwm, result, timeout); - mutex_unlock(&pwm_lock); - - return err; -} -EXPORT_SYMBOL_GPL(pwm_capture); - /** * pwm_adjust_config() - adjust the current PWM config to the PWM arguments * @pwm: PWM device @@ -585,24 +280,309 @@ int pwm_adjust_config(struct pwm_device *pwm) } EXPORT_SYMBOL_GPL(pwm_adjust_config); -static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) +/** + * pwm_capture() - capture and report a PWM signal + * @pwm: PWM device + * @result: structure to fill with capture result + * @timeout: time to wait, in milliseconds, before giving up on capture + * + * Returns: 0 on success or a negative error code on failure. + */ +int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, + unsigned long timeout) +{ + int err; + + if (!pwm || !pwm->chip->ops) + return -EINVAL; + + if (!pwm->chip->ops->capture) + return -ENOSYS; + + mutex_lock(&pwm_lock); + err = pwm->chip->ops->capture(pwm->chip, pwm, result, timeout); + mutex_unlock(&pwm_lock); + + return err; +} +EXPORT_SYMBOL_GPL(pwm_capture); + +static struct pwm_chip *pwmchip_find_by_name(const char *name) { struct pwm_chip *chip; unsigned long id, tmp; + if (!name) + return NULL; + mutex_lock(&pwm_lock); - idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) - if (chip->dev && device_match_fwnode(chip->dev, fwnode)) { + idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) { + const char *chip_name = dev_name(chip->dev); + + if (chip_name && strcmp(chip_name, name) == 0) { mutex_unlock(&pwm_lock); return chip; } + } mutex_unlock(&pwm_lock); - return ERR_PTR(-EPROBE_DEFER); + return NULL; } +static int pwm_device_request(struct pwm_device *pwm, const char *label) +{ + int err; + struct pwm_chip *chip = pwm->chip; + const struct pwm_ops *ops = chip->ops; + + if (test_bit(PWMF_REQUESTED, &pwm->flags)) + return -EBUSY; + + if (!try_module_get(chip->owner)) + return -ENODEV; + + if (ops->request) { + err = ops->request(chip, pwm); + if (err) { + module_put(chip->owner); + return err; + } + } + + if (ops->get_state) { + /* + * Zero-initialize state because most drivers are unaware of + * .usage_power. The other members of state are supposed to be + * set by lowlevel drivers. We still initialize the whole + * structure for simplicity even though this might paper over + * faulty implementations of .get_state(). + */ + struct pwm_state state = { 0, }; + + err = ops->get_state(chip, pwm, &state); + trace_pwm_get(pwm, &state, err); + + if (!err) + pwm->state = state; + + if (IS_ENABLED(CONFIG_PWM_DEBUG)) + pwm->last = pwm->state; + } + + set_bit(PWMF_REQUESTED, &pwm->flags); + pwm->label = label; + + return 0; +} + +/** + * pwm_request_from_chip() - request a PWM device relative to a PWM chip + * @chip: PWM chip + * @index: per-chip index of the PWM to request + * @label: a literal description string of this PWM + * + * Returns: A pointer to the PWM device at the given index of the given PWM + * chip. A negative error code is returned if the index is not valid for the + * specified PWM chip or if the PWM device cannot be requested. + */ +struct pwm_device *pwm_request_from_chip(struct pwm_chip *chip, + unsigned int index, + const char *label) +{ + struct pwm_device *pwm; + int err; + + if (!chip || index >= chip->npwm) + return ERR_PTR(-EINVAL); + + mutex_lock(&pwm_lock); + pwm = &chip->pwms[index]; + + err = pwm_device_request(pwm, label); + if (err < 0) + pwm = ERR_PTR(err); + + mutex_unlock(&pwm_lock); + return pwm; +} +EXPORT_SYMBOL_GPL(pwm_request_from_chip); + + +struct pwm_device * +of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *args) +{ + struct pwm_device *pwm; + + /* period in the second cell and flags in the third cell are optional */ + if (args->args_count < 1) + return ERR_PTR(-EINVAL); + + pwm = pwm_request_from_chip(chip, args->args[0], NULL); + if (IS_ERR(pwm)) + return pwm; + + if (args->args_count > 1) + pwm->args.period = args->args[1]; + + pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; + + return pwm; +} +EXPORT_SYMBOL_GPL(of_pwm_xlate_with_flags); + +struct pwm_device * +of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) +{ + struct pwm_device *pwm; + + pwm = pwm_request_from_chip(chip, 0, NULL); + if (IS_ERR(pwm)) + return pwm; + + if (args->args_count > 1) + pwm->args.period = args->args[0]; + + pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 1 && args->args[1] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; + + return pwm; +} +EXPORT_SYMBOL_GPL(of_pwm_single_xlate); + +static void of_pwmchip_add(struct pwm_chip *chip) +{ + if (!chip->dev || !chip->dev->of_node) + return; + + if (!chip->of_xlate) + chip->of_xlate = of_pwm_xlate_with_flags; + + of_node_get(chip->dev->of_node); +} + +static void of_pwmchip_remove(struct pwm_chip *chip) +{ + if (chip->dev) + of_node_put(chip->dev->of_node); +} + +static bool pwm_ops_check(const struct pwm_chip *chip) +{ + const struct pwm_ops *ops = chip->ops; + + if (!ops->apply) + return false; + + if (IS_ENABLED(CONFIG_PWM_DEBUG) && !ops->get_state) + dev_warn(chip->dev, + "Please implement the .get_state() callback\n"); + + return true; +} + +/** + * __pwmchip_add() - register a new PWM chip + * @chip: the PWM chip to add + * @owner: reference to the module providing the chip. + * + * Register a new PWM chip. @owner is supposed to be THIS_MODULE, use the + * pwmchip_add wrapper to do this right. + * + * Returns: 0 on success or a negative error code on failure. + */ +int __pwmchip_add(struct pwm_chip *chip, struct module *owner) +{ + unsigned int i; + int ret; + + if (!chip || !chip->dev || !chip->ops || !chip->npwm) + return -EINVAL; + + if (!pwm_ops_check(chip)) + return -EINVAL; + + chip->owner = owner; + + chip->pwms = kcalloc(chip->npwm, sizeof(*chip->pwms), GFP_KERNEL); + if (!chip->pwms) + return -ENOMEM; + + mutex_lock(&pwm_lock); + + ret = idr_alloc(&pwm_chips, chip, 0, 0, GFP_KERNEL); + if (ret < 0) { + mutex_unlock(&pwm_lock); + kfree(chip->pwms); + return ret; + } + + chip->id = ret; + + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; + + pwm->chip = chip; + pwm->hwpwm = i; + } + + mutex_unlock(&pwm_lock); + + if (IS_ENABLED(CONFIG_OF)) + of_pwmchip_add(chip); + + pwmchip_sysfs_export(chip); + + return 0; +} +EXPORT_SYMBOL_GPL(__pwmchip_add); + +/** + * pwmchip_remove() - remove a PWM chip + * @chip: the PWM chip to remove + * + * Removes a PWM chip. + */ +void pwmchip_remove(struct pwm_chip *chip) +{ + pwmchip_sysfs_unexport(chip); + + if (IS_ENABLED(CONFIG_OF)) + of_pwmchip_remove(chip); + + mutex_lock(&pwm_lock); + + idr_remove(&pwm_chips, chip->id); + + mutex_unlock(&pwm_lock); + + kfree(chip->pwms); +} +EXPORT_SYMBOL_GPL(pwmchip_remove); + +static void devm_pwmchip_remove(void *data) +{ + struct pwm_chip *chip = data; + + pwmchip_remove(chip); +} + +int __devm_pwmchip_add(struct device *dev, struct pwm_chip *chip, struct module *owner) +{ + int ret; + + ret = __pwmchip_add(chip, owner); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_pwmchip_remove, chip); +} +EXPORT_SYMBOL_GPL(__devm_pwmchip_add); + static struct device_link *pwm_device_link_add(struct device *dev, struct pwm_device *pwm) { @@ -629,6 +609,24 @@ static struct device_link *pwm_device_link_add(struct device *dev, return dl; } +static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) +{ + struct pwm_chip *chip; + unsigned long id, tmp; + + mutex_lock(&pwm_lock); + + idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) + if (chip->dev && device_match_fwnode(chip->dev, fwnode)) { + mutex_unlock(&pwm_lock); + return chip; + } + + mutex_unlock(&pwm_lock); + + return ERR_PTR(-EPROBE_DEFER); +} + /** * of_pwm_get() - request a PWM via the PWM framework * @dev: device for PWM consumer @@ -763,6 +761,9 @@ static struct pwm_device *acpi_pwm_get(const struct fwnode_handle *fwnode) return pwm; } +static DEFINE_MUTEX(pwm_lookup_lock); +static LIST_HEAD(pwm_lookup_list); + /** * pwm_add_table() - register PWM device consumers * @table: array of consumers to register From 65a63bf94ac296863674f80209a8e919efab45cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 29 Jan 2024 09:10:30 +0100 Subject: [PATCH 0357/1038] pwm: bcm2835: Drop write-only member of driver private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit fcc760729359 ("pwm: bcm2835: Allow PWM driver to be used in atomic context") struct bcm2835_pwm::dev is only assigned a value, but it's never used. So the dev member can just be dropped. Link: https://lore.kernel.org/r/20240129081029.2355612-2-u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-bcm2835.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c index 283cf27f25ba..711c174bbe23 100644 --- a/drivers/pwm/pwm-bcm2835.c +++ b/drivers/pwm/pwm-bcm2835.c @@ -25,7 +25,6 @@ struct bcm2835_pwm { struct pwm_chip chip; - struct device *dev; void __iomem *base; struct clk *clk; unsigned long rate; @@ -142,8 +141,6 @@ static int bcm2835_pwm_probe(struct platform_device *pdev) if (!pc) return -ENOMEM; - pc->dev = &pdev->dev; - pc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->base)) return PTR_ERR(pc->base); From c6c3f7e73dabd8c5aee0f7aad9720d8f5e3f5ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 26 Jan 2024 13:04:34 +0100 Subject: [PATCH 0358/1038] pwm: atmel-hlcdc: Don't use pwm consumer API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowlevel driver callbacks are not supposed to use the consumer API functions. Currently this works, but with the upcoming locking changes this probably results in dead locks. Reviewed-by: Claudiu Beznea Link: https://lore.kernel.org/r/6e34607828b290cd64ca9f82df40872853069f07.1706269232.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-hlcdc.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-atmel-hlcdc.c b/drivers/pwm/pwm-atmel-hlcdc.c index 1f6fc9a9fcf3..2d64af045fc5 100644 --- a/drivers/pwm/pwm-atmel-hlcdc.c +++ b/drivers/pwm/pwm-atmel-hlcdc.c @@ -183,9 +183,10 @@ static const struct atmel_hlcdc_pwm_errata atmel_hlcdc_pwm_sama5d3_errata = { static int atmel_hlcdc_pwm_suspend(struct device *dev) { struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); + struct pwm_device *pwm = &atmel->chip.pwms[0]; /* Keep the periph clock enabled if the PWM is still running. */ - if (!pwm_is_enabled(&atmel->chip.pwms[0])) + if (!pwm->state.enabled) clk_disable_unprepare(atmel->hlcdc->periph_clk); return 0; @@ -194,20 +195,17 @@ static int atmel_hlcdc_pwm_suspend(struct device *dev) static int atmel_hlcdc_pwm_resume(struct device *dev) { struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); - struct pwm_state state; + struct pwm_device *pwm = &atmel->chip.pwms[0]; int ret; - pwm_get_state(&atmel->chip.pwms[0], &state); - /* Re-enable the periph clock it was stopped during suspend. */ - if (!state.enabled) { + if (!pwm->state.enabled) { ret = clk_prepare_enable(atmel->hlcdc->periph_clk); if (ret) return ret; } - return atmel_hlcdc_pwm_apply(&atmel->chip, &atmel->chip.pwms[0], - &state); + return atmel_hlcdc_pwm_apply(&atmel->chip, pwm, &pwm->state); } static DEFINE_SIMPLE_DEV_PM_OPS(atmel_hlcdc_pwm_pm_ops, From f8b403f5442194e8d92a8de20b2cc443d5ec64ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 29 Jan 2024 09:56:25 +0100 Subject: [PATCH 0359/1038] pwm: atmel: Don't use pwm consumer API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowlevel driver callbacks are not supposed to use the consumer API functions. Currently this works, but with the upcoming locking changes this probably results in dead locks. Reviewed-by: Claudiu Beznea Link: https://lore.kernel.org/r/20240129085627.2386470-2-u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index 47bcc8a3bf9d..26573829e391 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -294,19 +294,16 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct atmel_pwm_chip *atmel_pwm = to_atmel_pwm_chip(chip); - struct pwm_state cstate; unsigned long cprd, cdty; u32 pres, val; int ret; - pwm_get_state(pwm, &cstate); - if (state->enabled) { unsigned long clkrate = clk_get_rate(atmel_pwm->clk); - if (cstate.enabled && - cstate.polarity == state->polarity && - cstate.period == state->period) { + if (pwm->state.enabled && + pwm->state.polarity == state->polarity && + pwm->state.period == state->period) { u32 cmr = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, PWM_CMR); cprd = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, @@ -328,7 +325,7 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, atmel_pwm_calculate_cdty(state, clkrate, cprd, pres, &cdty); - if (cstate.enabled) { + if (pwm->state.enabled) { atmel_pwm_disable(chip, pwm, false); } else { ret = clk_enable(atmel_pwm->clk); @@ -348,7 +345,7 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, atmel_pwm_ch_writel(atmel_pwm, pwm->hwpwm, PWM_CMR, val); atmel_pwm_set_cprd_cdty(chip, pwm, cprd, cdty); atmel_pwm_writel(atmel_pwm, PWM_ENA, 1 << pwm->hwpwm); - } else if (cstate.enabled) { + } else if (pwm->state.enabled) { atmel_pwm_disable(chip, pwm, true); } From 1706175c682fdb3fbe0b70d50b137aaa270959db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 26 Jan 2024 11:16:25 +0100 Subject: [PATCH 0360/1038] pwm: imx27: Don't use pwm consumer API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowlevel driver callbacks are not supposed to use the consumer API functions. Currently this works, but with the upcoming locking changes this maybe results in dead locks. So directly access pwm->state. Link: https://lore.kernel.org/r/20240126101624.2056801-2-u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-imx27.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index 7d9bc43f12b0..c825d3bcb329 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -204,8 +204,8 @@ static void pwm_imx27_wait_fifo_slot(struct pwm_chip *chip, sr = readl(imx->mmio_base + MX3_PWMSR); fifoav = FIELD_GET(MX3_PWMSR_FIFOAV, sr); if (fifoav == MX3_PWMSR_FIFOAV_4WORDS) { - period_ms = DIV_ROUND_UP_ULL(pwm_get_period(pwm), - NSEC_PER_MSEC); + period_ms = DIV_ROUND_UP_ULL(pwm->state.period, + NSEC_PER_MSEC); msleep(period_ms); sr = readl(imx->mmio_base + MX3_PWMSR); @@ -219,14 +219,11 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, { unsigned long period_cycles, duty_cycles, prescale; struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); - struct pwm_state cstate; unsigned long long c; unsigned long long clkrate; int ret; u32 cr; - pwm_get_state(pwm, &cstate); - clkrate = clk_get_rate(imx->clk_per); c = clkrate * state->period; @@ -254,7 +251,7 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, * Wait for a free FIFO slot if the PWM is already enabled, and flush * the FIFO if the PWM was disabled and is about to be enabled. */ - if (cstate.enabled) { + if (pwm->state.enabled) { pwm_imx27_wait_fifo_slot(chip, pwm); } else { ret = pwm_imx27_clk_prepare_enable(imx); From 0fa319a1427f7c8d0af4c255316624f7e6f649a0 Mon Sep 17 00:00:00 2001 From: Dharma Balasubiramani Date: Fri, 2 Feb 2024 05:47:32 +0530 Subject: [PATCH 0361/1038] dt-bindings: atmel,hlcdc: convert pwm bindings to json-schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert device tree bindings for Atmel's HLCDC PWM controller to YAML format. Signed-off-by: Dharma Balasubiramani Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20240202001733.91455-3-dharma.b@microchip.com Signed-off-by: Uwe Kleine-König --- .../bindings/pwm/atmel,hlcdc-pwm.yaml | 35 +++++++++++++++++++ .../bindings/pwm/atmel-hlcdc-pwm.txt | 29 --------------- 2 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml delete mode 100644 Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt diff --git a/Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml b/Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml new file mode 100644 index 000000000000..0e92868a2b68 --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/atmel,hlcdc-pwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Atmel's HLCDC's PWM controller + +maintainers: + - Nicolas Ferre + - Alexandre Belloni + - Claudiu Beznea + +description: + The LCDC integrates a Pulse Width Modulation (PWM) Controller. This block + generates the LCD contrast control signal (LCD_PWM) that controls the + display's contrast by software. LCDC_PWM is an 8-bit PWM signal that can be + converted to an analog voltage with a simple passive filter. LCD display + panels have different backlight specifications in terms of minimum/maximum + values for PWM frequency. If the LCDC PWM frequency range does not match the + LCD display panel, it is possible to use the standalone PWM Controller to + drive the backlight. + +properties: + compatible: + const: atmel,hlcdc-pwm + + "#pwm-cells": + const: 3 + +required: + - compatible + - "#pwm-cells" + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt b/Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt deleted file mode 100644 index afa501bf7f94..000000000000 --- a/Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt +++ /dev/null @@ -1,29 +0,0 @@ -Device-Tree bindings for Atmel's HLCDC (High-end LCD Controller) PWM driver - -The Atmel HLCDC PWM is subdevice of the HLCDC MFD device. -See ../mfd/atmel-hlcdc.txt for more details. - -Required properties: - - compatible: value should be one of the following: - "atmel,hlcdc-pwm" - - pinctr-names: the pin control state names. Should contain "default". - - pinctrl-0: should contain the pinctrl states described by pinctrl - default. - - #pwm-cells: should be set to 3. This PWM chip use the default 3 cells - bindings defined in pwm.yaml in this directory. - -Example: - - hlcdc: hlcdc@f0030000 { - compatible = "atmel,sama5d3-hlcdc"; - reg = <0xf0030000 0x2000>; - clocks = <&lcdc_clk>, <&lcdck>, <&clk32k>; - clock-names = "periph_clk","sys_clk", "slow_clk"; - - hlcdc_pwm: hlcdc-pwm { - compatible = "atmel,hlcdc-pwm"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_lcd_pwm>; - #pwm-cells = <3>; - }; - }; From 3ee35fd12b018e15ed540c921ce0bbb6254f4c2a Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 8 Feb 2024 12:35:28 +0530 Subject: [PATCH 0362/1038] pwm: dwc: access driver_data using dev_get_drvdata() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we're setting driver_data using dev_set_drvdata(), we can use dev_get_drvdata() for accessing it. Signed-off-by: Raag Jadav Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240212130247.9985-6-raag.jadav@intel.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index a4a057ae03ea..9a1b156ead26 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -73,8 +73,7 @@ static void dwc_pwm_remove(struct pci_dev *pci) static int dwc_pwm_suspend(struct device *dev) { - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); - struct dwc_pwm *dwc = pci_get_drvdata(pdev); + struct dwc_pwm *dwc = dev_get_drvdata(dev); int i; for (i = 0; i < DWC_TIMERS_TOTAL; i++) { @@ -93,8 +92,7 @@ static int dwc_pwm_suspend(struct device *dev) static int dwc_pwm_resume(struct device *dev) { - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); - struct dwc_pwm *dwc = pci_get_drvdata(pdev); + struct dwc_pwm *dwc = dev_get_drvdata(dev); int i; for (i = 0; i < DWC_TIMERS_TOTAL; i++) { From 0c2d3bd748833cdc0b0fc32c7d14a6b1f9f6ea3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 14 Feb 2024 15:04:53 +0100 Subject: [PATCH 0363/1038] dt-bindings: pwm: mediatek,mt2712: add compatible for MT7988 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MT7988 has on-SoC controller that can control up to 8 PWM interfaces. It differs from blocks on other SoCs (amount of PWMs & registers) so it needs its own compatible string. Signed-off-by: Rafał Miłecki Acked-by: Krzysztof Kozlowski Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20240214140454.6438-1-zajec5@gmail.com Signed-off-by: Uwe Kleine-König --- Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml b/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml index 0fbe8a6469eb..a5c308801619 100644 --- a/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml +++ b/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml @@ -24,6 +24,7 @@ properties: - mediatek,mt7629-pwm - mediatek,mt7981-pwm - mediatek,mt7986-pwm + - mediatek,mt7988-pwm - mediatek,mt8183-pwm - mediatek,mt8365-pwm - mediatek,mt8516-pwm From eb58bf4afd708eb3c64c7b9b2c5fbfacdcdee3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 14 Feb 2024 15:04:54 +0100 Subject: [PATCH 0364/1038] pwm: mediatek: add support for MT7988 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MT7988 uses new registers layout just like MT7981 but it supports 8 PWM interfaces. Signed-off-by: Rafał Miłecki Reviewed-by: Daniel Golle Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20240214140454.6438-2-zajec5@gmail.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mediatek.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index 562102a47ac0..292c8bd5b343 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -339,6 +339,13 @@ static const struct pwm_mediatek_of_data mt7986_pwm_data = { .reg_offset = mtk_pwm_reg_offset_v1, }; +static const struct pwm_mediatek_of_data mt7988_pwm_data = { + .num_pwms = 8, + .pwm45_fixup = false, + .has_ck_26m_sel = false, + .reg_offset = mtk_pwm_reg_offset_v2, +}; + static const struct pwm_mediatek_of_data mt8183_pwm_data = { .num_pwms = 4, .pwm45_fixup = false, @@ -369,6 +376,7 @@ static const struct of_device_id pwm_mediatek_of_match[] = { { .compatible = "mediatek,mt7629-pwm", .data = &mt7629_pwm_data }, { .compatible = "mediatek,mt7981-pwm", .data = &mt7981_pwm_data }, { .compatible = "mediatek,mt7986-pwm", .data = &mt7986_pwm_data }, + { .compatible = "mediatek,mt7988-pwm", .data = &mt7988_pwm_data }, { .compatible = "mediatek,mt8183-pwm", .data = &mt8183_pwm_data }, { .compatible = "mediatek,mt8365-pwm", .data = &mt8365_pwm_data }, { .compatible = "mediatek,mt8516-pwm", .data = &mt8516_pwm_data }, From 4e59267c7a20eb1c1ad9106e682cb6a0d8eb3111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:48 +0100 Subject: [PATCH 0365/1038] pwm: Provide an inline function to get the parent device of a given chip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently a pwm_chip stores in its struct device *dev member a pointer to the parent device. Preparing a change that embeds a full struct device in struct pwm_chip, this accessor function should be used in all drivers directly accessing chip->dev now. This way struct pwm_chip and this new function can be changed without having to touch all drivers in the same change set. Make use of this function in the framework's core sources. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/cc30090d2f9762bed9854a55612144bccc910781.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/core.c | 42 +++++++++++++++++++++--------------------- drivers/pwm/sysfs.c | 4 ++-- include/linux/pwm.h | 5 +++++ 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 1b4c3d0caa82..830a697826af 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -71,18 +71,18 @@ static void pwm_apply_debug(struct pwm_device *pwm, if (s2.polarity != state->polarity && state->duty_cycle < state->period) - dev_warn(chip->dev, ".apply ignored .polarity\n"); + dev_warn(pwmchip_parent(chip), ".apply ignored .polarity\n"); if (state->enabled && last->polarity == state->polarity && last->period > s2.period && last->period <= state->period) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply didn't pick the best available period (requested: %llu, applied: %llu, possible: %llu)\n", state->period, s2.period, last->period); if (state->enabled && state->period < s2.period) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply is supposed to round down period (requested: %llu, applied: %llu)\n", state->period, s2.period); @@ -91,20 +91,20 @@ static void pwm_apply_debug(struct pwm_device *pwm, last->period == s2.period && last->duty_cycle > s2.duty_cycle && last->duty_cycle <= state->duty_cycle) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply didn't pick the best available duty cycle (requested: %llu/%llu, applied: %llu/%llu, possible: %llu/%llu)\n", state->duty_cycle, state->period, s2.duty_cycle, s2.period, last->duty_cycle, last->period); if (state->enabled && state->duty_cycle < s2.duty_cycle) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply is supposed to round down duty_cycle (requested: %llu/%llu, applied: %llu/%llu)\n", state->duty_cycle, state->period, s2.duty_cycle, s2.period); if (!state->enabled && s2.enabled && s2.duty_cycle > 0) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "requested disabled, but yielded enabled with duty > 0\n"); /* reapply the state that the driver reported being configured. */ @@ -112,7 +112,7 @@ static void pwm_apply_debug(struct pwm_device *pwm, trace_pwm_apply(pwm, &s1, err); if (err) { *last = s1; - dev_err(chip->dev, "failed to reapply current setting\n"); + dev_err(pwmchip_parent(chip), "failed to reapply current setting\n"); return; } @@ -127,7 +127,7 @@ static void pwm_apply_debug(struct pwm_device *pwm, s1.polarity != last->polarity || (s1.enabled && s1.period != last->period) || (s1.enabled && s1.duty_cycle != last->duty_cycle)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), ".apply is not idempotent (ena=%d pol=%d %llu/%llu) -> (ena=%d pol=%d %llu/%llu)\n", s1.enabled, s1.polarity, s1.duty_cycle, s1.period, last->enabled, last->polarity, last->duty_cycle, @@ -318,7 +318,7 @@ static struct pwm_chip *pwmchip_find_by_name(const char *name) mutex_lock(&pwm_lock); idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) { - const char *chip_name = dev_name(chip->dev); + const char *chip_name = dev_name(pwmchip_parent(chip)); if (chip_name && strcmp(chip_name, name) == 0) { mutex_unlock(&pwm_lock); @@ -456,19 +456,19 @@ EXPORT_SYMBOL_GPL(of_pwm_single_xlate); static void of_pwmchip_add(struct pwm_chip *chip) { - if (!chip->dev || !chip->dev->of_node) + if (!pwmchip_parent(chip) || !pwmchip_parent(chip)->of_node) return; if (!chip->of_xlate) chip->of_xlate = of_pwm_xlate_with_flags; - of_node_get(chip->dev->of_node); + of_node_get(pwmchip_parent(chip)->of_node); } static void of_pwmchip_remove(struct pwm_chip *chip) { - if (chip->dev) - of_node_put(chip->dev->of_node); + if (pwmchip_parent(chip)) + of_node_put(pwmchip_parent(chip)->of_node); } static bool pwm_ops_check(const struct pwm_chip *chip) @@ -479,7 +479,7 @@ static bool pwm_ops_check(const struct pwm_chip *chip) return false; if (IS_ENABLED(CONFIG_PWM_DEBUG) && !ops->get_state) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "Please implement the .get_state() callback\n"); return true; @@ -500,7 +500,7 @@ int __pwmchip_add(struct pwm_chip *chip, struct module *owner) unsigned int i; int ret; - if (!chip || !chip->dev || !chip->ops || !chip->npwm) + if (!chip || !pwmchip_parent(chip) || !chip->ops || !chip->npwm) return -EINVAL; if (!pwm_ops_check(chip)) @@ -594,15 +594,15 @@ static struct device_link *pwm_device_link_add(struct device *dev, * impact the PM sequence ordering: the PWM supplier may get * suspended before the consumer. */ - dev_warn(pwm->chip->dev, + dev_warn(pwmchip_parent(pwm->chip), "No consumer device specified to create a link to\n"); return NULL; } - dl = device_link_add(dev, pwm->chip->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + dl = device_link_add(dev, pwmchip_parent(pwm->chip), DL_FLAG_AUTOREMOVE_CONSUMER); if (!dl) { dev_err(dev, "failed to create device link to %s\n", - dev_name(pwm->chip->dev)); + dev_name(pwmchip_parent(pwm->chip))); return ERR_PTR(-EINVAL); } @@ -617,7 +617,7 @@ static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) mutex_lock(&pwm_lock); idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) - if (chip->dev && device_match_fwnode(chip->dev, fwnode)) { + if (pwmchip_parent(chip) && device_match_fwnode(pwmchip_parent(chip), fwnode)) { mutex_unlock(&pwm_lock); return chip; } @@ -1085,8 +1085,8 @@ static int pwm_seq_show(struct seq_file *s, void *v) seq_printf(s, "%s%d: %s/%s, %d PWM device%s\n", (char *)s->private, chip->id, - chip->dev->bus ? chip->dev->bus->name : "no-bus", - dev_name(chip->dev), chip->npwm, + pwmchip_parent(chip)->bus ? pwmchip_parent(chip)->bus->name : "no-bus", + dev_name(pwmchip_parent(chip)), chip->npwm, (chip->npwm != 1) ? "s" : ""); pwm_dbg_show(chip, s); diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index 1698609d91c8..3f434a771fb5 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -509,10 +509,10 @@ void pwmchip_sysfs_export(struct pwm_chip *chip) * If device_create() fails the pwm_chip is still usable by * the kernel it's just not exported. */ - parent = device_create(&pwm_class, chip->dev, MKDEV(0, 0), chip, + parent = device_create(&pwm_class, pwmchip_parent(chip), MKDEV(0, 0), chip, "pwmchip%d", chip->id); if (IS_ERR(parent)) { - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "device_create failed for pwm_chip sysfs export\n"); } } diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 8ffe9ae7a23a..07af6910bdce 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -289,6 +289,11 @@ struct pwm_chip { struct pwm_device *pwms; }; +static inline struct device *pwmchip_parent(const struct pwm_chip *chip) +{ + return chip->dev; +} + #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ int pwm_apply_might_sleep(struct pwm_device *pwm, const struct pwm_state *state); From 24003d501f112a9827aa8c59c00e12b2d5cbddf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:49 +0100 Subject: [PATCH 0366/1038] pwm: Provide wrappers for storing and getting driver private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These functions are useful to store and query driver private data depending on the pwm_chip. After struct pwm_chip got its own struct device, this can make use of dev_get_drvdata() and dev_set_drvdata() on that device. These functions are required already now to convert drivers to pwmchip_alloc() which must happen before changing pwm_chip::dev. Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/67514cdf29d29bd8b4ad8d44fac87f6ae6dca1e5.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- include/linux/pwm.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 07af6910bdce..29a7d9140f77 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -272,6 +272,7 @@ struct pwm_ops { * @npwm: number of PWMs controlled by this chip * @of_xlate: request a PWM device given a device tree PWM specifier * @atomic: can the driver's ->apply() be called in atomic context + * @driver_data: Private pointer for driver specific info * @pwms: array of PWM devices allocated by the framework */ struct pwm_chip { @@ -286,6 +287,7 @@ struct pwm_chip { bool atomic; /* only used internally by the PWM framework */ + void *driver_data; struct pwm_device *pwms; }; @@ -294,6 +296,24 @@ static inline struct device *pwmchip_parent(const struct pwm_chip *chip) return chip->dev; } +static inline void *pwmchip_get_drvdata(struct pwm_chip *chip) +{ + /* + * After pwm_chip got a dedicated struct device, this can be replaced by + * dev_get_drvdata(&chip->dev); + */ + return chip->driver_data; +} + +static inline void pwmchip_set_drvdata(struct pwm_chip *chip, void *data) +{ + /* + * After pwm_chip got a dedicated struct device, this can be replaced by + * dev_set_drvdata(&chip->dev, data); + */ + chip->driver_data = data; +} + #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ int pwm_apply_might_sleep(struct pwm_device *pwm, const struct pwm_state *state); From 024913dbf99f06f09d5c06c940ca39821697d41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:50 +0100 Subject: [PATCH 0367/1038] pwm: Provide pwmchip_alloc() function and a devm variant of it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function allocates a struct pwm_chip and driver data. Compared to the status quo the split into pwm_chip and driver data is new, otherwise it doesn't change anything relevant (yet). The intention is that after all drivers are switched to use this allocation function, its possible to add a struct device to struct pwm_chip to properly track the latter's lifetime without touching all drivers again. Proper lifetime tracking is a necessary precondition to introduce character device support for PWMs (that implements atomic setting and doesn't suffer from the sysfs overhead of the /sys/class/pwm userspace support). The new function pwmchip_priv() (obviously?) only works for chips allocated with pwmchip_alloc(). Link: https://lore.kernel.org/r/9577d6053a5a52536057dc8654ff567181c2da82.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- .../driver-api/driver-model/devres.rst | 1 + Documentation/driver-api/pwm.rst | 11 ++-- drivers/pwm/core.c | 58 +++++++++++++++++++ include/linux/pwm.h | 22 +++++++ 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index c5f99d834ec5..e4df72c408d2 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -420,6 +420,7 @@ POWER devm_reboot_mode_unregister() PWM + devm_pwmchip_alloc() devm_pwmchip_add() devm_pwm_get() devm_fwnode_pwm_get() diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst index 3c28ccc4b611..b41b1c56477f 100644 --- a/Documentation/driver-api/pwm.rst +++ b/Documentation/driver-api/pwm.rst @@ -143,11 +143,12 @@ to implement the pwm_*() functions itself. This means that it's impossible to have multiple PWM drivers in the system. For this reason it's mandatory for new drivers to use the generic PWM framework. -A new PWM controller/chip can be added using pwmchip_add() and removed -again with pwmchip_remove(). pwmchip_add() takes a filled in struct -pwm_chip as argument which provides a description of the PWM chip, the -number of PWM devices provided by the chip and the chip-specific -implementation of the supported PWM operations to the framework. +A new PWM controller/chip can be allocated using pwmchip_alloc(), then +registered using pwmchip_add() and removed again with pwmchip_remove(). To undo +pwmchip_alloc() use pwmchip_put(). pwmchip_add() takes a filled in struct +pwm_chip as argument which provides a description of the PWM chip, the number +of PWM devices provided by the chip and the chip-specific implementation of the +supported PWM operations to the framework. When implementing polarity support in a PWM driver, make sure to respect the signal conventions in the PWM framework. By definition, normal polarity diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 830a697826af..d70f793ce4b3 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -454,6 +454,64 @@ of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) } EXPORT_SYMBOL_GPL(of_pwm_single_xlate); +#define PWMCHIP_ALIGN ARCH_DMA_MINALIGN + +static void *pwmchip_priv(struct pwm_chip *chip) +{ + return (void *)chip + ALIGN(sizeof(*chip), PWMCHIP_ALIGN); +} + +/* This is the counterpart to pwmchip_alloc() */ +void pwmchip_put(struct pwm_chip *chip) +{ + kfree(chip); +} +EXPORT_SYMBOL_GPL(pwmchip_put); + +struct pwm_chip *pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv) +{ + struct pwm_chip *chip; + size_t alloc_size; + + alloc_size = size_add(ALIGN(sizeof(*chip), PWMCHIP_ALIGN), sizeof_priv); + + chip = kzalloc(alloc_size, GFP_KERNEL); + if (!chip) + return ERR_PTR(-ENOMEM); + + chip->dev = parent; + chip->npwm = npwm; + + pwmchip_set_drvdata(chip, pwmchip_priv(chip)); + + return chip; +} +EXPORT_SYMBOL_GPL(pwmchip_alloc); + +static void devm_pwmchip_put(void *data) +{ + struct pwm_chip *chip = data; + + pwmchip_put(chip); +} + +struct pwm_chip *devm_pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv) +{ + struct pwm_chip *chip; + int ret; + + chip = pwmchip_alloc(parent, npwm, sizeof_priv); + if (IS_ERR(chip)) + return chip; + + ret = devm_add_action_or_reset(parent, devm_pwmchip_put, chip); + if (ret) + return ERR_PTR(ret); + + return chip; +} +EXPORT_SYMBOL_GPL(devm_pwmchip_alloc); + static void of_pwmchip_add(struct pwm_chip *chip) { if (!pwmchip_parent(chip) || !pwmchip_parent(chip)->of_node) diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 29a7d9140f77..4a6568dfdf3f 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -403,6 +403,10 @@ static inline bool pwm_might_sleep(struct pwm_device *pwm) int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, unsigned long timeout); +void pwmchip_put(struct pwm_chip *chip); +struct pwm_chip *pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv); +struct pwm_chip *devm_pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv); + int __pwmchip_add(struct pwm_chip *chip, struct module *owner); #define pwmchip_add(chip) __pwmchip_add(chip, THIS_MODULE) void pwmchip_remove(struct pwm_chip *chip); @@ -475,6 +479,24 @@ static inline int pwm_capture(struct pwm_device *pwm, return -EINVAL; } +static inline void pwmchip_put(struct pwm_chip *chip) +{ +} + +static inline struct pwm_chip *pwmchip_alloc(struct device *parent, + unsigned int npwm, + size_t sizeof_priv) +{ + return ERR_PTR(-EINVAL); +} + +static inline struct pwm_chip *devm_pwmchip_alloc(struct device *parent, + unsigned int npwm, + size_t sizeof_priv) +{ + return pwmchip_alloc(parent, npwm, sizeof_priv); +} + static inline int pwmchip_add(struct pwm_chip *chip) { return -EINVAL; From ddabe9093c172445c133dd2a46fb01ac265a8f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:51 +0100 Subject: [PATCH 0368/1038] pwm: ab8500: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/24028d8d0add621a0c054235e6281a05a83d8fb4.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-ab8500.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-ab8500.c b/drivers/pwm/pwm-ab8500.c index 670d33daea84..719e4ccf1800 100644 --- a/drivers/pwm/pwm-ab8500.c +++ b/drivers/pwm/pwm-ab8500.c @@ -92,12 +92,12 @@ static int ab8500_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, * when disabled. */ if (!state->enabled || duty_steps == 0) { - ret = abx500_mask_and_set_register_interruptible(chip->dev, + ret = abx500_mask_and_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL7_REG, 1 << ab8500->hwid, 0); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM, Error %d\n", + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM, Error %d\n", pwm->label, ret); return ret; } @@ -115,22 +115,22 @@ static int ab8500_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, reg = AB8500_PWM_OUT_CTRL1_REG + (ab8500->hwid * 2); - ret = abx500_set_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, reg, lower_val); if (ret < 0) return ret; - ret = abx500_set_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, (reg + 1), higher_val); if (ret < 0) return ret; /* enable */ - ret = abx500_mask_and_set_register_interruptible(chip->dev, + ret = abx500_mask_and_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL7_REG, 1 << ab8500->hwid, 1 << ab8500->hwid); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM, Error %d\n", + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM, Error %d\n", pwm->label, ret); return ret; @@ -144,7 +144,7 @@ static int ab8500_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct ab8500_pwm_chip *ab8500 = ab8500_pwm_from_chip(chip); unsigned int div, duty_steps; - ret = abx500_get_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_get_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL7_REG, &ctrl7); if (ret) @@ -157,13 +157,13 @@ static int ab8500_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, return 0; } - ret = abx500_get_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_get_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL1_REG + (ab8500->hwid * 2), &lower_val); if (ret) return ret; - ret = abx500_get_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_get_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL2_REG + (ab8500->hwid * 2), &higher_val); if (ret) From 96af28dc44b9a13c04d6b70a9ad5a8fa0daf1d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:52 +0100 Subject: [PATCH 0369/1038] pwm: ab8500: Introduce a local pwm_chip variable in .probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This simplifies converting the driver to pwmchip_alloc() as there is only a single code line left that makes use of struct ab8500_pwm_chip::chip. Link: https://lore.kernel.org/r/7fae4833b97728abc8bd400663fdd331c86a761b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-ab8500.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-ab8500.c b/drivers/pwm/pwm-ab8500.c index 719e4ccf1800..9043e5643f44 100644 --- a/drivers/pwm/pwm-ab8500.c +++ b/drivers/pwm/pwm-ab8500.c @@ -185,6 +185,7 @@ static const struct pwm_ops ab8500_pwm_ops = { static int ab8500_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct ab8500_pwm_chip *ab8500; int err; @@ -199,12 +200,13 @@ static int ab8500_pwm_probe(struct platform_device *pdev) if (ab8500 == NULL) return -ENOMEM; - ab8500->chip.dev = &pdev->dev; - ab8500->chip.ops = &ab8500_pwm_ops; - ab8500->chip.npwm = 1; + chip = &ab8500->chip; + chip->dev = &pdev->dev; + chip->ops = &ab8500_pwm_ops; + chip->npwm = 1; ab8500->hwid = pdev->id - 1; - err = devm_pwmchip_add(&pdev->dev, &ab8500->chip); + err = devm_pwmchip_add(&pdev->dev, chip); if (err < 0) return dev_err_probe(&pdev->dev, err, "Failed to add pwm chip\n"); From 6357c2cd05e167f885edd784254da59e311197bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:53 +0100 Subject: [PATCH 0370/1038] pwm: ab8500: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-ab8500 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/9c952baafe7e53c482adf23215138724b61e376b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-ab8500.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-ab8500.c b/drivers/pwm/pwm-ab8500.c index 9043e5643f44..f000adab85b0 100644 --- a/drivers/pwm/pwm-ab8500.c +++ b/drivers/pwm/pwm-ab8500.c @@ -24,13 +24,12 @@ #define AB8500_PWM_CLKRATE 9600000 struct ab8500_pwm_chip { - struct pwm_chip chip; unsigned int hwid; }; static struct ab8500_pwm_chip *ab8500_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct ab8500_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int ab8500_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -196,14 +195,13 @@ static int ab8500_pwm_probe(struct platform_device *pdev) * Nothing to be done in probe, this is required to get the * device which is required for ab8500 read and write */ - ab8500 = devm_kzalloc(&pdev->dev, sizeof(*ab8500), GFP_KERNEL); - if (ab8500 == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*ab8500)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + ab8500 = ab8500_pwm_from_chip(chip); - chip = &ab8500->chip; - chip->dev = &pdev->dev; chip->ops = &ab8500_pwm_ops; - chip->npwm = 1; ab8500->hwid = pdev->id - 1; err = devm_pwmchip_add(&pdev->dev, chip); From 5dd820cbfc04e94f8c636bc4ace5125c602472fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:54 +0100 Subject: [PATCH 0371/1038] pwm: apple: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-apple driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/34cf20a82ca07bb4ec0578b193daa5caed37825e.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-apple.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-apple.c b/drivers/pwm/pwm-apple.c index 4d755b628d9e..6e58aca2f13c 100644 --- a/drivers/pwm/pwm-apple.c +++ b/drivers/pwm/pwm-apple.c @@ -32,14 +32,13 @@ #define APPLE_PWM_CTRL_OUTPUT_ENABLE BIT(14) struct apple_pwm { - struct pwm_chip chip; void __iomem *base; u64 clkrate; }; static inline struct apple_pwm *to_apple_pwm(struct pwm_chip *chip) { - return container_of(chip, struct apple_pwm, chip); + return pwmchip_get_drvdata(chip); } static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -103,13 +102,16 @@ static const struct pwm_ops apple_pwm_ops = { static int apple_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct apple_pwm *fpwm; struct clk *clk; int ret; - fpwm = devm_kzalloc(&pdev->dev, sizeof(*fpwm), GFP_KERNEL); - if (!fpwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*fpwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + fpwm = to_apple_pwm(chip); fpwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(fpwm->base)) @@ -129,11 +131,9 @@ static int apple_pwm_probe(struct platform_device *pdev) if (fpwm->clkrate > NSEC_PER_SEC) return dev_err_probe(&pdev->dev, -EINVAL, "pwm clock out of range"); - fpwm->chip.dev = &pdev->dev; - fpwm->chip.npwm = 1; - fpwm->chip.ops = &apple_pwm_ops; + chip->ops = &apple_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &fpwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "unable to add pwm chip"); From cc2b5bc6147beaab5bfefa9326bffa6be3ef94bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:55 +0100 Subject: [PATCH 0372/1038] pwm: atmel: Change prototype of a helper to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given atmel_pwm_chip. To just not have to do that, rework atmel_pwm_enable_clk_if_on() to take a pwm_chip. Link: https://lore.kernel.org/r/c9a92f77760e401debfe0c9bfc086222f31fb3c4.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index 26573829e391..d96afc249b77 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -459,8 +459,9 @@ static const struct of_device_id atmel_pwm_dt_ids[] = { }; MODULE_DEVICE_TABLE(of, atmel_pwm_dt_ids); -static int atmel_pwm_enable_clk_if_on(struct atmel_pwm_chip *atmel_pwm, bool on) +static int atmel_pwm_enable_clk_if_on(struct pwm_chip *chip, bool on) { + struct atmel_pwm_chip *atmel_pwm = to_atmel_pwm_chip(chip); unsigned int i, cnt = 0; unsigned long sr; int ret = 0; @@ -469,7 +470,7 @@ static int atmel_pwm_enable_clk_if_on(struct atmel_pwm_chip *atmel_pwm, bool on) if (!sr) return 0; - cnt = bitmap_weight(&sr, atmel_pwm->chip.npwm); + cnt = bitmap_weight(&sr, chip->npwm); if (!on) goto disable_clk; @@ -477,7 +478,7 @@ static int atmel_pwm_enable_clk_if_on(struct atmel_pwm_chip *atmel_pwm, bool on) for (i = 0; i < cnt; i++) { ret = clk_enable(atmel_pwm->clk); if (ret) { - dev_err(atmel_pwm->chip.dev, + dev_err(chip->dev, "failed to enable clock for pwm %pe\n", ERR_PTR(ret)); @@ -498,6 +499,7 @@ disable_clk: static int atmel_pwm_probe(struct platform_device *pdev) { struct atmel_pwm_chip *atmel_pwm; + struct pwm_chip *chip; int ret; atmel_pwm = devm_kzalloc(&pdev->dev, sizeof(*atmel_pwm), GFP_KERNEL); @@ -518,15 +520,16 @@ static int atmel_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(atmel_pwm->clk), "failed to get prepared PWM clock\n"); - atmel_pwm->chip.dev = &pdev->dev; - atmel_pwm->chip.ops = &atmel_pwm_ops; - atmel_pwm->chip.npwm = 4; + chip = &atmel_pwm->chip; + chip->dev = &pdev->dev; + chip->ops = &atmel_pwm_ops; + chip->npwm = 4; - ret = atmel_pwm_enable_clk_if_on(atmel_pwm, true); + ret = atmel_pwm_enable_clk_if_on(chip, true); if (ret < 0) return ret; - ret = devm_pwmchip_add(&pdev->dev, &atmel_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); goto disable_clk; @@ -535,7 +538,7 @@ static int atmel_pwm_probe(struct platform_device *pdev) return 0; disable_clk: - atmel_pwm_enable_clk_if_on(atmel_pwm, false); + atmel_pwm_enable_clk_if_on(chip, false); return ret; } From f9e4d438e12b669f3227fdc56baa8c286a5a79e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:56 +0100 Subject: [PATCH 0373/1038] pwm: atmel: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/f8ca10033204111e4eb2e7a4dad0bd1f7586c735.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index d96afc249b77..920bad27fbee 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -210,7 +210,7 @@ static int atmel_pwm_calculate_cprd_and_pres(struct pwm_chip *chip, shift = fls(cycles) - atmel_pwm->data->cfg.period_bits; if (shift > PWM_MAX_PRES) { - dev_err(chip->dev, "pres exceeds the maximum value\n"); + dev_err(pwmchip_parent(chip), "pres exceeds the maximum value\n"); return -EINVAL; } else if (shift > 0) { *pres = shift; @@ -318,7 +318,7 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = atmel_pwm_calculate_cprd_and_pres(chip, clkrate, state, &cprd, &pres); if (ret) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to calculate cprd and prescaler\n"); return ret; } @@ -330,7 +330,7 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } else { ret = clk_enable(atmel_pwm->clk); if (ret) { - dev_err(chip->dev, "failed to enable clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable clock\n"); return ret; } } @@ -478,7 +478,7 @@ static int atmel_pwm_enable_clk_if_on(struct pwm_chip *chip, bool on) for (i = 0; i < cnt; i++) { ret = clk_enable(atmel_pwm->clk); if (ret) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to enable clock for pwm %pe\n", ERR_PTR(ret)); From 8de8ccaca9f892d470145ff25b195c0d566421cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:57 +0100 Subject: [PATCH 0374/1038] pwm: atmel: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-atmel driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/f2a83af5660de461b4dcaf76929e9813bddfeff1.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index 920bad27fbee..b2f0abbbad63 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -77,7 +77,6 @@ struct atmel_pwm_data { }; struct atmel_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; const struct atmel_pwm_data *data; @@ -99,7 +98,7 @@ struct atmel_pwm_chip { static inline struct atmel_pwm_chip *to_atmel_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct atmel_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 atmel_pwm_readl(struct atmel_pwm_chip *chip, @@ -502,10 +501,11 @@ static int atmel_pwm_probe(struct platform_device *pdev) struct pwm_chip *chip; int ret; - atmel_pwm = devm_kzalloc(&pdev->dev, sizeof(*atmel_pwm), GFP_KERNEL); - if (!atmel_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 4, sizeof(*atmel_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + atmel_pwm = to_atmel_pwm_chip(chip); atmel_pwm->data = of_device_get_match_data(&pdev->dev); atmel_pwm->update_pending = 0; @@ -520,10 +520,7 @@ static int atmel_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(atmel_pwm->clk), "failed to get prepared PWM clock\n"); - chip = &atmel_pwm->chip; - chip->dev = &pdev->dev; chip->ops = &atmel_pwm_ops; - chip->npwm = 4; ret = atmel_pwm_enable_clk_if_on(chip, true); if (ret < 0) From 0ba76822eb65a92c0762d0b0fa79b45ad909d892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:58 +0100 Subject: [PATCH 0375/1038] pwm: atmel-hlcdc: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct atmel_hlcdc_pwm. Use the pwm_chip as driver data instead of the atmel_hlcdc_pwm to get access to the pwm_chip in the .suspend() and .resume() callbacks and atmel_hlcdc_pwm_remove() without using atmel->chip. Link: https://lore.kernel.org/r/0e97342f15540c7330d405eaaf3e68baa8e1e488.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-hlcdc.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/drivers/pwm/pwm-atmel-hlcdc.c b/drivers/pwm/pwm-atmel-hlcdc.c index 2d64af045fc5..718f1b1cce0b 100644 --- a/drivers/pwm/pwm-atmel-hlcdc.c +++ b/drivers/pwm/pwm-atmel-hlcdc.c @@ -182,8 +182,9 @@ static const struct atmel_hlcdc_pwm_errata atmel_hlcdc_pwm_sama5d3_errata = { static int atmel_hlcdc_pwm_suspend(struct device *dev) { - struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); - struct pwm_device *pwm = &atmel->chip.pwms[0]; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_hlcdc_pwm *atmel = to_atmel_hlcdc_pwm(chip); + struct pwm_device *pwm = &chip->pwms[0]; /* Keep the periph clock enabled if the PWM is still running. */ if (!pwm->state.enabled) @@ -194,8 +195,9 @@ static int atmel_hlcdc_pwm_suspend(struct device *dev) static int atmel_hlcdc_pwm_resume(struct device *dev) { - struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); - struct pwm_device *pwm = &atmel->chip.pwms[0]; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_hlcdc_pwm *atmel = to_atmel_hlcdc_pwm(chip); + struct pwm_device *pwm = &chip->pwms[0]; int ret; /* Re-enable the periph clock it was stopped during suspend. */ @@ -205,7 +207,7 @@ static int atmel_hlcdc_pwm_resume(struct device *dev) return ret; } - return atmel_hlcdc_pwm_apply(&atmel->chip, pwm, &pwm->state); + return atmel_hlcdc_pwm_apply(chip, pwm, &pwm->state); } static DEFINE_SIMPLE_DEV_PM_OPS(atmel_hlcdc_pwm_pm_ops, @@ -241,6 +243,7 @@ static int atmel_hlcdc_pwm_probe(struct platform_device *pdev) { const struct of_device_id *match; struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct atmel_hlcdc_pwm *atmel; struct atmel_hlcdc *hlcdc; int ret; @@ -260,26 +263,28 @@ static int atmel_hlcdc_pwm_probe(struct platform_device *pdev) atmel->errata = match->data; atmel->hlcdc = hlcdc; - atmel->chip.ops = &atmel_hlcdc_pwm_ops; - atmel->chip.dev = dev; - atmel->chip.npwm = 1; + chip = &atmel->chip; + chip->ops = &atmel_hlcdc_pwm_ops; + chip->dev = dev; + chip->npwm = 1; - ret = pwmchip_add(&atmel->chip); + ret = pwmchip_add(chip); if (ret) { clk_disable_unprepare(hlcdc->periph_clk); return ret; } - platform_set_drvdata(pdev, atmel); + platform_set_drvdata(pdev, chip); return 0; } static void atmel_hlcdc_pwm_remove(struct platform_device *pdev) { - struct atmel_hlcdc_pwm *atmel = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct atmel_hlcdc_pwm *atmel = to_atmel_hlcdc_pwm(chip); - pwmchip_remove(&atmel->chip); + pwmchip_remove(chip); clk_disable_unprepare(atmel->hlcdc->periph_clk); } From 93dcf8e00e16d76a5f0d689ebad13761a1213c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:30:59 +0100 Subject: [PATCH 0376/1038] pwm: atmel-hlcdc: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-atmel-hlcdc driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/4724c6a0f052160ac80ba5a3065c9470778b7457.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-hlcdc.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-atmel-hlcdc.c b/drivers/pwm/pwm-atmel-hlcdc.c index 718f1b1cce0b..2afb302be02c 100644 --- a/drivers/pwm/pwm-atmel-hlcdc.c +++ b/drivers/pwm/pwm-atmel-hlcdc.c @@ -28,7 +28,6 @@ struct atmel_hlcdc_pwm_errata { }; struct atmel_hlcdc_pwm { - struct pwm_chip chip; struct atmel_hlcdc *hlcdc; struct clk *cur_clk; const struct atmel_hlcdc_pwm_errata *errata; @@ -36,7 +35,7 @@ struct atmel_hlcdc_pwm { static inline struct atmel_hlcdc_pwm *to_atmel_hlcdc_pwm(struct pwm_chip *chip) { - return container_of(chip, struct atmel_hlcdc_pwm, chip); + return pwmchip_get_drvdata(chip); } static int atmel_hlcdc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -250,9 +249,10 @@ static int atmel_hlcdc_pwm_probe(struct platform_device *pdev) hlcdc = dev_get_drvdata(dev->parent); - atmel = devm_kzalloc(dev, sizeof(*atmel), GFP_KERNEL); - if (!atmel) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 1, sizeof(*atmel)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + atmel = to_atmel_hlcdc_pwm(chip); ret = clk_prepare_enable(hlcdc->periph_clk); if (ret) @@ -263,10 +263,7 @@ static int atmel_hlcdc_pwm_probe(struct platform_device *pdev) atmel->errata = match->data; atmel->hlcdc = hlcdc; - chip = &atmel->chip; chip->ops = &atmel_hlcdc_pwm_ops; - chip->dev = dev; - chip->npwm = 1; ret = pwmchip_add(chip); if (ret) { From 44fe6578b805527c4a3631801687bcb4088125bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:00 +0100 Subject: [PATCH 0377/1038] pwm: atmel-tcb: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/463d43cab01f97f3c4247c63cc08e76aa15684c4.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-tcb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c index d42c897cb85e..80121ab81a01 100644 --- a/drivers/pwm/pwm-atmel-tcb.c +++ b/drivers/pwm/pwm-atmel-tcb.c @@ -327,7 +327,7 @@ static int atmel_tcb_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if ((atcbpwm && atcbpwm->duty > 0 && atcbpwm->duty != atcbpwm->period) && (atcbpwm->div != i || atcbpwm->period != period)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to configure period_ns: PWM group already configured with a different value\n"); return -EINVAL; } From dfab73eb282969725f0c327652fd4d9c6f208e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:01 +0100 Subject: [PATCH 0378/1038] pwm: atmel-tcb: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct atmel_tcb_pwm_chip. Use the pwm_chip as driver data instead of the atmel_tcb_pwm_chip to get access to the pwm_chip in the .suspend() and .resume() callbacks and atmel_tcb_pwm_remove() without using tcbpwm->chip. Link: https://lore.kernel.org/r/6a90083e9d1ab1c34422161593d6d7a669143217.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-tcb.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c index 80121ab81a01..9d928e26b403 100644 --- a/drivers/pwm/pwm-atmel-tcb.c +++ b/drivers/pwm/pwm-atmel-tcb.c @@ -388,6 +388,7 @@ static const struct of_device_id atmel_tcb_of_match[] = { static int atmel_tcb_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; const struct of_device_id *match; struct atmel_tcb_pwm_chip *tcbpwm; const struct atmel_tcb_config *config; @@ -436,9 +437,10 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) } } - tcbpwm->chip.dev = &pdev->dev; - tcbpwm->chip.ops = &atmel_tcb_pwm_ops; - tcbpwm->chip.npwm = NPWM; + chip = &tcbpwm->chip; + chip->dev = &pdev->dev; + chip->ops = &atmel_tcb_pwm_ops; + chip->npwm = NPWM; tcbpwm->channel = channel; tcbpwm->width = config->counter_width; @@ -448,11 +450,11 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) spin_lock_init(&tcbpwm->lock); - err = pwmchip_add(&tcbpwm->chip); + err = pwmchip_add(chip); if (err < 0) goto err_disable_clk; - platform_set_drvdata(pdev, tcbpwm); + platform_set_drvdata(pdev, chip); return 0; @@ -473,9 +475,10 @@ err_slow_clk: static void atmel_tcb_pwm_remove(struct platform_device *pdev) { - struct atmel_tcb_pwm_chip *tcbpwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct atmel_tcb_pwm_chip *tcbpwm = to_tcb_chip(chip); - pwmchip_remove(&tcbpwm->chip); + pwmchip_remove(chip); clk_disable_unprepare(tcbpwm->slow_clk); clk_put(tcbpwm->gclk); @@ -491,7 +494,8 @@ MODULE_DEVICE_TABLE(of, atmel_tcb_pwm_dt_ids); static int atmel_tcb_pwm_suspend(struct device *dev) { - struct atmel_tcb_pwm_chip *tcbpwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_tcb_pwm_chip *tcbpwm = to_tcb_chip(chip); struct atmel_tcb_channel *chan = &tcbpwm->bkup; unsigned int channel = tcbpwm->channel; @@ -505,7 +509,8 @@ static int atmel_tcb_pwm_suspend(struct device *dev) static int atmel_tcb_pwm_resume(struct device *dev) { - struct atmel_tcb_pwm_chip *tcbpwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_tcb_pwm_chip *tcbpwm = to_tcb_chip(chip); struct atmel_tcb_channel *chan = &tcbpwm->bkup; unsigned int channel = tcbpwm->channel; From af184748c6807e95ce51388a5b162ad8fdebe04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:02 +0100 Subject: [PATCH 0379/1038] pwm: atmel-tcb: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-atmel-tcb driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/c845e6c9d27c8a4037755b2ae702b0039947a3c1.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-atmel-tcb.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c index 9d928e26b403..528e54c5999d 100644 --- a/drivers/pwm/pwm-atmel-tcb.c +++ b/drivers/pwm/pwm-atmel-tcb.c @@ -47,7 +47,6 @@ struct atmel_tcb_channel { }; struct atmel_tcb_pwm_chip { - struct pwm_chip chip; spinlock_t lock; u8 channel; u8 width; @@ -63,7 +62,7 @@ static const u8 atmel_tcb_divisors[] = { 2, 8, 32, 128, 0, }; static inline struct atmel_tcb_pwm_chip *to_tcb_chip(struct pwm_chip *chip) { - return container_of(chip, struct atmel_tcb_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int atmel_tcb_pwm_request(struct pwm_chip *chip, @@ -397,9 +396,10 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) int err; int channel; - tcbpwm = devm_kzalloc(&pdev->dev, sizeof(*tcbpwm), GFP_KERNEL); - if (tcbpwm == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, NPWM, sizeof(*tcbpwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + tcbpwm = to_tcb_chip(chip); err = of_property_read_u32(np, "reg", &channel); if (err < 0) { @@ -437,10 +437,7 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) } } - chip = &tcbpwm->chip; - chip->dev = &pdev->dev; chip->ops = &atmel_tcb_pwm_ops; - chip->npwm = NPWM; tcbpwm->channel = channel; tcbpwm->width = config->counter_width; From 19c09ed0943b46006ff22283936ae2e4d827e237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:03 +0100 Subject: [PATCH 0380/1038] pwm: bcm2835: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-bcm2835 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/441489772bdfbb2cd206fd534b36c63e8c50843a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-bcm2835.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c index 711c174bbe23..aa35acbb0cbc 100644 --- a/drivers/pwm/pwm-bcm2835.c +++ b/drivers/pwm/pwm-bcm2835.c @@ -24,7 +24,6 @@ #define PERIOD_MIN 0x2 struct bcm2835_pwm { - struct pwm_chip chip; void __iomem *base; struct clk *clk; unsigned long rate; @@ -32,7 +31,7 @@ struct bcm2835_pwm { static inline struct bcm2835_pwm *to_bcm2835_pwm(struct pwm_chip *chip) { - return container_of(chip, struct bcm2835_pwm, chip); + return pwmchip_get_drvdata(chip); } static int bcm2835_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -134,12 +133,14 @@ static void devm_clk_rate_exclusive_put(void *data) static int bcm2835_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct bcm2835_pwm *pc; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_bcm2835_pwm(chip); pc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->base)) @@ -165,14 +166,12 @@ static int bcm2835_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, -EINVAL, "failed to get clock rate\n"); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &bcm2835_pwm_ops; - pc->chip.atomic = true; - pc->chip.npwm = 2; + chip->ops = &bcm2835_pwm_ops; + chip->atomic = true; platform_set_drvdata(pdev, pc); - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add pwmchip\n"); From d502386c2b59d3df6a0ceba8d7bee031be5a0b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:04 +0100 Subject: [PATCH 0381/1038] pwm: bcm-iproc: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-bcm-iproc driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/cdc6136b94f685c85a615e9e5ccbd7ebb2f91cfc.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-bcm-iproc.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-bcm-iproc.c b/drivers/pwm/pwm-bcm-iproc.c index 758254025683..f4c9f10e490e 100644 --- a/drivers/pwm/pwm-bcm-iproc.c +++ b/drivers/pwm/pwm-bcm-iproc.c @@ -34,14 +34,13 @@ #define IPROC_PWM_PRESCALE_MAX 0x3f struct iproc_pwmc { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct iproc_pwmc *to_iproc_pwmc(struct pwm_chip *chip) { - return container_of(chip, struct iproc_pwmc, chip); + return pwmchip_get_drvdata(chip); } static void iproc_pwmc_enable(struct iproc_pwmc *ip, unsigned int channel) @@ -187,20 +186,20 @@ static const struct pwm_ops iproc_pwm_ops = { static int iproc_pwmc_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct iproc_pwmc *ip; unsigned int i; u32 value; int ret; - ip = devm_kzalloc(&pdev->dev, sizeof(*ip), GFP_KERNEL); - if (!ip) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 4, sizeof(*ip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + ip = to_iproc_pwmc(chip); platform_set_drvdata(pdev, ip); - ip->chip.dev = &pdev->dev; - ip->chip.ops = &iproc_pwm_ops; - ip->chip.npwm = 4; + chip->ops = &iproc_pwm_ops; ip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ip->base)) @@ -214,14 +213,14 @@ static int iproc_pwmc_probe(struct platform_device *pdev) /* Set full drive and normal polarity for all channels */ value = readl(ip->base + IPROC_PWM_CTRL_OFFSET); - for (i = 0; i < ip->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { value &= ~(1 << IPROC_PWM_CTRL_TYPE_SHIFT(i)); value |= 1 << IPROC_PWM_CTRL_POLARITY_SHIFT(i); } writel(value, ip->base + IPROC_PWM_CTRL_OFFSET); - ret = devm_pwmchip_add(&pdev->dev, &ip->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); From 2089ecaa49b1b610fdae9b59eec200065ca1fc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:05 +0100 Subject: [PATCH 0382/1038] pwm: bcm-kona: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/f0086ed37f30c4d25bf226e722f5ae52a73974d9.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-bcm-kona.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-bcm-kona.c b/drivers/pwm/pwm-bcm-kona.c index 45046a5c20a5..47acae8d7478 100644 --- a/drivers/pwm/pwm-bcm-kona.c +++ b/drivers/pwm/pwm-bcm-kona.c @@ -164,7 +164,7 @@ static int kona_pwmc_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_prepare_enable(kp->clk); if (ret < 0) { - dev_err(chip->dev, "failed to enable clock: %d\n", ret); + dev_err(pwmchip_parent(chip), "failed to enable clock: %d\n", ret); return ret; } @@ -193,7 +193,7 @@ static int kona_pwmc_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = clk_prepare_enable(kp->clk); if (ret < 0) { - dev_err(chip->dev, "failed to enable clock: %d\n", ret); + dev_err(pwmchip_parent(chip), "failed to enable clock: %d\n", ret); return ret; } From e5c368b821de56d7c824f72c0e93e8fdda62428c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:06 +0100 Subject: [PATCH 0383/1038] pwm: bcm-kona: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-bcm-kona driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/b50fe94dc72af602e7061839b24f48af6e005ef8.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-bcm-kona.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-bcm-kona.c b/drivers/pwm/pwm-bcm-kona.c index 47acae8d7478..022c078aae84 100644 --- a/drivers/pwm/pwm-bcm-kona.c +++ b/drivers/pwm/pwm-bcm-kona.c @@ -56,14 +56,13 @@ #define DUTY_CYCLE_HIGH_MAX 0x00ffffff struct kona_pwmc { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct kona_pwmc *to_kona_pwmc(struct pwm_chip *chip) { - return container_of(chip, struct kona_pwmc, chip); + return pwmchip_get_drvdata(chip); } /* @@ -273,18 +272,18 @@ static const struct pwm_ops kona_pwm_ops = { static int kona_pwmc_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct kona_pwmc *kp; unsigned int chan; unsigned int value = 0; int ret = 0; - kp = devm_kzalloc(&pdev->dev, sizeof(*kp), GFP_KERNEL); - if (kp == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 6, sizeof(*kp)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + kp = to_kona_pwmc(chip); - kp->chip.dev = &pdev->dev; - kp->chip.ops = &kona_pwm_ops; - kp->chip.npwm = 6; + chip->ops = &kona_pwm_ops; kp->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(kp->base)) @@ -304,14 +303,14 @@ static int kona_pwmc_probe(struct platform_device *pdev) } /* Set push/pull for all channels */ - for (chan = 0; chan < kp->chip.npwm; chan++) + for (chan = 0; chan < chip->npwm; chan++) value |= (1 << PWM_CONTROL_TYPE_SHIFT(chan)); writel(value, kp->base + PWM_CONTROL_OFFSET); clk_disable_unprepare(kp->clk); - ret = devm_pwmchip_add(&pdev->dev, &kp->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); From 5874eaf869806e4c94acbba4cb7df50a87dfe38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:07 +0100 Subject: [PATCH 0384/1038] pwm: berlin: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct berlin_pwm_chip. Use the pwm_chip as driver data instead of the berlin_pwm_chip to get access to the pwm_chip in berlin_pwm_suspend() and berlin_pwm_resume() without using bpc->chip. Link: https://lore.kernel.org/r/d2a0c5b664ef1bfd4719c645c069717d63fb4e65.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-berlin.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-berlin.c b/drivers/pwm/pwm-berlin.c index 442913232dc0..d910017b5feb 100644 --- a/drivers/pwm/pwm-berlin.c +++ b/drivers/pwm/pwm-berlin.c @@ -198,6 +198,7 @@ MODULE_DEVICE_TABLE(of, berlin_pwm_match); static int berlin_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct berlin_pwm_chip *bpc; int ret; @@ -213,25 +214,27 @@ static int berlin_pwm_probe(struct platform_device *pdev) if (IS_ERR(bpc->clk)) return PTR_ERR(bpc->clk); - bpc->chip.dev = &pdev->dev; - bpc->chip.ops = &berlin_pwm_ops; - bpc->chip.npwm = BERLIN_PWM_NUMPWMS; + chip = &bpc->chip; + chip->dev = &pdev->dev; + chip->ops = &berlin_pwm_ops; + chip->npwm = BERLIN_PWM_NUMPWMS; - ret = devm_pwmchip_add(&pdev->dev, &bpc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); - platform_set_drvdata(pdev, bpc); + platform_set_drvdata(pdev, chip); return 0; } static int berlin_pwm_suspend(struct device *dev) { - struct berlin_pwm_chip *bpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct berlin_pwm_chip *bpc = to_berlin_pwm_chip(chip); unsigned int i; - for (i = 0; i < bpc->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { struct berlin_pwm_channel *channel = &bpc->channel[i]; channel->enable = berlin_pwm_readl(bpc, i, BERLIN_PWM_ENABLE); @@ -247,7 +250,8 @@ static int berlin_pwm_suspend(struct device *dev) static int berlin_pwm_resume(struct device *dev) { - struct berlin_pwm_chip *bpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct berlin_pwm_chip *bpc = to_berlin_pwm_chip(chip); unsigned int i; int ret; @@ -255,7 +259,7 @@ static int berlin_pwm_resume(struct device *dev) if (ret) return ret; - for (i = 0; i < bpc->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { struct berlin_pwm_channel *channel = &bpc->channel[i]; berlin_pwm_writel(bpc, i, channel->ctrl, BERLIN_PWM_CONTROL); From bf756bfd243ec4da3069986280ef32fa5d85639f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:08 +0100 Subject: [PATCH 0385/1038] pwm: berlin: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-berlin driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/52866502c96a80d1c30be003dc1f5a89a4d230cc.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-berlin.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-berlin.c b/drivers/pwm/pwm-berlin.c index d910017b5feb..831aed228caf 100644 --- a/drivers/pwm/pwm-berlin.c +++ b/drivers/pwm/pwm-berlin.c @@ -49,7 +49,6 @@ struct berlin_pwm_channel { }; struct berlin_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct berlin_pwm_channel channel[BERLIN_PWM_NUMPWMS]; @@ -57,7 +56,7 @@ struct berlin_pwm_chip { static inline struct berlin_pwm_chip *to_berlin_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct berlin_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 berlin_pwm_readl(struct berlin_pwm_chip *bpc, @@ -202,9 +201,10 @@ static int berlin_pwm_probe(struct platform_device *pdev) struct berlin_pwm_chip *bpc; int ret; - bpc = devm_kzalloc(&pdev->dev, sizeof(*bpc), GFP_KERNEL); - if (!bpc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, BERLIN_PWM_NUMPWMS, sizeof(*bpc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + bpc = to_berlin_pwm_chip(chip); bpc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(bpc->base)) @@ -214,10 +214,7 @@ static int berlin_pwm_probe(struct platform_device *pdev) if (IS_ERR(bpc->clk)) return PTR_ERR(bpc->clk); - chip = &bpc->chip; - chip->dev = &pdev->dev; chip->ops = &berlin_pwm_ops; - chip->npwm = BERLIN_PWM_NUMPWMS; ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) From f3d1efd083b3142e512dde85dda8f315cd3ad905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:09 +0100 Subject: [PATCH 0386/1038] pwm: brcmstb: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-brcmstb driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/adf9cb04f5d84ae604e97d4dc0708ff3677d72d7.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-brcmstb.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-brcmstb.c b/drivers/pwm/pwm-brcmstb.c index 0fdeb0b2dbf3..82d27d07ba91 100644 --- a/drivers/pwm/pwm-brcmstb.c +++ b/drivers/pwm/pwm-brcmstb.c @@ -54,7 +54,6 @@ struct brcmstb_pwm { void __iomem *base; struct clk *clk; - struct pwm_chip chip; }; static inline u32 brcmstb_pwm_readl(struct brcmstb_pwm *p, @@ -77,7 +76,7 @@ static inline void brcmstb_pwm_writel(struct brcmstb_pwm *p, u32 value, static inline struct brcmstb_pwm *to_brcmstb_pwm(struct pwm_chip *chip) { - return container_of(chip, struct brcmstb_pwm, chip); + return pwmchip_get_drvdata(chip); } /* @@ -230,12 +229,14 @@ MODULE_DEVICE_TABLE(of, brcmstb_pwm_of_match); static int brcmstb_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct brcmstb_pwm *p; int ret; - p = devm_kzalloc(&pdev->dev, sizeof(*p), GFP_KERNEL); - if (!p) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*p)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + p = to_brcmstb_pwm(chip); p->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(p->clk)) @@ -244,15 +245,13 @@ static int brcmstb_pwm_probe(struct platform_device *pdev) platform_set_drvdata(pdev, p); - p->chip.dev = &pdev->dev; - p->chip.ops = &brcmstb_pwm_ops; - p->chip.npwm = 2; + chip->ops = &brcmstb_pwm_ops; p->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(p->base)) return PTR_ERR(p->base); - ret = devm_pwmchip_add(&pdev->dev, &p->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); From fc6549a9368302bb029058aa049cc472989e2735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:10 +0100 Subject: [PATCH 0387/1038] pwm: clk: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct pwm_clk_chip. Use the pwm_chip as driver data instead of the pwm_clk_chip to get access to the pwm_chip in pwm_clk_remove() without using pcchip->chip. Link: https://lore.kernel.org/r/cfbb7c571489ac3d9ca13835b3bd038aaca69638.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-clk.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-clk.c b/drivers/pwm/pwm-clk.c index 9dd88b386907..aef35d9a6a61 100644 --- a/drivers/pwm/pwm-clk.c +++ b/drivers/pwm/pwm-clk.c @@ -81,35 +81,38 @@ static const struct pwm_ops pwm_clk_ops = { static int pwm_clk_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_clk_chip *pcchip; int ret; pcchip = devm_kzalloc(&pdev->dev, sizeof(*pcchip), GFP_KERNEL); if (!pcchip) return -ENOMEM; + chip = &pcchip->chip; pcchip->clk = devm_clk_get_prepared(&pdev->dev, NULL); if (IS_ERR(pcchip->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(pcchip->clk), "Failed to get clock\n"); - pcchip->chip.dev = &pdev->dev; - pcchip->chip.ops = &pwm_clk_ops; - pcchip->chip.npwm = 1; + chip->dev = &pdev->dev; + chip->ops = &pwm_clk_ops; + chip->npwm = 1; - ret = pwmchip_add(&pcchip->chip); + ret = pwmchip_add(chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Failed to add pwm chip\n"); - platform_set_drvdata(pdev, pcchip); + platform_set_drvdata(pdev, chip); return 0; } static void pwm_clk_remove(struct platform_device *pdev) { - struct pwm_clk_chip *pcchip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct pwm_clk_chip *pcchip = to_pwm_clk_chip(chip); - pwmchip_remove(&pcchip->chip); + pwmchip_remove(chip); if (pcchip->clk_enabled) clk_disable(pcchip->clk); From 8e87e3dcb298d467e4f69c8c2565a3f9b836232c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:11 +0100 Subject: [PATCH 0388/1038] pwm: clk: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-clk driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_pwm_clk_chip() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/c02ccc955fe7a0aec7ce0ccaf5cd2bd902ae687f.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-clk.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-clk.c b/drivers/pwm/pwm-clk.c index aef35d9a6a61..c19a482d7e28 100644 --- a/drivers/pwm/pwm-clk.c +++ b/drivers/pwm/pwm-clk.c @@ -28,12 +28,14 @@ #include struct pwm_clk_chip { - struct pwm_chip chip; struct clk *clk; bool clk_enabled; }; -#define to_pwm_clk_chip(_chip) container_of(_chip, struct pwm_clk_chip, chip) +static inline struct pwm_clk_chip *to_pwm_clk_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int pwm_clk_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) @@ -85,19 +87,17 @@ static int pwm_clk_probe(struct platform_device *pdev) struct pwm_clk_chip *pcchip; int ret; - pcchip = devm_kzalloc(&pdev->dev, sizeof(*pcchip), GFP_KERNEL); - if (!pcchip) - return -ENOMEM; - chip = &pcchip->chip; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*pcchip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pcchip = to_pwm_clk_chip(chip); pcchip->clk = devm_clk_get_prepared(&pdev->dev, NULL); if (IS_ERR(pcchip->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(pcchip->clk), "Failed to get clock\n"); - chip->dev = &pdev->dev; chip->ops = &pwm_clk_ops; - chip->npwm = 1; ret = pwmchip_add(chip); if (ret < 0) From 5412170f10adeb11db6cf800f2a5612be88ea52e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:12 +0100 Subject: [PATCH 0389/1038] pwm: clps711x: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-clps711x driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/334e633bb8e4c26dc59883b068466387769b65f9.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-clps711x.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-clps711x.c b/drivers/pwm/pwm-clps711x.c index f3b4af7963be..c950e1dbd2b8 100644 --- a/drivers/pwm/pwm-clps711x.c +++ b/drivers/pwm/pwm-clps711x.c @@ -12,7 +12,6 @@ #include struct clps711x_chip { - struct pwm_chip chip; void __iomem *pmpcon; struct clk *clk; spinlock_t lock; @@ -20,7 +19,7 @@ struct clps711x_chip { static inline struct clps711x_chip *to_clps711x_chip(struct pwm_chip *chip) { - return container_of(chip, struct clps711x_chip, chip); + return pwmchip_get_drvdata(chip); } static int clps711x_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -76,11 +75,13 @@ static const struct pwm_ops clps711x_pwm_ops = { static int clps711x_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct clps711x_chip *priv; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_clps711x_chip(chip); priv->pmpcon = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->pmpcon)) @@ -90,13 +91,11 @@ static int clps711x_pwm_probe(struct platform_device *pdev) if (IS_ERR(priv->clk)) return PTR_ERR(priv->clk); - priv->chip.ops = &clps711x_pwm_ops; - priv->chip.dev = &pdev->dev; - priv->chip.npwm = 2; + chip->ops = &clps711x_pwm_ops; spin_lock_init(&priv->lock); - return devm_pwmchip_add(&pdev->dev, &priv->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static const struct of_device_id __maybe_unused clps711x_pwm_dt_ids[] = { From 4709f383083ae927f0e1a5e282ee9ace3ffb5f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:13 +0100 Subject: [PATCH 0390/1038] pwm: crc: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct crystalcove_pwm *crc_pwm just obtained from it. This also has the advantage of not using struct crystalcove_pwm::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/66d55649e13526bbd95d7bfd1cacfa0beb9efd43.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-crc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-crc.c b/drivers/pwm/pwm-crc.c index e09358901ab5..d985a9d3336d 100644 --- a/drivers/pwm/pwm-crc.c +++ b/drivers/pwm/pwm-crc.c @@ -55,7 +55,7 @@ static int crc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct crystalcove_pwm *crc_pwm = to_crc_pwm(chip); - struct device *dev = crc_pwm->chip.dev; + struct device *dev = chip->dev; int err; if (state->period > PWM_MAX_PERIOD_NS) { @@ -125,7 +125,7 @@ static int crc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { struct crystalcove_pwm *crc_pwm = to_crc_pwm(chip); - struct device *dev = crc_pwm->chip.dev; + struct device *dev = chip->dev; unsigned int clk_div, clk_div_reg, duty_cycle_reg; int error; From 5b86d1112b84f0736f1972316f4bf5a7cf657913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:14 +0100 Subject: [PATCH 0391/1038] pwm: crc: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/a41b62365295be9debd4a9aaa80ca87fca35b320.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-crc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-crc.c b/drivers/pwm/pwm-crc.c index d985a9d3336d..c8a852345a97 100644 --- a/drivers/pwm/pwm-crc.c +++ b/drivers/pwm/pwm-crc.c @@ -55,7 +55,7 @@ static int crc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct crystalcove_pwm *crc_pwm = to_crc_pwm(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int err; if (state->period > PWM_MAX_PERIOD_NS) { @@ -125,7 +125,7 @@ static int crc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { struct crystalcove_pwm *crc_pwm = to_crc_pwm(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); unsigned int clk_div, clk_div_reg, duty_cycle_reg; int error; From b1a80d5053f3d94b132642aacfcadc55782d1e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:15 +0100 Subject: [PATCH 0392/1038] pwm: crc: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-crc driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/bbe4896eabc240c678c66fabb6329f4e6cd04eda.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-crc.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-crc.c b/drivers/pwm/pwm-crc.c index c8a852345a97..98ee5cdbd0ba 100644 --- a/drivers/pwm/pwm-crc.c +++ b/drivers/pwm/pwm-crc.c @@ -26,17 +26,15 @@ /** * struct crystalcove_pwm - Crystal Cove PWM controller - * @chip: the abstract pwm_chip structure. * @regmap: the regmap from the parent device. */ struct crystalcove_pwm { - struct pwm_chip chip; struct regmap *regmap; }; static inline struct crystalcove_pwm *to_crc_pwm(struct pwm_chip *chip) { - return container_of(chip, struct crystalcove_pwm, chip); + return pwmchip_get_drvdata(chip); } static int crc_pwm_calc_clk_div(int period_ns) @@ -160,22 +158,22 @@ static const struct pwm_ops crc_pwm_ops = { static int crystalcove_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct crystalcove_pwm *crc_pwm; struct device *dev = pdev->dev.parent; struct intel_soc_pmic *pmic = dev_get_drvdata(dev); - crc_pwm = devm_kzalloc(&pdev->dev, sizeof(*crc_pwm), GFP_KERNEL); - if (!crc_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*crc_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + crc_pwm = to_crc_pwm(chip); - crc_pwm->chip.dev = &pdev->dev; - crc_pwm->chip.ops = &crc_pwm_ops; - crc_pwm->chip.npwm = 1; + chip->ops = &crc_pwm_ops; /* get the PMIC regmap */ crc_pwm->regmap = pmic->regmap; - return devm_pwmchip_add(&pdev->dev, &crc_pwm->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static struct platform_driver crystalcove_pwm_driver = { From 7256c2e79b8ed922d52d3487a2d5b022c6f37c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:16 +0100 Subject: [PATCH 0393/1038] pwm: cros-ec: Change prototype of helpers to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pwm_chip allocation and registration is about to change. For that the number of PWM devices must be known earlier in cros_ec_pwm_probe(). So make cros_ec_pwm_get_duty() and cros_ec_num_pwms() independent of struct cros_ec_pwm_device which is only available later. Reviewed-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/c0cfc75c1cba0f735555e0138387143ec101feb3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-cros-ec.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 339cedf3a7b1..e556f6218dd7 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -93,9 +93,8 @@ static int cros_ec_pwm_set_duty(struct cros_ec_pwm_device *ec_pwm, u8 index, return cros_ec_cmd_xfer_status(ec, msg); } -static int cros_ec_pwm_get_duty(struct cros_ec_pwm_device *ec_pwm, u8 index) +static int cros_ec_pwm_get_duty(struct cros_ec_device *ec, bool use_pwm_type, u8 index) { - struct cros_ec_device *ec = ec_pwm->ec; struct { struct cros_ec_command msg; union { @@ -115,7 +114,7 @@ static int cros_ec_pwm_get_duty(struct cros_ec_pwm_device *ec_pwm, u8 index) msg->insize = sizeof(*resp); msg->outsize = sizeof(*params); - if (ec_pwm->use_pwm_type) { + if (use_pwm_type) { ret = cros_ec_dt_type_to_pwm_type(index, ¶ms->pwm_type); if (ret) { dev_err(ec->dev, "Invalid PWM type index: %d\n", index); @@ -171,7 +170,7 @@ static int cros_ec_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct cros_ec_pwm *channel = &ec_pwm->channel[pwm->hwpwm]; int ret; - ret = cros_ec_pwm_get_duty(ec_pwm, pwm->hwpwm); + ret = cros_ec_pwm_get_duty(ec_pwm->ec, ec_pwm->use_pwm_type, pwm->hwpwm); if (ret < 0) { dev_err(chip->dev, "error getting initial duty: %d\n", ret); return ret; @@ -226,13 +225,17 @@ static const struct pwm_ops cros_ec_pwm_ops = { * of PWMs it supports directly, so we have to read the pwm duty cycle for * subsequent channels until we get an error. */ -static int cros_ec_num_pwms(struct cros_ec_pwm_device *ec_pwm) +static int cros_ec_num_pwms(struct cros_ec_device *ec) { int i, ret; /* The index field is only 8 bits */ for (i = 0; i <= U8_MAX; i++) { - ret = cros_ec_pwm_get_duty(ec_pwm, i); + /* + * Note that this function is only called when use_pwm_type is + * false. With use_pwm_type == true the number of PWMs is fixed. + */ + ret = cros_ec_pwm_get_duty(ec, false, i); /* * We look for SUCCESS, INVALID_COMMAND, or INVALID_PARAM * responses; everything else is treated as an error. @@ -283,7 +286,7 @@ static int cros_ec_pwm_probe(struct platform_device *pdev) if (ec_pwm->use_pwm_type) { chip->npwm = CROS_EC_PWM_DT_COUNT; } else { - ret = cros_ec_num_pwms(ec_pwm); + ret = cros_ec_num_pwms(ec); if (ret < 0) return dev_err_probe(dev, ret, "Couldn't find PWMs\n"); chip->npwm = ret; From 19a568a8d3c4dc7cec13c662d8a4a30363fdec9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:17 +0100 Subject: [PATCH 0394/1038] pwm: cros-ec: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Reviewed-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/486c40609bd9e339d7e5576e2501f63111b59b8b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-cros-ec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index e556f6218dd7..968e300e9f06 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -172,7 +172,7 @@ static int cros_ec_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, ret = cros_ec_pwm_get_duty(ec_pwm->ec, ec_pwm->use_pwm_type, pwm->hwpwm); if (ret < 0) { - dev_err(chip->dev, "error getting initial duty: %d\n", ret); + dev_err(pwmchip_parent(chip), "error getting initial duty: %d\n", ret); return ret; } From 452be9421eda39cfa13c2ace430c940ec60995b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:18 +0100 Subject: [PATCH 0395/1038] pwm: cros-ec: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-cros-ec driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. The probe function had to be changed a bit because the number of PWMs must be determined before allocation of the pwm_chip and its private data now. Reviewed-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/b0afe9e25340f0a274ff3806687cf37a6cc31557.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-cros-ec.c | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 968e300e9f06..606ccfdaf4cc 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -19,13 +19,11 @@ * struct cros_ec_pwm_device - Driver data for EC PWM * * @ec: Pointer to EC device - * @chip: PWM controller chip * @use_pwm_type: Use PWM types instead of generic channels * @channel: array with per-channel data */ struct cros_ec_pwm_device { struct cros_ec_device *ec; - struct pwm_chip chip; bool use_pwm_type; struct cros_ec_pwm *channel; }; @@ -40,7 +38,7 @@ struct cros_ec_pwm { static inline struct cros_ec_pwm_device *pwm_to_cros_ec_pwm(struct pwm_chip *chip) { - return container_of(chip, struct cros_ec_pwm_device, chip); + return pwmchip_get_drvdata(chip); } static int cros_ec_dt_type_to_pwm_type(u8 dt_index, u8 *pwm_type) @@ -264,34 +262,35 @@ static int cros_ec_pwm_probe(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; struct cros_ec_pwm_device *ec_pwm; struct pwm_chip *chip; + bool use_pwm_type = false; + unsigned int npwm; int ret; if (!ec) return dev_err_probe(dev, -EINVAL, "no parent EC device\n"); - ec_pwm = devm_kzalloc(dev, sizeof(*ec_pwm), GFP_KERNEL); - if (!ec_pwm) - return -ENOMEM; - chip = &ec_pwm->chip; - ec_pwm->ec = ec; - - if (of_device_is_compatible(np, "google,cros-ec-pwm-type")) - ec_pwm->use_pwm_type = true; - - /* PWM chip */ - chip->dev = dev; - chip->ops = &cros_ec_pwm_ops; - chip->of_xlate = cros_ec_pwm_xlate; - - if (ec_pwm->use_pwm_type) { - chip->npwm = CROS_EC_PWM_DT_COUNT; + if (of_device_is_compatible(np, "google,cros-ec-pwm-type")) { + use_pwm_type = true; + npwm = CROS_EC_PWM_DT_COUNT; } else { ret = cros_ec_num_pwms(ec); if (ret < 0) return dev_err_probe(dev, ret, "Couldn't find PWMs\n"); - chip->npwm = ret; + npwm = ret; } + chip = devm_pwmchip_alloc(dev, npwm, sizeof(*ec_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + ec_pwm = pwm_to_cros_ec_pwm(chip); + ec_pwm->use_pwm_type = use_pwm_type; + ec_pwm->ec = ec; + + /* PWM chip */ + chip->ops = &cros_ec_pwm_ops; + chip->of_xlate = cros_ec_pwm_xlate; + ec_pwm->channel = devm_kcalloc(dev, chip->npwm, sizeof(*ec_pwm->channel), GFP_KERNEL); if (!ec_pwm->channel) From 5319aaa9bb121b4a4fddabf207eae9c57dc62e19 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Thu, 15 Feb 2024 14:10:50 +0100 Subject: [PATCH 0396/1038] regulator: lp87565: Use bitfield helpers Use the FIELD_PREP() helper, instead open-coding the same operation. Signed-off-by: Geert Uytterhoeven Link: https://msgid.link/r/5e8781d31c983caf6bdabe36c5552077422cb8da.1708002575.git.geert+renesas@glider.be Signed-off-by: Mark Brown --- drivers/regulator/lp87565-regulator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/regulator/lp87565-regulator.c b/drivers/regulator/lp87565-regulator.c index 61ee5cf3f241..1259b5d20153 100644 --- a/drivers/regulator/lp87565-regulator.c +++ b/drivers/regulator/lp87565-regulator.c @@ -5,6 +5,7 @@ * Copyright (C) 2017 Texas Instruments Incorporated - https://www.ti.com/ */ +#include #include #include #include @@ -99,7 +100,7 @@ static int lp87565_buck_set_ramp_delay(struct regulator_dev *rdev, ret = regmap_update_bits(rdev->regmap, regulators[id].ctrl2_reg, LP87565_BUCK_CTRL_2_SLEW_RATE, - reg << __ffs(LP87565_BUCK_CTRL_2_SLEW_RATE)); + FIELD_PREP(LP87565_BUCK_CTRL_2_SLEW_RATE, reg)); if (ret) { dev_err(&rdev->dev, "SLEW RATE write failed: %d\n", ret); return ret; From 2601b6fb7d715fe5a808d12232ed9ae05fb86eef Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Thu, 15 Feb 2024 14:09:18 +0100 Subject: [PATCH 0397/1038] regulator: lp873x: Use bitfield helpers Use the FIELD_PREP() helper, instead open-coding the same operation. Signed-off-by: Geert Uytterhoeven Link: https://msgid.link/r/15edeaa9a644c4025234088da3dabb023fce04ac.1708002503.git.geert+renesas@glider.be Signed-off-by: Mark Brown --- drivers/regulator/lp873x-regulator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/regulator/lp873x-regulator.c b/drivers/regulator/lp873x-regulator.c index 8dfdd1db2070..84a134cfcd9c 100644 --- a/drivers/regulator/lp873x-regulator.c +++ b/drivers/regulator/lp873x-regulator.c @@ -5,6 +5,7 @@ * Copyright (C) 2016 Texas Instruments Incorporated - https://www.ti.com/ */ +#include #include #include #include @@ -93,7 +94,7 @@ static int lp873x_buck_set_ramp_delay(struct regulator_dev *rdev, ret = regmap_update_bits(lp873->regmap, regulators[id].ctrl2_reg, LP873X_BUCK0_CTRL_2_BUCK0_SLEW_RATE, - reg << __ffs(LP873X_BUCK0_CTRL_2_BUCK0_SLEW_RATE)); + FIELD_PREP(LP873X_BUCK0_CTRL_2_BUCK0_SLEW_RATE, reg)); if (ret) { dev_err(lp873->dev, "SLEW RATE write failed: %d\n", ret); return ret; From 125b28b18926e050e770160a5897b81082304f8c Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 15 Feb 2024 13:16:03 +0000 Subject: [PATCH 0398/1038] spi: dw: remove redundant assignment to variable len The variable id len being initialized with a value that is never read, it is being re-assigned later on in a for-loop. The initialization is redundant and can be removed. Cleans up clang scan build warning: drivers/spi/spi-dw-dma.c:580:17: warning: Although the value stored to 'len' is used in the enclosing expression, the value is never actually read from 'len' [deadcode.DeadStores] Signed-off-by: Colin Ian King Acked-by: Serge Semin Link: https://msgid.link/r/20240215131603.2062332-1-colin.i.king@gmail.com Signed-off-by: Mark Brown --- drivers/spi/spi-dw-dma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c index 0ecbb6c36e23..f4c209e5f52b 100644 --- a/drivers/spi/spi-dw-dma.c +++ b/drivers/spi/spi-dw-dma.c @@ -577,7 +577,7 @@ static int dw_spi_dma_transfer_one(struct dw_spi *dws, sg_init_table(&tx_tmp, 1); sg_init_table(&rx_tmp, 1); - for (base = 0, len = 0; base < xfer->len; base += len) { + for (base = 0; base < xfer->len; base += len) { /* Fetch next Tx DMA data chunk */ if (!tx_len) { tx_sg = !tx_sg ? &xfer->tx_sg.sgl[0] : sg_next(tx_sg); From 99769a52464d9f4c3b44bf903d08743cd53c49f6 Mon Sep 17 00:00:00 2001 From: Dhruva Gole Date: Thu, 15 Feb 2024 14:24:05 +0530 Subject: [PATCH 0399/1038] spi: Update the "master/slave" terminology in documentation Update the master/slave terminology wherever possible to adopt usage of the controller/host/target. Some parts have been left untouched because they were sysfs entries and will probably end up being inaccurate if simply replaced here. Signed-off-by: Dhruva Gole Link: https://msgid.link/r/20240215085404.1711976-1-d-gole@ti.com Signed-off-by: Mark Brown --- Documentation/spi/spi-summary.rst | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Documentation/spi/spi-summary.rst b/Documentation/spi/spi-summary.rst index e714472007f1..c8395cf7e4fc 100644 --- a/Documentation/spi/spi-summary.rst +++ b/Documentation/spi/spi-summary.rst @@ -19,14 +19,14 @@ commonly used. Each clock cycle shifts data out and data in; the clock doesn't cycle except when there is a data bit to shift. Not all data bits are used though; not every protocol uses those full duplex capabilities. -SPI hosts use a fourth "chip select" line to activate a given SPI slave +SPI hosts use a fourth "chip select" line to activate a given SPI target device, so those three signal wires may be connected to several chips -in parallel. All SPI slaves support chipselects; they are usually active -low signals, labeled nCSx for slave 'x' (e.g. nCS0). Some devices have +in parallel. All SPI targets support chipselects; they are usually active +low signals, labeled nCSx for target 'x' (e.g. nCS0). Some devices have other signals, often including an interrupt to the host. Unlike serial busses like USB or SMBus, even low level protocols for -SPI slave functions are usually not interoperable between vendors +SPI target functions are usually not interoperable between vendors (except for commodities like SPI memory chips). - SPI may be used for request/response style device protocols, as with @@ -43,8 +43,8 @@ SPI slave functions are usually not interoperable between vendors - Sometimes SPI is used to daisy-chain devices, like shift registers. -In the same way, SPI slaves will only rarely support any kind of automatic -discovery/enumeration protocol. The tree of slave devices accessible from +In the same way, SPI targets will only rarely support any kind of automatic +discovery/enumeration protocol. The tree of target devices accessible from a given SPI host controller will normally be set up manually, with configuration tables. @@ -75,7 +75,7 @@ protocol supported by every MMC or SD memory card. (The older "DataFlash" cards, predating MMC cards but using the same connectors and card shape, support only SPI.) Some PC hardware uses SPI flash for BIOS code. -SPI slave chips range from digital/analog converters used for analog +SPI target chips range from digital/analog converters used for analog sensors and codecs, to memory, to peripherals like USB controllers or Ethernet adapters; and more. @@ -119,7 +119,7 @@ trailing clock edge (CPHA=1), that's SPI mode 1. Note that the clock mode is relevant as soon as the chipselect goes active. So the host must set the clock to inactive before selecting -a slave, and the slave can tell the chosen polarity by sampling the +a target, and the target can tell the chosen polarity by sampling the clock level when its select line goes active. That's why many devices support for example both modes 0 and 3: they don't care about polarity, and always clock data in/out on rising clock edges. @@ -142,13 +142,13 @@ There are two types of SPI driver, here called: Controller drivers ... controllers may be built into System-On-Chip - processors, and often support both Master and Slave roles. + processors, and often support both Controller and target roles. These drivers touch hardware registers and may use DMA. Or they can be PIO bitbangers, needing just GPIO pins. Protocol drivers ... these pass messages through the controller - driver to communicate with a Slave or Master device on the + driver to communicate with a target or Controller device on the other side of an SPI link. So for example one protocol driver might talk to the MTD layer to export @@ -184,17 +184,17 @@ shows up in sysfs in several locations:: MOSI, and MISO. /sys/devices/.../CTLR/slave ... virtual file for (un)registering the - slave device for an SPI slave controller. - Writing the driver name of an SPI slave handler to this file - registers the slave device; writing "(null)" unregisters the slave + target device for an SPI target controller. + Writing the driver name of an SPI target handler to this file + registers the target device; writing "(null)" unregisters the target device. - Reading from this file shows the name of the slave device ("(null)" + Reading from this file shows the name of the target device ("(null)" if not registered). /sys/class/spi_slave/spiB ... symlink to a logical node which could hold - class related state for the SPI slave controller on bus "B". When + class related state for the SPI target controller on bus "B". When registered, a single spiB.* device is present here, possible sharing - the physical SPI bus segment with other SPI slave devices. + the physical SPI bus segment with other SPI target devices. At this time, the only class-specific state is the bus number ("B" in "spiB"), so those /sys/class entries are only useful to quickly identify busses. @@ -270,10 +270,10 @@ same SOC controller is used. For example, on one board SPI might use an external clock, where another derives the SPI clock from current settings of some master clock. -Declare Slave Devices +Declare target Devices ^^^^^^^^^^^^^^^^^^^^^ -The second kind of information is a list of what SPI slave devices exist +The second kind of information is a list of what SPI target devices exist on the target board, often with some board-specific data needed for the driver to work correctly. @@ -469,7 +469,7 @@ routines are available to allocate and zero-initialize an spi_message with several transfers. -How do I write an "SPI Master Controller Driver"? +How do I write an "SPI Controller Driver"? ------------------------------------------------- An SPI controller will probably be registered on the platform_bus; write a driver to bind to the device, whichever bus is involved. @@ -527,7 +527,7 @@ SPI Host Controller Methods Drivers may change the defaults provided by board_info, and then call spi_setup(spi) to invoke this routine. It may sleep. - Unless each SPI slave has its own configuration registers, don't + Unless each SPI target has its own configuration registers, don't change them right away ... otherwise drivers could corrupt I/O that's in progress for other SPI devices. From 4aee14527e4fd92d4a7487c430b54d099b99b4d1 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 14 Feb 2024 13:59:57 +0100 Subject: [PATCH 0400/1038] dt-bindings: mmc: renesas,sdhi: Document R-Car V4M support Document support for the SD Card/MMC Interface in the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Acked-by: Conor Dooley Link: https://lore.kernel.org/r/fffc5a0a73c4cc8e8d7c5d93679531cc24e006ca.1707915511.git.geert+renesas@glider.be Signed-off-by: Ulf Hansson --- Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml index f7a4c6bc70f6..29f2400247eb 100644 --- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml +++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml @@ -67,6 +67,7 @@ properties: - renesas,sdhi-r8a779a0 # R-Car V3U - renesas,sdhi-r8a779f0 # R-Car S4-8 - renesas,sdhi-r8a779g0 # R-Car V4H + - renesas,sdhi-r8a779h0 # R-Car V4M - const: renesas,rcar-gen4-sdhi # R-Car Gen4 reg: From 25e69172db8a31ef8564ce1cf755ac5cb8374daa Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Thu, 15 Feb 2024 14:28:18 +0100 Subject: [PATCH 0401/1038] mmc: davinci_mmc: Drop dangling variable The sg_miter conversion left a dangling unused variable. Drop it. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402142042.vg0lnLdb-lkp@intel.com/ Fixes: ed01d210fd91 ("mmc: davinci_mmc: Use sg_miter for PIO") Signed-off-by: Linus Walleij Acked-by: Bartosz Golaszewski Link: https://lore.kernel.org/r/20240215-mmc-fix-davinci-v1-1-a593678ca7bf@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/davinci_mmc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c index c46577305138..8bd938919687 100644 --- a/drivers/mmc/host/davinci_mmc.c +++ b/drivers/mmc/host/davinci_mmc.c @@ -211,7 +211,6 @@ static void davinci_fifo_data_trans(struct mmc_davinci_host *host, unsigned int n) { struct sg_mapping_iter *sgm = &host->sg_miter; - size_t sglen; u8 *p; unsigned int i; @@ -224,7 +223,6 @@ static void davinci_fifo_data_trans(struct mmc_davinci_host *host, return; } p = sgm->addr; - sglen = sgm->length; /* NOTE: we never transfer more than rw_threshold bytes * to/from the fifo here; there's no I/O overlap. From e18afcb7b2a12b635ac10081f943fcf84ddacc51 Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Tue, 13 Feb 2024 01:41:58 +0100 Subject: [PATCH 0402/1038] ACPI: processor_idle: Fix memory leak in acpi_processor_power_exit() After unregistering the CPU idle device, the memory associated with it is not freed, leading to a memory leak: unreferenced object 0xffff896282f6c000 (size 1024): comm "swapper/0", pid 1, jiffies 4294893170 hex dump (first 32 bytes): 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace (crc 8836a742): [] kmalloc_trace+0x29d/0x340 [] acpi_processor_power_init+0xf3/0x1c0 [] __acpi_processor_start+0xd3/0xf0 [] acpi_processor_start+0x2c/0x50 [] really_probe+0xe2/0x480 [] __driver_probe_device+0x78/0x160 [] driver_probe_device+0x1f/0x90 [] __driver_attach+0xce/0x1c0 [] bus_for_each_dev+0x70/0xc0 [] bus_add_driver+0x112/0x210 [] driver_register+0x55/0x100 [] acpi_processor_driver_init+0x3b/0xc0 [] do_one_initcall+0x41/0x300 [] kernel_init_freeable+0x320/0x470 [] kernel_init+0x16/0x1b0 [] ret_from_fork+0x2d/0x50 Fix this by freeing the CPU idle device after unregistering it. Fixes: 3d339dcbb56d ("cpuidle / ACPI : move cpuidle_device field out of the acpi_processor_power structure") Signed-off-by: Armin Wolf Signed-off-by: Rafael J. Wysocki --- drivers/acpi/processor_idle.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 55437f5e0c3a..bd6a7857ce05 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -1430,6 +1430,8 @@ int acpi_processor_power_exit(struct acpi_processor *pr) acpi_processor_registered--; if (acpi_processor_registered == 0) cpuidle_unregister_driver(&acpi_idle_driver); + + kfree(dev); } pr->flags.power_setup_done = 0; From 0793e511c4c66c38dd26add86f7236bcdc70c3b5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Harasymchuk Date: Sun, 11 Feb 2024 01:08:07 +0100 Subject: [PATCH 0403/1038] ACPI: resource: Add IRQ override quirk for ASUS ExpertBook B2502FBA In order to fix the keyboard on ASUS ExpertBook B2502FBA, add an IRQ override quirk for it in analogy with how it was done for other members of this machine family. Link: https://lore.kernel.org/linux-acpi/20230411183144.6932-1-pmenzel@molgen.mpg.de Link: https://bugzilla.kernel.org/show_bug.cgi?id=217323 Signed-off-by: Sviatoslav Harasymchuk [ rjw: Subject and changelog rewrite, fix broken white space ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 65ce43ecfa8c..c99b0cf5e22e 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -496,6 +496,13 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "B2502CBA"), }, }, + { + /* Asus ExpertBook B2502FBA */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "B2502FBA"), + }, + }, { /* Asus Vivobook E1504GA */ .matches = { From 1269b6d7222f761b6f5fb85b19f7ab76a5bbf803 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 13 Feb 2024 15:46:05 +0200 Subject: [PATCH 0404/1038] ACPI: utils: Make acpi_handle_path() not static acpi_handle_path() will soon be required for node name comparison elsewhere in ACPI framework. Remove the static keyword and add the prototype to include/linux/acpi.h. Signed-off-by: Sakari Ailus Signed-off-by: Rafael J. Wysocki --- drivers/acpi/utils.c | 2 +- include/linux/acpi.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index abac5cc25477..202234ba54bd 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -559,7 +559,7 @@ EXPORT_SYMBOL(acpi_evaluate_ost); * * Caller must free the returned buffer */ -static char *acpi_handle_path(acpi_handle handle) +char *acpi_handle_path(acpi_handle handle) { struct acpi_buffer buffer = { .length = ACPI_ALLOCATE_BUFFER, diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b7165e52b3c6..a170c389dd74 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1170,6 +1170,7 @@ static inline void acpi_ec_set_gpe_wake_mask(u8 action) {} #endif #ifdef CONFIG_ACPI +char *acpi_handle_path(acpi_handle handle); __printf(3, 4) void acpi_handle_printk(const char *level, acpi_handle handle, const char *fmt, ...); From 5bd4edbbf920b0ab72b19f486e20fb552e794a88 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Tue, 13 Feb 2024 15:46:06 +0200 Subject: [PATCH 0405/1038] ACPI: property: Ignore bad graph port nodes on Dell XPS 9315 Some systems were shipped with both Windows and Linux camera descriptions. In general, if Linux description exist, they will be used and Windows description ignored. In this case the Linux descriptions were buggy so use Windows definition instead. This patch ignores the bad graph port nodes on Dell XPS 9315 and there are likely other such systems, too. The corresponding information has been added to ipu-bridge to cover the missing bits. Signed-off-by: Sakari Ailus Signed-off-by: Rafael J. Wysocki --- drivers/acpi/internal.h | 1 + drivers/acpi/mipi-disco-img.c | 71 +++++++++++++++++++++++++++++++++++ drivers/acpi/property.c | 3 ++ 3 files changed, 75 insertions(+) diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 6588525c45ef..e0145df519bd 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -301,5 +301,6 @@ void acpi_mipi_check_crs_csi2(acpi_handle handle); void acpi_mipi_scan_crs_csi2(void); void acpi_mipi_init_crs_csi2_swnodes(void); void acpi_mipi_crs_csi2_cleanup(void); +bool acpi_graph_ignore_port(acpi_handle handle); #endif /* _ACPI_INTERNAL_H_ */ diff --git a/drivers/acpi/mipi-disco-img.c b/drivers/acpi/mipi-disco-img.c index 7286cf4579bc..da71eb4bf6a6 100644 --- a/drivers/acpi/mipi-disco-img.c +++ b/drivers/acpi/mipi-disco-img.c @@ -19,6 +19,7 @@ */ #include +#include #include #include #include @@ -723,3 +724,73 @@ void acpi_mipi_crs_csi2_cleanup(void) list_for_each_entry_safe(csi2, csi2_tmp, &acpi_mipi_crs_csi2_list, entry) acpi_mipi_del_crs_csi2(csi2); } + +static const struct dmi_system_id dmi_ignore_port_nodes[] = { + { + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 9315"), + }, + }, + { 0 } +}; + +static const char *strnext(const char *s1, const char *s2) +{ + s1 = strstr(s1, s2); + + if (!s1) + return NULL; + + return s1 + strlen(s2); +} + +/** + * acpi_graph_ignore_port - Tell whether a port node should be ignored + * @handle: The ACPI handle of the node (which may be a port node) + * + * Returns true if a port node should be ignored and the data to that should + * come from other sources instead (Windows ACPI definitions and + * ipu-bridge). This is currently used to ignore bad port nodes related to IPU6 + * ("IPU?") and camera sensor devices ("LNK?") in certain Dell systems with + * Intel VSC. + */ +bool acpi_graph_ignore_port(acpi_handle handle) +{ + const char *path = NULL, *orig_path; + static bool dmi_tested, ignore_port; + + if (!dmi_tested) { + ignore_port = dmi_first_match(dmi_ignore_port_nodes); + dmi_tested = true; + } + + if (!ignore_port) + return false; + + /* Check if the device is either "IPU" or "LNK" (sensor). */ + orig_path = acpi_handle_path(handle); + if (!orig_path) + return false; + path = strnext(orig_path, "IPU"); + if (!path) + path = strnext(orig_path, "LNK"); + if (!path) + goto out_free; + + if (!(path[0] >= '0' && path[0] <= '9' && path[1] == '.')) + goto out_free; + + /* Check if the node has a "PRT" prefix. */ + path = strnext(path, "PRT"); + if (path && path[0] >= '0' && path[0] <= '9' && !path[1]) { + acpi_handle_debug(handle, "ignoring data node\n"); + + kfree(orig_path); + return true; + } + +out_free: + kfree(orig_path); + return false; +} diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index a6ead5204046..2b73580c9f36 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -80,6 +80,9 @@ static bool acpi_nondev_subnode_extract(union acpi_object *desc, struct acpi_data_node *dn; bool result; + if (acpi_graph_ignore_port(handle)) + return false; + dn = kzalloc(sizeof(*dn), GFP_KERNEL); if (!dn) return false; From bad5247a2c4f7eab6fb922af3362740a562dc665 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:49 +0100 Subject: [PATCH 0406/1038] ima: Align ima_inode_post_setattr() definition with LSM infrastructure Change ima_inode_post_setattr() definition, so that it can be registered as implementation of the inode_post_setattr hook (to be introduced). Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 +- include/linux/ima.h | 4 ++-- security/integrity/ima/ima_appraise.c | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/attr.c b/fs/attr.c index 5a13f0c8495f..b53ae408ad4f 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -502,7 +502,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); - ima_inode_post_setattr(idmap, dentry); + ima_inode_post_setattr(idmap, dentry, ia_valid); evm_inode_post_setattr(dentry, ia_valid); } diff --git a/include/linux/ima.h b/include/linux/ima.h index 86b57757c7b1..910a2f11a906 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -186,7 +186,7 @@ static inline void ima_post_key_create_or_update(struct key *keyring, #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); extern void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry); + struct dentry *dentry, int ia_valid); extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len); extern int ima_inode_set_acl(struct mnt_idmap *idmap, @@ -206,7 +206,7 @@ static inline bool is_ima_appraise_enabled(void) } static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry) + struct dentry *dentry, int ia_valid) { return; } diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 870dde67707b..36c2938a5c69 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -629,6 +629,7 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * ima_inode_post_setattr - reflect file metadata changes * @idmap: idmap of the mount the inode was found from * @dentry: pointer to the affected dentry + * @ia_valid: for the UID and GID status * * Changes to a dentry's metadata might result in needing to appraise. * @@ -636,7 +637,7 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * to lock the inode's i_mutex. */ void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry) + struct dentry *dentry, int ia_valid) { struct inode *inode = d_backing_inode(dentry); struct integrity_iint_cache *iint; From 0298c5a9b168f0d74ea3bf881301c4bd9252d367 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:50 +0100 Subject: [PATCH 0407/1038] ima: Align ima_file_mprotect() definition with LSM infrastructure Change ima_file_mprotect() definition, so that it can be registered as implementation of the file_mprotect hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 5 +++-- security/integrity/ima/ima_main.c | 6 ++++-- security/security.c | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/linux/ima.h b/include/linux/ima.h index 910a2f11a906..b66353f679e8 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -23,7 +23,8 @@ extern void ima_post_create_tmpfile(struct mnt_idmap *idmap, extern void ima_file_free(struct file *file); extern int ima_file_mmap(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags); -extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot); +extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot); extern int ima_load_data(enum kernel_load_data_id id, bool contents); extern int ima_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description); @@ -84,7 +85,7 @@ static inline int ima_file_mmap(struct file *file, unsigned long reqprot, } static inline int ima_file_mprotect(struct vm_area_struct *vma, - unsigned long prot) + unsigned long reqprot, unsigned long prot) { return 0; } diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index cc1217ac2c6f..b3f5e8401056 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -455,7 +455,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, /** * ima_file_mprotect - based on policy, limit mprotect change * @vma: vm_area_struct protection is set to - * @prot: contains the protection that will be applied by the kernel. + * @reqprot: protection requested by the application + * @prot: protection that will be applied by the kernel * * Files can be mmap'ed read/write and later changed to execute to circumvent * IMA's mmap appraisal policy rules. Due to locking issues (mmap semaphore @@ -465,7 +466,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, * * On mprotect change success, return 0. On failure, return -EACESS. */ -int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) +int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot) { struct ima_template_desc *template = NULL; struct file *file; diff --git a/security/security.c b/security/security.c index 3aaad75c9ce8..36933ac45d3b 100644 --- a/security/security.c +++ b/security/security.c @@ -2831,7 +2831,7 @@ int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, ret = call_int_hook(file_mprotect, 0, vma, reqprot, prot); if (ret) return ret; - return ima_file_mprotect(vma, prot); + return ima_file_mprotect(vma, reqprot, prot); } /** From fbd0506e5c5874fd52403fd38e3e77d895689870 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:51 +0100 Subject: [PATCH 0408/1038] ima: Align ima_inode_setxattr() definition with LSM infrastructure Change ima_inode_setxattr() definition, so that it can be registered as implementation of the inode_setxattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 11 +++++++---- security/integrity/ima/ima_appraise.c | 5 +++-- security/security.c | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/include/linux/ima.h b/include/linux/ima.h index b66353f679e8..077324309c11 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -188,8 +188,9 @@ static inline void ima_post_key_create_or_update(struct key *keyring, extern bool is_ima_appraise_enabled(void); extern void ima_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid); -extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len); +extern int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags); extern int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); @@ -212,10 +213,12 @@ static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, return; } -static inline int ima_inode_setxattr(struct dentry *dentry, +static inline int ima_inode_setxattr(struct mnt_idmap *idmap, + struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) + size_t xattr_value_len, + int flags) { return 0; } diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 36c2938a5c69..cb2d0d11aa77 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -750,8 +750,9 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; } -int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) +int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xvalue = xattr_value; int digsig = 0; diff --git a/security/security.c b/security/security.c index 36933ac45d3b..b4f6b7467e08 100644 --- a/security/security.c +++ b/security/security.c @@ -2269,7 +2269,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap, ret = cap_inode_setxattr(dentry, name, value, size, flags); if (ret) return ret; - ret = ima_inode_setxattr(dentry, name, value, size); + ret = ima_inode_setxattr(idmap, dentry, name, value, size, flags); if (ret) return ret; return evm_inode_setxattr(idmap, dentry, name, value, size); From 526864dd2f60c16bcdc84e9e7bc6d69d08cfee21 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:52 +0100 Subject: [PATCH 0409/1038] ima: Align ima_inode_removexattr() definition with LSM infrastructure Change ima_inode_removexattr() definition, so that it can be registered as implementation of the inode_removexattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 7 +++++-- security/integrity/ima/ima_appraise.c | 3 ++- security/security.c | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/linux/ima.h b/include/linux/ima.h index 077324309c11..678a03fddd7e 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -200,7 +200,9 @@ static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, { return ima_inode_set_acl(idmap, dentry, acl_name, NULL); } -extern int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name); + +extern int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name); #else static inline bool is_ima_appraise_enabled(void) { @@ -231,7 +233,8 @@ static inline int ima_inode_set_acl(struct mnt_idmap *idmap, return 0; } -static inline int ima_inode_removexattr(struct dentry *dentry, +static inline int ima_inode_removexattr(struct mnt_idmap *idmap, + struct dentry *dentry, const char *xattr_name) { return 0; diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index cb2d0d11aa77..36abc84ba299 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -790,7 +790,8 @@ int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return 0; } -int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name) +int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { int result; diff --git a/security/security.c b/security/security.c index b4f6b7467e08..58e20362b679 100644 --- a/security/security.c +++ b/security/security.c @@ -2430,7 +2430,7 @@ int security_inode_removexattr(struct mnt_idmap *idmap, ret = cap_inode_removexattr(idmap, dentry, name); if (ret) return ret; - ret = ima_inode_removexattr(dentry, name); + ret = ima_inode_removexattr(idmap, dentry, name); if (ret) return ret; return evm_inode_removexattr(idmap, dentry, name); From fec5f85e468d6f47851d531ec81f9da821768e00 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:53 +0100 Subject: [PATCH 0410/1038] ima: Align ima_post_read_file() definition with LSM infrastructure Change ima_post_read_file() definition, by making "void *buf" a "char *buf", so that it can be registered as implementation of the post_read_file hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 4 ++-- security/integrity/ima/ima_main.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/ima.h b/include/linux/ima.h index 678a03fddd7e..31ef6c3c3207 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -30,7 +30,7 @@ extern int ima_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description); extern int ima_read_file(struct file *file, enum kernel_read_file_id id, bool contents); -extern int ima_post_read_file(struct file *file, void *buf, loff_t size, +extern int ima_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id); extern void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry); @@ -108,7 +108,7 @@ static inline int ima_read_file(struct file *file, enum kernel_read_file_id id, return 0; } -static inline int ima_post_read_file(struct file *file, void *buf, loff_t size, +static inline int ima_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id) { return 0; diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index b3f5e8401056..02021ee467d3 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -803,7 +803,7 @@ const int read_idmap[READING_MAX_ID] = { * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_read_file(struct file *file, void *buf, loff_t size, +int ima_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id read_id) { enum ima_hooks func; From 784111d0093e007950cc20033daf3d74ac388821 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:54 +0100 Subject: [PATCH 0411/1038] evm: Align evm_inode_post_setattr() definition with LSM infrastructure Change evm_inode_post_setattr() definition, so that it can be registered as implementation of the inode_post_setattr hook (to be introduced). Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 +- include/linux/evm.h | 6 ++++-- security/integrity/evm/evm_main.c | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fs/attr.c b/fs/attr.c index b53ae408ad4f..adeba0ec40f1 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -503,7 +503,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); ima_inode_post_setattr(idmap, dentry, ia_valid); - evm_inode_post_setattr(dentry, ia_valid); + evm_inode_post_setattr(idmap, dentry, ia_valid); } return error; diff --git a/include/linux/evm.h b/include/linux/evm.h index 36ec884320d9..5cc386312b5a 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -23,7 +23,8 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry, struct integrity_iint_cache *iint); extern int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); -extern void evm_inode_post_setattr(struct dentry *dentry, int ia_valid); +extern void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid); extern int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, size_t size); @@ -98,7 +99,8 @@ static inline int evm_inode_setattr(struct mnt_idmap *idmap, return 0; } -static inline void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) +static inline void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { return; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index cc7956d7878b..ac34f21122cd 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -870,6 +870,7 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /** * evm_inode_post_setattr - update 'security.evm' after modifying metadata + * @idmap: idmap of the idmapped mount * @dentry: pointer to the affected dentry * @ia_valid: for the UID and GID status * @@ -879,7 +880,8 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) +void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) { if (!evm_revalidate_status(NULL)) return; From 2b6a4054f8c2758cf5c1d78f6ba7006a940b31ce Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:55 +0100 Subject: [PATCH 0412/1038] evm: Align evm_inode_setxattr() definition with LSM infrastructure Change evm_inode_setxattr() definition, so that it can be registered as implementation of the inode_setxattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/evm.h | 4 ++-- security/integrity/evm/evm_main.c | 3 ++- security/security.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/linux/evm.h b/include/linux/evm.h index 5cc386312b5a..7de24c1ada90 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -27,7 +27,7 @@ extern void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid); extern int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, - const void *value, size_t size); + const void *value, size_t size, int flags); extern void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, @@ -107,7 +107,7 @@ static inline void evm_inode_post_setattr(struct mnt_idmap *idmap, static inline int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, - const void *value, size_t size) + const void *value, size_t size, int flags) { return 0; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index ac34f21122cd..12ba3207fd31 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -581,6 +581,7 @@ out: * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value * @xattr_value_len: pointer to the new extended attribute value length + * @flags: flags to pass into filesystem operations * * Before allowing the 'security.evm' protected xattr to be updated, * verify the existing value is valid. As only the kernel should have @@ -590,7 +591,7 @@ out: */ int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xattr_data = xattr_value; diff --git a/security/security.c b/security/security.c index 58e20362b679..b99c9a69d4c2 100644 --- a/security/security.c +++ b/security/security.c @@ -2272,7 +2272,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap, ret = ima_inode_setxattr(idmap, dentry, name, value, size, flags); if (ret) return ret; - return evm_inode_setxattr(idmap, dentry, name, value, size); + return evm_inode_setxattr(idmap, dentry, name, value, size, flags); } /** From 779cb1947e270504c66a96fc0b7e6e31c748b1e1 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:56 +0100 Subject: [PATCH 0413/1038] evm: Align evm_inode_post_setxattr() definition with LSM infrastructure Change evm_inode_post_setxattr() definition, so that it can be registered as implementation of the inode_post_setxattr hook. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/evm.h | 8 +++++--- security/integrity/evm/evm_main.c | 4 +++- security/security.c | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/evm.h b/include/linux/evm.h index 7de24c1ada90..3faabdd47852 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -31,7 +31,8 @@ extern int evm_inode_setxattr(struct mnt_idmap *idmap, extern void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len); + size_t xattr_value_len, + int flags); extern int evm_inode_copy_up_xattr(const char *name); extern int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *xattr_name); @@ -56,7 +57,7 @@ static inline void evm_inode_post_set_acl(struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) { - return evm_inode_post_setxattr(dentry, acl_name, NULL, 0); + return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); } int evm_inode_init_security(struct inode *inode, struct inode *dir, @@ -115,7 +116,8 @@ static inline int evm_inode_setxattr(struct mnt_idmap *idmap, static inline void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) + size_t xattr_value_len, + int flags) { return; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 12ba3207fd31..d35143179699 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -753,6 +753,7 @@ bool evm_revalidate_status(const char *xattr_name) * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value * @xattr_value_len: pointer to the new extended attribute value length + * @flags: flags to pass into filesystem operations * * Update the HMAC stored in 'security.evm' to reflect the change. * @@ -761,7 +762,8 @@ bool evm_revalidate_status(const char *xattr_name) * i_mutex lock. */ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) + const void *xattr_value, size_t xattr_value_len, + int flags) { if (!evm_revalidate_status(xattr_name)) return; diff --git a/security/security.c b/security/security.c index b99c9a69d4c2..dc188201a4f5 100644 --- a/security/security.c +++ b/security/security.c @@ -2367,7 +2367,7 @@ void security_inode_post_setxattr(struct dentry *dentry, const char *name, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return; call_void_hook(inode_post_setxattr, dentry, name, value, size, flags); - evm_inode_post_setxattr(dentry, name, value, size); + evm_inode_post_setxattr(dentry, name, value, size, flags); } /** From 314a8dc728d038378795236f6b5199265f921f45 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:57 +0100 Subject: [PATCH 0414/1038] security: Align inode_setattr hook definition with EVM Add the idmap parameter to the definition, so that evm_inode_setattr() can be registered as this hook implementation. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 3 ++- security/security.c | 2 +- security/selinux/hooks.c | 3 ++- security/smack/smack_lsm.c | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 76458b6d53da..b00b16d58413 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -135,7 +135,8 @@ LSM_HOOK(int, 0, inode_readlink, struct dentry *dentry) LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, bool rcu) LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) -LSM_HOOK(int, 0, inode_setattr, struct dentry *dentry, struct iattr *attr) +LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) LSM_HOOK(int, 0, inode_getattr, const struct path *path) LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, diff --git a/security/security.c b/security/security.c index dc188201a4f5..9dc601d45960 100644 --- a/security/security.c +++ b/security/security.c @@ -2215,7 +2215,7 @@ int security_inode_setattr(struct mnt_idmap *idmap, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_setattr, 0, dentry, attr); + ret = call_int_hook(inode_setattr, 0, idmap, dentry, attr); if (ret) return ret; return evm_inode_setattr(idmap, dentry, attr); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index a6bf90ace84c..cedb4cbf072e 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3136,7 +3136,8 @@ static int selinux_inode_permission(struct inode *inode, int mask) return rc; } -static int selinux_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int selinux_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) { const struct cred *cred = current_cred(); struct inode *inode = d_backing_inode(dentry); diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 0fdbf04cc258..1b6abfdf7173 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -1233,12 +1233,14 @@ static int smack_inode_permission(struct inode *inode, int mask) /** * smack_inode_setattr - Smack check for setting attributes + * @idmap: idmap of the mount * @dentry: the object * @iattr: for the force flag * * Returns 0 if access is permitted, an error code otherwise */ -static int smack_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int smack_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) { struct smk_audit_info ad; int rc; From 77fa6f314f0376176ef6bf3d84403e0d8b54ce28 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:58 +0100 Subject: [PATCH 0415/1038] security: Introduce inode_post_setattr hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_setattr hook. At inode_setattr hook, EVM verifies the file's existing HMAC value. At inode_post_setattr, EVM re-calculates the file's HMAC based on the modified file attributes and other file metadata. Other LSMs could similarly take some action after successful file attribute change. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Casey Schaufler Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/attr.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 7 +++++++ security/security.c | 16 ++++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/fs/attr.c b/fs/attr.c index adeba0ec40f1..990e1b3a3c91 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -502,6 +502,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); + security_inode_post_setattr(idmap, dentry, ia_valid); ima_inode_post_setattr(idmap, dentry, ia_valid); evm_inode_post_setattr(idmap, dentry, ia_valid); } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index b00b16d58413..a0e9e48015a4 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -137,6 +137,8 @@ LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) +LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) LSM_HOOK(int, 0, inode_getattr, const struct path *path) LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, diff --git a/include/linux/security.h b/include/linux/security.h index d0eb20f90b26..56c841aa3994 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -361,6 +361,8 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode, int security_inode_permission(struct inode *inode, int mask); int security_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); +void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid); int security_inode_getattr(const struct path *path); int security_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, @@ -879,6 +881,11 @@ static inline int security_inode_setattr(struct mnt_idmap *idmap, return 0; } +static inline void +security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) +{ } + static inline int security_inode_getattr(const struct path *path) { return 0; diff --git a/security/security.c b/security/security.c index 9dc601d45960..56527d5415e2 100644 --- a/security/security.c +++ b/security/security.c @@ -2222,6 +2222,22 @@ int security_inode_setattr(struct mnt_idmap *idmap, } EXPORT_SYMBOL_GPL(security_inode_setattr); +/** + * security_inode_post_setattr() - Update the inode after a setattr operation + * @idmap: idmap of the mount + * @dentry: file + * @ia_valid: file attributes set + * + * Update inode security field after successful setting file attributes. + */ +void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_setattr, idmap, dentry, ia_valid); +} + /** * security_inode_getattr() - Check if getting file attributes is allowed * @path: file From dae52cbf5887ac51c3574648124cfe475a9b3246 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:30:59 +0100 Subject: [PATCH 0416/1038] security: Introduce inode_post_removexattr hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_removexattr hook. At inode_removexattr hook, EVM verifies the file's existing HMAC value. At inode_post_removexattr, EVM re-calculates the file's HMAC with the passed xattr removed and other file metadata. Other LSMs could similarly take some action after successful xattr removal. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/xattr.c | 9 +++++---- include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 5 +++++ security/security.c | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/fs/xattr.c b/fs/xattr.c index 09d927603433..f891c260a971 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -552,11 +552,12 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap, goto out; error = __vfs_removexattr(idmap, dentry, name); + if (error) + return error; - if (!error) { - fsnotify_xattr(dentry); - evm_inode_post_removexattr(dentry, name); - } + fsnotify_xattr(dentry); + security_inode_post_removexattr(dentry, name); + evm_inode_post_removexattr(dentry, name); out: return error; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index a0e9e48015a4..f849f7d5bb53 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -149,6 +149,8 @@ LSM_HOOK(int, 0, inode_getxattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_listxattr, struct dentry *dentry) LSM_HOOK(int, 0, inode_removexattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name) +LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry, + const char *name) LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, diff --git a/include/linux/security.h b/include/linux/security.h index 56c841aa3994..84ae03690340 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -380,6 +380,7 @@ int security_inode_getxattr(struct dentry *dentry, const char *name); int security_inode_listxattr(struct dentry *dentry); int security_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name); +void security_inode_post_removexattr(struct dentry *dentry, const char *name); int security_inode_need_killpriv(struct dentry *dentry); int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry); int security_inode_getsecurity(struct mnt_idmap *idmap, @@ -942,6 +943,10 @@ static inline int security_inode_removexattr(struct mnt_idmap *idmap, return cap_inode_removexattr(idmap, dentry, name); } +static inline void security_inode_post_removexattr(struct dentry *dentry, + const char *name) +{ } + static inline int security_inode_need_killpriv(struct dentry *dentry) { return cap_inode_need_killpriv(dentry); diff --git a/security/security.c b/security/security.c index 56527d5415e2..d77fdd3ca1bc 100644 --- a/security/security.c +++ b/security/security.c @@ -2452,6 +2452,20 @@ int security_inode_removexattr(struct mnt_idmap *idmap, return evm_inode_removexattr(idmap, dentry, name); } +/** + * security_inode_post_removexattr() - Update the inode after a removexattr op + * @dentry: file + * @name: xattr name + * + * Update the inode after a successful removexattr operation. + */ +void security_inode_post_removexattr(struct dentry *dentry, const char *name) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_removexattr, dentry, name); +} + /** * security_inode_need_killpriv() - Check if security_inode_killpriv() required * @dentry: associated dentry From 8f46ff5767b0b18329140d80d6bcabd818f42c4c Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:00 +0100 Subject: [PATCH 0417/1038] security: Introduce file_post_open hook In preparation to move IMA and EVM to the LSM infrastructure, introduce the file_post_open hook. Also, export security_file_post_open() for NFS. Based on policy, IMA calculates the digest of the file content and extends the TPM with the digest, verifies the file's integrity based on the digest, and/or includes the file digest in the audit log. LSMs could similarly take action depending on the file content and the access mask requested with open(). The new hook returns a value and can cause the open to be aborted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/namei.c | 2 ++ fs/nfsd/vfs.c | 6 ++++++ include/linux/lsm_hook_defs.h | 1 + include/linux/security.h | 6 ++++++ security/security.c | 17 +++++++++++++++++ 5 files changed, 32 insertions(+) diff --git a/fs/namei.c b/fs/namei.c index 4e0de939fea1..ef867f1d6704 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3639,6 +3639,8 @@ static int do_open(struct nameidata *nd, error = may_open(idmap, &nd->path, acc_mode, open_flag); if (!error && !(file->f_mode & FMODE_OPENED)) error = vfs_open(&nd->path, file); + if (!error) + error = security_file_post_open(file, op->acc_mode); if (!error) error = ima_file_check(file, op->acc_mode); if (!error && do_truncate) diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index b7c7a9273ea0..e44d8239545b 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -877,6 +877,12 @@ __nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, goto out; } + host_err = security_file_post_open(file, may_flags); + if (host_err) { + fput(file); + goto out; + } + host_err = ima_file_check(file, may_flags); if (host_err) { fput(file); diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index f849f7d5bb53..3c84942d2818 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -191,6 +191,7 @@ LSM_HOOK(int, 0, file_send_sigiotask, struct task_struct *tsk, struct fown_struct *fown, int sig) LSM_HOOK(int, 0, file_receive, struct file *file) LSM_HOOK(int, 0, file_open, struct file *file) +LSM_HOOK(int, 0, file_post_open, struct file *file, int mask) LSM_HOOK(int, 0, file_truncate, struct file *file) LSM_HOOK(int, 0, task_alloc, struct task_struct *task, unsigned long clone_flags) diff --git a/include/linux/security.h b/include/linux/security.h index 84ae03690340..97f2212c13b6 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -411,6 +411,7 @@ int security_file_send_sigiotask(struct task_struct *tsk, struct fown_struct *fown, int sig); int security_file_receive(struct file *file); int security_file_open(struct file *file); +int security_file_post_open(struct file *file, int mask); int security_file_truncate(struct file *file); int security_task_alloc(struct task_struct *task, unsigned long clone_flags); void security_task_free(struct task_struct *task); @@ -1074,6 +1075,11 @@ static inline int security_file_open(struct file *file) return 0; } +static inline int security_file_post_open(struct file *file, int mask) +{ + return 0; +} + static inline int security_file_truncate(struct file *file) { return 0; diff --git a/security/security.c b/security/security.c index d77fdd3ca1bc..207405a84902 100644 --- a/security/security.c +++ b/security/security.c @@ -2966,6 +2966,23 @@ int security_file_open(struct file *file) return fsnotify_open_perm(file); } +/** + * security_file_post_open() - Evaluate a file after it has been opened + * @file: the file + * @mask: access mask + * + * Evaluate an opened file and the access mask requested with open(). The hook + * is useful for LSMs that require the file content to be available in order to + * make decisions. + * + * Return: Returns 0 if permission is granted. + */ +int security_file_post_open(struct file *file, int mask) +{ + return call_int_hook(file_post_open, 0, file, mask); +} +EXPORT_SYMBOL_GPL(security_file_post_open); + /** * security_file_truncate() - Check if truncating a file is allowed * @file: file From f09068b5a114ed28d2df2e82a7d30dde0145dc69 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:01 +0100 Subject: [PATCH 0418/1038] security: Introduce file_release hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the file_release hook. IMA calculates at file close the new digest of the file content and writes it to security.ima, so that appraisal at next file access succeeds. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Acked-by: Christian Brauner Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/file_table.c | 1 + include/linux/lsm_hook_defs.h | 1 + include/linux/security.h | 4 ++++ security/security.c | 11 +++++++++++ 4 files changed, 17 insertions(+) diff --git a/fs/file_table.c b/fs/file_table.c index b991f90571b4..725407f374fa 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -367,6 +367,7 @@ static void __fput(struct file *file) eventpoll_release(file); locks_remove_file(file); + security_file_release(file); ima_file_free(file); if (unlikely(file->f_flags & FASYNC)) { if (file->f_op->fasync) diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 3c84942d2818..7f9e9240606e 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -173,6 +173,7 @@ LSM_HOOK(int, 0, kernfs_init_security, struct kernfs_node *kn_dir, struct kernfs_node *kn) LSM_HOOK(int, 0, file_permission, struct file *file, int mask) LSM_HOOK(int, 0, file_alloc_security, struct file *file) +LSM_HOOK(void, LSM_RET_VOID, file_release, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file) LSM_HOOK(int, 0, file_ioctl, struct file *file, unsigned int cmd, unsigned long arg) diff --git a/include/linux/security.h b/include/linux/security.h index 97f2212c13b6..2997348afcb7 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -395,6 +395,7 @@ int security_kernfs_init_security(struct kernfs_node *kn_dir, struct kernfs_node *kn); int security_file_permission(struct file *file, int mask); int security_file_alloc(struct file *file); +void security_file_release(struct file *file); void security_file_free(struct file *file); int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); int security_file_ioctl_compat(struct file *file, unsigned int cmd, @@ -1008,6 +1009,9 @@ static inline int security_file_alloc(struct file *file) return 0; } +static inline void security_file_release(struct file *file) +{ } + static inline void security_file_free(struct file *file) { } diff --git a/security/security.c b/security/security.c index 207405a84902..99004c44ce55 100644 --- a/security/security.c +++ b/security/security.c @@ -2718,6 +2718,17 @@ int security_file_alloc(struct file *file) return rc; } +/** + * security_file_release() - Perform actions before releasing the file ref + * @file: the file + * + * Perform actions before releasing the last reference to a file. + */ +void security_file_release(struct file *file) +{ + call_void_hook(file_release, file); +} + /** * security_file_free() - Free a file's LSM blob * @file: the file From 08abce60d63fb55f440c393f4508e99064f2fd91 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:02 +0100 Subject: [PATCH 0419/1038] security: Introduce path_post_mknod hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the path_post_mknod hook. IMA-appraisal requires all existing files in policy to have a file hash/signature stored in security.ima. An exception is made for empty files created by mknod, by tagging them as new files. LSMs could also take some action after files are created. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Reviewed-by: Stefan Berger Signed-off-by: Paul Moore --- fs/namei.c | 5 +++++ include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 5 +++++ security/security.c | 14 ++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/fs/namei.c b/fs/namei.c index ef867f1d6704..9280aa5d60a7 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -4063,6 +4063,11 @@ retry: dentry, mode, 0); break; } + + if (error) + goto out2; + + security_path_post_mknod(idmap, dentry); out2: done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 7f9e9240606e..dba5d8204dc5 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -94,6 +94,8 @@ LSM_HOOK(int, 0, path_mkdir, const struct path *dir, struct dentry *dentry, LSM_HOOK(int, 0, path_rmdir, const struct path *dir, struct dentry *dentry) LSM_HOOK(int, 0, path_mknod, const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev) +LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, struct mnt_idmap *idmap, + struct dentry *dentry) LSM_HOOK(int, 0, path_truncate, const struct path *path) LSM_HOOK(int, 0, path_symlink, const struct path *dir, struct dentry *dentry, const char *old_name) diff --git a/include/linux/security.h b/include/linux/security.h index 2997348afcb7..977dd9f7f51a 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1893,6 +1893,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, umode_t m int security_path_rmdir(const struct path *dir, struct dentry *dentry); int security_path_mknod(const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev); +void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry); int security_path_truncate(const struct path *path); int security_path_symlink(const struct path *dir, struct dentry *dentry, const char *old_name); @@ -1927,6 +1928,10 @@ static inline int security_path_mknod(const struct path *dir, struct dentry *den return 0; } +static inline void security_path_post_mknod(struct mnt_idmap *idmap, + struct dentry *dentry) +{ } + static inline int security_path_truncate(const struct path *path) { return 0; diff --git a/security/security.c b/security/security.c index 99004c44ce55..ee4582e1d4d0 100644 --- a/security/security.c +++ b/security/security.c @@ -1800,6 +1800,20 @@ int security_path_mknod(const struct path *dir, struct dentry *dentry, } EXPORT_SYMBOL(security_path_mknod); +/** + * security_path_post_mknod() - Update inode security field after file creation + * @idmap: idmap of the mount + * @dentry: new file + * + * Update inode security field after a file has been created. + */ +void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(path_post_mknod, idmap, dentry); +} + /** * security_path_mkdir() - Check if creating a new directory is allowed * @dir: parent directory From a7811e34d100acf24870eb949c5ae3e49dde18b9 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:03 +0100 Subject: [PATCH 0420/1038] security: Introduce inode_post_create_tmpfile hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_create_tmpfile hook. As temp files can be made persistent, treat new temp files like other new files, so that the file hash is calculated and stored in the security xattr. LSMs could also take some action after temp files have been created. The new hook cannot return an error and cannot cause the operation to be canceled. Signed-off-by: Roberto Sassu Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Reviewed-by: Stefan Berger Signed-off-by: Paul Moore --- fs/namei.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 6 ++++++ security/security.c | 15 +++++++++++++++ 4 files changed, 24 insertions(+) diff --git a/fs/namei.c b/fs/namei.c index 9280aa5d60a7..c7ac7fdf0f25 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3705,6 +3705,7 @@ static int vfs_tmpfile(struct mnt_idmap *idmap, inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } + security_inode_post_create_tmpfile(idmap, inode); ima_post_create_tmpfile(idmap, inode); return 0; } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index dba5d8204dc5..87f60b47dfca 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -121,6 +121,8 @@ LSM_HOOK(int, 0, inode_init_security_anon, struct inode *inode, const struct qstr *name, const struct inode *context_inode) LSM_HOOK(int, 0, inode_create, struct inode *dir, struct dentry *dentry, umode_t mode) +LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, struct mnt_idmap *idmap, + struct inode *inode) LSM_HOOK(int, 0, inode_link, struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) LSM_HOOK(int, 0, inode_unlink, struct inode *dir, struct dentry *dentry) diff --git a/include/linux/security.h b/include/linux/security.h index 977dd9f7f51a..1cb604282617 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -344,6 +344,8 @@ int security_inode_init_security_anon(struct inode *inode, const struct qstr *name, const struct inode *context_inode); int security_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode); +void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode); int security_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry); int security_inode_unlink(struct inode *dir, struct dentry *dentry); @@ -811,6 +813,10 @@ static inline int security_inode_create(struct inode *dir, return 0; } +static inline void +security_inode_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode) +{ } + static inline int security_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) diff --git a/security/security.c b/security/security.c index ee4582e1d4d0..710db090aa8b 100644 --- a/security/security.c +++ b/security/security.c @@ -2013,6 +2013,21 @@ int security_inode_create(struct inode *dir, struct dentry *dentry, } EXPORT_SYMBOL_GPL(security_inode_create); +/** + * security_inode_post_create_tmpfile() - Update inode security of new tmpfile + * @idmap: idmap of the mount + * @inode: inode of the new tmpfile + * + * Update inode security data after a tmpfile has been created. + */ +void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode) +{ + if (unlikely(IS_PRIVATE(inode))) + return; + call_void_hook(inode_post_create_tmpfile, idmap, inode); +} + /** * security_inode_link() - Check if creating a hard link is allowed * @old_dentry: existing file From 8b9d0b825c6573d654c8b8039ea79920926305c2 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:04 +0100 Subject: [PATCH 0421/1038] security: Introduce inode_post_set_acl hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_set_acl hook. At inode_set_acl hook, EVM verifies the file's existing HMAC value. At inode_post_set_acl, EVM re-calculates the file's HMAC based on the modified POSIX ACL and other file metadata. Other LSMs could similarly take some action after successful POSIX ACL change. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/posix_acl.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 7 +++++++ security/security.c | 17 +++++++++++++++++ 4 files changed, 27 insertions(+) diff --git a/fs/posix_acl.c b/fs/posix_acl.c index e1af20893ebe..7556ee9f8a51 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -1137,6 +1137,7 @@ retry_deleg: error = -EIO; if (!error) { fsnotify_xattr(dentry); + security_inode_post_set_acl(dentry, acl_name, kacl); evm_inode_post_set_acl(dentry, acl_name, kacl); } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 87f60b47dfca..b0125c99f80a 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -157,6 +157,8 @@ LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) +LSM_HOOK(void, LSM_RET_VOID, inode_post_set_acl, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap, diff --git a/include/linux/security.h b/include/linux/security.h index 1cb604282617..c372797e1617 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -372,6 +372,8 @@ int security_inode_setxattr(struct mnt_idmap *idmap, int security_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); +void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl); int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); int security_inode_remove_acl(struct mnt_idmap *idmap, @@ -915,6 +917,11 @@ static inline int security_inode_set_acl(struct mnt_idmap *idmap, return 0; } +static inline void security_inode_post_set_acl(struct dentry *dentry, + const char *acl_name, + struct posix_acl *kacl) +{ } + static inline int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) diff --git a/security/security.c b/security/security.c index 710db090aa8b..52f62f785087 100644 --- a/security/security.c +++ b/security/security.c @@ -2350,6 +2350,23 @@ int security_inode_set_acl(struct mnt_idmap *idmap, return evm_inode_set_acl(idmap, dentry, acl_name, kacl); } +/** + * security_inode_post_set_acl() - Update inode security from posix acls set + * @dentry: file + * @acl_name: acl name + * @kacl: acl struct + * + * Update inode security data after successfully setting posix acls on @dentry. + * The posix acls in @kacl are identified by @acl_name. + */ +void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_set_acl, dentry, acl_name, kacl); +} + /** * security_inode_get_acl() - Check if reading posix acls is allowed * @idmap: idmap of the mount From 2d705d8024143c272a764320c880ccd3230bb699 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:05 +0100 Subject: [PATCH 0422/1038] security: Introduce inode_post_remove_acl hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the inode_post_remove_acl hook. At inode_remove_acl hook, EVM verifies the file's existing HMAC value. At inode_post_remove_acl, EVM re-calculates the file's HMAC with the passed POSIX ACL removed and other file metadata. Other LSMs could similarly take some action after successful POSIX ACL removal. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Acked-by: Christian Brauner Signed-off-by: Paul Moore --- fs/posix_acl.c | 1 + include/linux/lsm_hook_defs.h | 2 ++ include/linux/security.h | 8 ++++++++ security/security.c | 17 +++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 7556ee9f8a51..0d2371240c1b 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -1246,6 +1246,7 @@ retry_deleg: error = -EIO; if (!error) { fsnotify_xattr(dentry); + security_inode_post_remove_acl(idmap, dentry, acl_name); evm_inode_post_remove_acl(idmap, dentry, acl_name); } diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index b0125c99f80a..7e414ba26333 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -163,6 +163,8 @@ LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) +LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_need_killpriv, struct dentry *dentry) LSM_HOOK(int, 0, inode_killpriv, struct mnt_idmap *idmap, struct dentry *dentry) diff --git a/include/linux/security.h b/include/linux/security.h index c372797e1617..4b03c76b91f1 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -378,6 +378,9 @@ int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); int security_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); +void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name); void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); int security_inode_getxattr(struct dentry *dentry, const char *name); @@ -936,6 +939,11 @@ static inline int security_inode_remove_acl(struct mnt_idmap *idmap, return 0; } +static inline void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name) +{ } + static inline void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { } diff --git a/security/security.c b/security/security.c index 52f62f785087..3bed660fc950 100644 --- a/security/security.c +++ b/security/security.c @@ -2413,6 +2413,23 @@ int security_inode_remove_acl(struct mnt_idmap *idmap, return evm_inode_remove_acl(idmap, dentry, acl_name); } +/** + * security_inode_post_remove_acl() - Update inode security after rm posix acls + * @idmap: idmap of the mount + * @dentry: file + * @acl_name: acl name + * + * Update inode security data after successfully removing posix acls on + * @dentry in @idmap. The posix acls are identified by @acl_name. + */ +void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_remove_acl, idmap, dentry, acl_name); +} + /** * security_inode_post_setxattr() - Update the inode after a setxattr operation * @dentry: file From b8d997032a46fcf47d5bda011c0d1e87b20c08ba Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:06 +0100 Subject: [PATCH 0423/1038] security: Introduce key_post_create_or_update hook In preparation for moving IMA and EVM to the LSM infrastructure, introduce the key_post_create_or_update hook. Depending on policy, IMA measures the key content after creation or update, so that remote verifiers are aware of the operation. Other LSMs could similarly take some action after successful key creation or update. The new hook cannot return an error and cannot cause the operation to be reverted. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Acked-by: Casey Schaufler Reviewed-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 3 +++ include/linux/security.h | 11 +++++++++++ security/keys/key.c | 7 ++++++- security/security.c | 19 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 7e414ba26333..7e4683035d34 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -405,6 +405,9 @@ LSM_HOOK(void, LSM_RET_VOID, key_free, struct key *key) LSM_HOOK(int, 0, key_permission, key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm) LSM_HOOK(int, 0, key_getsecurity, struct key *key, char **buffer) +LSM_HOOK(void, LSM_RET_VOID, key_post_create_or_update, struct key *keyring, + struct key *key, const void *payload, size_t payload_len, + unsigned long flags, bool create) #endif /* CONFIG_KEYS */ #ifdef CONFIG_AUDIT diff --git a/include/linux/security.h b/include/linux/security.h index 4b03c76b91f1..8436f9abf43d 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2004,6 +2004,9 @@ void security_key_free(struct key *key); int security_key_permission(key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm); int security_key_getsecurity(struct key *key, char **_buffer); +void security_key_post_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t payload_len, + unsigned long flags, bool create); #else @@ -2031,6 +2034,14 @@ static inline int security_key_getsecurity(struct key *key, char **_buffer) return 0; } +static inline void security_key_post_create_or_update(struct key *keyring, + struct key *key, + const void *payload, + size_t payload_len, + unsigned long flags, + bool create) +{ } + #endif #endif /* CONFIG_KEYS */ diff --git a/security/keys/key.c b/security/keys/key.c index 5b10641debd5..31a8b9408b7c 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -930,6 +930,8 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, goto error_link_end; } + security_key_post_create_or_update(keyring, key, payload, plen, flags, + true); ima_post_key_create_or_update(keyring, key, payload, plen, flags, true); @@ -963,10 +965,13 @@ error: key_ref = __key_update(key_ref, &prep); - if (!IS_ERR(key_ref)) + if (!IS_ERR(key_ref)) { + security_key_post_create_or_update(keyring, key, payload, plen, + flags, false); ima_post_key_create_or_update(keyring, key, payload, plen, flags, false); + } goto error_free_prep; } diff --git a/security/security.c b/security/security.c index 3bed660fc950..6c23c620e3c1 100644 --- a/security/security.c +++ b/security/security.c @@ -5453,6 +5453,25 @@ int security_key_getsecurity(struct key *key, char **buffer) *buffer = NULL; return call_int_hook(key_getsecurity, 0, key, buffer); } + +/** + * security_key_post_create_or_update() - Notification of key create or update + * @keyring: keyring to which the key is linked to + * @key: created or updated key + * @payload: data used to instantiate or update the key + * @payload_len: length of payload + * @flags: key flags + * @create: flag indicating whether the key was created or updated + * + * Notify the caller of a key creation or update. + */ +void security_key_post_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t payload_len, + unsigned long flags, bool create) +{ + call_void_hook(key_post_create_or_update, keyring, key, payload, + payload_len, flags, create); +} #endif /* CONFIG_KEYS */ #ifdef CONFIG_AUDIT From 06cca5110774f7b59a1685431ac697865588f4ca Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:07 +0100 Subject: [PATCH 0424/1038] integrity: Move integrity_kernel_module_request() to IMA In preparation for removing the 'integrity' LSM, move integrity_kernel_module_request() to IMA, and rename it to ima_kernel_module_request(). Rewrite the function documentation, to explain better what the problem is. Compile it conditionally if CONFIG_INTEGRITY_ASYMMETRIC_KEYS is enabled, and call it from security.c (removed afterwards with the move of IMA to the LSM infrastructure). Adding this hook cannot be avoided, since IMA has no control on the flags passed to crypto_alloc_sig() in public_key_verify_signature(), and thus cannot pass CRYPTO_NOLOAD, which solved the problem for EVM hashing with commit e2861fa71641 ("evm: Don't deadlock if a crypto algorithm is unavailable"). EVM alone does not need to implement this hook, first because there is no mutex to deadlock, and second because even if it had it, there should be a recursive call. However, since verification from EVM can be initiated only by setting inode metadata, deadlock would occur if modprobe would do the same while loading a kernel module (which is unlikely). Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/ima.h | 10 ++++++++ include/linux/integrity.h | 13 ---------- security/integrity/digsig_asymmetric.c | 23 ------------------ security/integrity/ima/ima_main.c | 33 ++++++++++++++++++++++++++ security/security.c | 2 +- 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/include/linux/ima.h b/include/linux/ima.h index 31ef6c3c3207..0f9af283cbc8 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -256,4 +256,14 @@ static inline bool ima_appraise_signature(enum kernel_read_file_id func) return false; } #endif /* CONFIG_IMA_APPRAISE && CONFIG_INTEGRITY_TRUSTED_KEYRING */ + +#if defined(CONFIG_IMA) && defined(CONFIG_INTEGRITY_ASYMMETRIC_KEYS) +extern int ima_kernel_module_request(char *kmod_name); +#else +static inline int ima_kernel_module_request(char *kmod_name) +{ + return 0; +} + +#endif #endif /* _LINUX_IMA_H */ diff --git a/include/linux/integrity.h b/include/linux/integrity.h index 2ea0f2f65ab6..ef0f63ef5ebc 100644 --- a/include/linux/integrity.h +++ b/include/linux/integrity.h @@ -42,17 +42,4 @@ static inline void integrity_load_keys(void) } #endif /* CONFIG_INTEGRITY */ -#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS - -extern int integrity_kernel_module_request(char *kmod_name); - -#else - -static inline int integrity_kernel_module_request(char *kmod_name) -{ - return 0; -} - -#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */ - #endif /* _LINUX_INTEGRITY_H */ diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index 895f4b9ce8c6..de603cf42ac7 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -132,26 +132,3 @@ out: pr_debug("%s() = %d\n", __func__, ret); return ret; } - -/** - * integrity_kernel_module_request - prevent crypto-pkcs1pad(rsa,*) requests - * @kmod_name: kernel module name - * - * We have situation, when public_key_verify_signature() in case of RSA - * algorithm use alg_name to store internal information in order to - * construct an algorithm on the fly, but crypto_larval_lookup() will try - * to use alg_name in order to load kernel module with same name. - * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, - * we are safe to fail such module request from crypto_larval_lookup(). - * - * In this way we prevent modprobe execution during digsig verification - * and avoid possible deadlock if modprobe and/or it's dependencies - * also signed with digsig. - */ -int integrity_kernel_module_request(char *kmod_name) -{ - if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) - return -EINVAL; - - return 0; -} diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 02021ee467d3..3891b83efdb3 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -1091,6 +1091,39 @@ int ima_measure_critical_data(const char *event_label, } EXPORT_SYMBOL_GPL(ima_measure_critical_data); +#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS + +/** + * ima_kernel_module_request - Prevent crypto-pkcs1pad(rsa,*) requests + * @kmod_name: kernel module name + * + * Avoid a verification loop where verifying the signature of the modprobe + * binary requires executing modprobe itself. Since the modprobe iint->mutex + * is already held when the signature verification is performed, a deadlock + * occurs as soon as modprobe is executed within the critical region, since + * the same lock cannot be taken again. + * + * This happens when public_key_verify_signature(), in case of RSA algorithm, + * use alg_name to store internal information in order to construct an + * algorithm on the fly, but crypto_larval_lookup() will try to use alg_name + * in order to load a kernel module with same name. + * + * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, + * we are safe to fail such module request from crypto_larval_lookup(), and + * avoid the verification loop. + * + * Return: Zero if it is safe to load the kernel module, -EINVAL otherwise. + */ +int ima_kernel_module_request(char *kmod_name) +{ + if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) + return -EINVAL; + + return 0; +} + +#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */ + static int __init init_ima(void) { int error; diff --git a/security/security.c b/security/security.c index 6c23c620e3c1..bc3c6790aeaf 100644 --- a/security/security.c +++ b/security/security.c @@ -3249,7 +3249,7 @@ int security_kernel_module_request(char *kmod_name) ret = call_int_hook(kernel_module_request, 0, kmod_name); if (ret) return ret; - return integrity_kernel_module_request(kmod_name); + return ima_kernel_module_request(kmod_name); } /** From cd3cec0a02c7338ce2901c574f3935b8f6984aab Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:08 +0100 Subject: [PATCH 0425/1038] ima: Move to LSM infrastructure Move hardcoded IMA function calls (not appraisal-specific functions) from various places in the kernel to the LSM infrastructure, by introducing a new LSM named 'ima' (at the end of the LSM list and always enabled like 'integrity'). Having IMA before EVM in the Makefile is sufficient to preserve the relative order of the new 'ima' LSM in respect to the upcoming 'evm' LSM, and thus the order of IMA and EVM function calls as when they were hardcoded. Make moved functions as static (except ima_post_key_create_or_update(), which is not in ima_main.c), and register them as implementation of the respective hooks in the new function init_ima_lsm(). Select CONFIG_SECURITY_PATH, to ensure that the path-based LSM hook path_post_mknod is always available and ima_post_path_mknod() is always executed to mark files as new, as before the move. A slight difference is that IMA and EVM functions registered for the inode_post_setattr, inode_post_removexattr, path_post_mknod, inode_post_create_tmpfile, inode_post_set_acl and inode_post_remove_acl won't be executed for private inodes. Since those inodes are supposed to be fs-internal, they should not be of interest to IMA or EVM. The S_PRIVATE flag is used for anonymous inodes, hugetlbfs, reiserfs xattrs, XFS scrub and kernel-internal tmpfs files. Conditionally register ima_post_key_create_or_update() if CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS is enabled. Also, conditionally register ima_kernel_module_request() if CONFIG_INTEGRITY_ASYMMETRIC_KEYS is enabled. Finally, add the LSM_ID_IMA case in lsm_list_modules_test.c. Signed-off-by: Roberto Sassu Acked-by: Chuck Lever Acked-by: Casey Schaufler Acked-by: Christian Brauner Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/file_table.c | 2 - fs/namei.c | 6 - fs/nfsd/vfs.c | 7 -- fs/open.c | 1 - include/linux/ima.h | 104 ------------------ include/uapi/linux/lsm.h | 1 + security/integrity/Makefile | 1 + security/integrity/ima/Kconfig | 1 + security/integrity/ima/ima.h | 6 + security/integrity/ima/ima_main.c | 78 +++++++++---- security/integrity/integrity.h | 1 + security/keys/key.c | 9 +- security/security.c | 63 ++--------- .../selftests/lsm/lsm_list_modules_test.c | 3 + 14 files changed, 83 insertions(+), 200 deletions(-) diff --git a/fs/file_table.c b/fs/file_table.c index 725407f374fa..1a0962981b48 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -368,7 +367,6 @@ static void __fput(struct file *file) locks_remove_file(file); security_file_release(file); - ima_file_free(file); if (unlikely(file->f_flags & FASYNC)) { if (file->f_op->fasync) file->f_op->fasync(-1, file, 0); diff --git a/fs/namei.c b/fs/namei.c index c7ac7fdf0f25..1bf7435cdf76 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -3641,8 +3640,6 @@ static int do_open(struct nameidata *nd, error = vfs_open(&nd->path, file); if (!error) error = security_file_post_open(file, op->acc_mode); - if (!error) - error = ima_file_check(file, op->acc_mode); if (!error && do_truncate) error = handle_truncate(idmap, file); if (unlikely(error > 0)) { @@ -3706,7 +3703,6 @@ static int vfs_tmpfile(struct mnt_idmap *idmap, spin_unlock(&inode->i_lock); } security_inode_post_create_tmpfile(idmap, inode); - ima_post_create_tmpfile(idmap, inode); return 0; } @@ -4052,8 +4048,6 @@ retry: case 0: case S_IFREG: error = vfs_create(idmap, path.dentry->d_inode, dentry, mode, true); - if (!error) - ima_post_path_mknod(idmap, dentry); break; case S_IFCHR: case S_IFBLK: error = vfs_mknod(idmap, path.dentry->d_inode, diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index e44d8239545b..4349dde92154 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -883,12 +882,6 @@ __nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, goto out; } - host_err = ima_file_check(file, may_flags); - if (host_err) { - fput(file); - goto out; - } - if (may_flags & NFSD_MAY_64BIT_COOKIE) file->f_mode |= FMODE_64BITHASH; else diff --git a/fs/open.c b/fs/open.c index a84d21e55c39..1cf05a1413aa 100644 --- a/fs/open.c +++ b/fs/open.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include diff --git a/include/linux/ima.h b/include/linux/ima.h index 0f9af283cbc8..23ae24b60ecf 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -16,24 +16,6 @@ struct linux_binprm; #ifdef CONFIG_IMA extern enum hash_algo ima_get_current_hash_algo(void); -extern int ima_bprm_check(struct linux_binprm *bprm); -extern int ima_file_check(struct file *file, int mask); -extern void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode); -extern void ima_file_free(struct file *file); -extern int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags); -extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, - unsigned long prot); -extern int ima_load_data(enum kernel_load_data_id id, bool contents); -extern int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id id, char *description); -extern int ima_read_file(struct file *file, enum kernel_read_file_id id, - bool contents); -extern int ima_post_read_file(struct file *file, char *buf, loff_t size, - enum kernel_read_file_id id); -extern void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry); extern int ima_file_hash(struct file *file, char *buf, size_t buf_size); extern int ima_inode_hash(struct inode *inode, char *buf, size_t buf_size); extern void ima_kexec_cmdline(int kernel_fd, const void *buf, int size); @@ -58,68 +40,6 @@ static inline enum hash_algo ima_get_current_hash_algo(void) return HASH_ALGO__LAST; } -static inline int ima_bprm_check(struct linux_binprm *bprm) -{ - return 0; -} - -static inline int ima_file_check(struct file *file, int mask) -{ - return 0; -} - -static inline void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode) -{ -} - -static inline void ima_file_free(struct file *file) -{ - return; -} - -static inline int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags) -{ - return 0; -} - -static inline int ima_file_mprotect(struct vm_area_struct *vma, - unsigned long reqprot, unsigned long prot) -{ - return 0; -} - -static inline int ima_load_data(enum kernel_load_data_id id, bool contents) -{ - return 0; -} - -static inline int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id id, - char *description) -{ - return 0; -} - -static inline int ima_read_file(struct file *file, enum kernel_read_file_id id, - bool contents) -{ - return 0; -} - -static inline int ima_post_read_file(struct file *file, char *buf, loff_t size, - enum kernel_read_file_id id) -{ - return 0; -} - -static inline void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry) -{ - return; -} - static inline int ima_file_hash(struct file *file, char *buf, size_t buf_size) { return -EOPNOTSUPP; @@ -170,20 +90,6 @@ static inline void ima_add_kexec_buffer(struct kimage *image) {} #endif -#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS -extern void ima_post_key_create_or_update(struct key *keyring, - struct key *key, - const void *payload, size_t plen, - unsigned long flags, bool create); -#else -static inline void ima_post_key_create_or_update(struct key *keyring, - struct key *key, - const void *payload, - size_t plen, - unsigned long flags, - bool create) {} -#endif /* CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS */ - #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); extern void ima_inode_post_setattr(struct mnt_idmap *idmap, @@ -256,14 +162,4 @@ static inline bool ima_appraise_signature(enum kernel_read_file_id func) return false; } #endif /* CONFIG_IMA_APPRAISE && CONFIG_INTEGRITY_TRUSTED_KEYRING */ - -#if defined(CONFIG_IMA) && defined(CONFIG_INTEGRITY_ASYMMETRIC_KEYS) -extern int ima_kernel_module_request(char *kmod_name); -#else -static inline int ima_kernel_module_request(char *kmod_name) -{ - return 0; -} - -#endif #endif /* _LINUX_IMA_H */ diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h index f8aef9ade549..b3b7fd699b63 100644 --- a/include/uapi/linux/lsm.h +++ b/include/uapi/linux/lsm.h @@ -62,6 +62,7 @@ struct lsm_ctx { #define LSM_ID_LOCKDOWN 108 #define LSM_ID_BPF 109 #define LSM_ID_LANDLOCK 110 +#define LSM_ID_IMA 111 /* * LSM_ATTR_XXX definitions identify different LSM attributes diff --git a/security/integrity/Makefile b/security/integrity/Makefile index d0ffe37dc1d6..92b63039c654 100644 --- a/security/integrity/Makefile +++ b/security/integrity/Makefile @@ -18,5 +18,6 @@ integrity-$(CONFIG_LOAD_IPL_KEYS) += platform_certs/load_ipl_s390.o integrity-$(CONFIG_LOAD_PPC_KEYS) += platform_certs/efi_parser.o \ platform_certs/load_powerpc.o \ platform_certs/keyring_handler.o +# The relative order of the 'ima' and 'evm' LSMs depends on the order below. obj-$(CONFIG_IMA) += ima/ obj-$(CONFIG_EVM) += evm/ diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index b98bfe9efd0c..475c32615006 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -8,6 +8,7 @@ config IMA select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_HASH_INFO + select SECURITY_PATH select TCG_TPM if HAS_IOMEM select TCG_TIS if TCG_TPM && X86 select TCG_CRB if TCG_TPM && ACPI diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index c29db699c996..c0412100023e 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -127,6 +127,12 @@ void ima_load_kexec_buffer(void); static inline void ima_load_kexec_buffer(void) {} #endif /* CONFIG_HAVE_IMA_KEXEC */ +#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS +void ima_post_key_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t plen, + unsigned long flags, bool create); +#endif + /* * The default binary_runtime_measurements list format is defined as the * platform native format. The canonical format is defined as little-endian. diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 3891b83efdb3..d0826e864c29 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -189,7 +189,7 @@ static void ima_check_last_writer(struct integrity_iint_cache *iint, * * Flag files that changed, based on i_version */ -void ima_file_free(struct file *file) +static void ima_file_free(struct file *file) { struct inode *inode = file_inode(file); struct integrity_iint_cache *iint; @@ -427,8 +427,8 @@ out: * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags) +static int ima_file_mmap(struct file *file, unsigned long reqprot, + unsigned long prot, unsigned long flags) { u32 secid; int ret; @@ -466,8 +466,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, * * On mprotect change success, return 0. On failure, return -EACESS. */ -int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, - unsigned long prot) +static int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot) { struct ima_template_desc *template = NULL; struct file *file; @@ -525,7 +525,7 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_bprm_check(struct linux_binprm *bprm) +static int ima_bprm_check(struct linux_binprm *bprm) { int ret; u32 secid; @@ -551,7 +551,7 @@ int ima_bprm_check(struct linux_binprm *bprm) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_file_check(struct file *file, int mask) +static int ima_file_check(struct file *file, int mask) { u32 secid; @@ -560,7 +560,6 @@ int ima_file_check(struct file *file, int mask) mask & (MAY_READ | MAY_WRITE | MAY_EXEC | MAY_APPEND), FILE_CHECK); } -EXPORT_SYMBOL_GPL(ima_file_check); static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, size_t buf_size) @@ -685,8 +684,9 @@ EXPORT_SYMBOL_GPL(ima_inode_hash); * Skip calling process_measurement(), but indicate which newly, created * tmpfiles are in policy. */ -void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode) +static void ima_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode) + { struct integrity_iint_cache *iint; int must_appraise; @@ -717,8 +717,7 @@ void ima_post_create_tmpfile(struct mnt_idmap *idmap, * Mark files created via the mknodat syscall as new, so that the * file data can be written later. */ -void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry) +static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) { struct integrity_iint_cache *iint; struct inode *inode = dentry->d_inode; @@ -753,8 +752,8 @@ void ima_post_path_mknod(struct mnt_idmap *idmap, * * For permission return 0, otherwise return -EACCES. */ -int ima_read_file(struct file *file, enum kernel_read_file_id read_id, - bool contents) +static int ima_read_file(struct file *file, enum kernel_read_file_id read_id, + bool contents) { enum ima_hooks func; u32 secid; @@ -803,8 +802,8 @@ const int read_idmap[READING_MAX_ID] = { * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_read_file(struct file *file, char *buf, loff_t size, - enum kernel_read_file_id read_id) +static int ima_post_read_file(struct file *file, char *buf, loff_t size, + enum kernel_read_file_id read_id) { enum ima_hooks func; u32 secid; @@ -837,7 +836,7 @@ int ima_post_read_file(struct file *file, char *buf, loff_t size, * * For permission return 0, otherwise return -EACCES. */ -int ima_load_data(enum kernel_load_data_id id, bool contents) +static int ima_load_data(enum kernel_load_data_id id, bool contents) { bool ima_enforce, sig_enforce; @@ -891,9 +890,9 @@ int ima_load_data(enum kernel_load_data_id id, bool contents) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id load_id, - char *description) +static int ima_post_load_data(char *buf, loff_t size, + enum kernel_load_data_id load_id, + char *description) { if (load_id == LOADING_FIRMWARE) { if ((ima_appraise & IMA_APPRAISE_FIRMWARE) && @@ -1114,7 +1113,7 @@ EXPORT_SYMBOL_GPL(ima_measure_critical_data); * * Return: Zero if it is safe to load the kernel module, -EINVAL otherwise. */ -int ima_kernel_module_request(char *kmod_name) +static int ima_kernel_module_request(char *kmod_name) { if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) return -EINVAL; @@ -1155,4 +1154,41 @@ static int __init init_ima(void) return error; } +static struct security_hook_list ima_hooks[] __ro_after_init = { + LSM_HOOK_INIT(bprm_check_security, ima_bprm_check), + LSM_HOOK_INIT(file_post_open, ima_file_check), + LSM_HOOK_INIT(inode_post_create_tmpfile, ima_post_create_tmpfile), + LSM_HOOK_INIT(file_release, ima_file_free), + LSM_HOOK_INIT(mmap_file, ima_file_mmap), + LSM_HOOK_INIT(file_mprotect, ima_file_mprotect), + LSM_HOOK_INIT(kernel_load_data, ima_load_data), + LSM_HOOK_INIT(kernel_post_load_data, ima_post_load_data), + LSM_HOOK_INIT(kernel_read_file, ima_read_file), + LSM_HOOK_INIT(kernel_post_read_file, ima_post_read_file), + LSM_HOOK_INIT(path_post_mknod, ima_post_path_mknod), +#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS + LSM_HOOK_INIT(key_post_create_or_update, ima_post_key_create_or_update), +#endif +#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS + LSM_HOOK_INIT(kernel_module_request, ima_kernel_module_request), +#endif +}; + +static const struct lsm_id ima_lsmid = { + .name = "ima", + .id = LSM_ID_IMA, +}; + +static int __init init_ima_lsm(void) +{ + security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid); + return 0; +} + +DEFINE_LSM(ima) = { + .name = "ima", + .init = init_ima_lsm, + .order = LSM_ORDER_LAST, +}; + late_initcall(init_ima); /* Start IMA after the TPM is available */ diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 9561db7cf6b4..59eaddd84434 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -18,6 +18,7 @@ #include #include #include +#include /* iint action cache flags */ #define IMA_MEASURE 0x00000001 diff --git a/security/keys/key.c b/security/keys/key.c index 31a8b9408b7c..560790038329 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "internal.h" @@ -932,8 +931,6 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, security_key_post_create_or_update(keyring, key, payload, plen, flags, true); - ima_post_key_create_or_update(keyring, key, payload, plen, - flags, true); key_ref = make_key_ref(key, is_key_possessed(keyring_ref)); @@ -965,13 +962,9 @@ error: key_ref = __key_update(key_ref, &prep); - if (!IS_ERR(key_ref)) { + if (!IS_ERR(key_ref)) security_key_post_create_or_update(keyring, key, payload, plen, flags, false); - ima_post_key_create_or_update(keyring, key, - payload, plen, - flags, false); - } goto error_free_prep; } diff --git a/security/security.c b/security/security.c index bc3c6790aeaf..8495890a5a37 100644 --- a/security/security.c +++ b/security/security.c @@ -50,7 +50,8 @@ (IS_ENABLED(CONFIG_SECURITY_SAFESETID) ? 1 : 0) + \ (IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \ - (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0)) + (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0) + \ + (IS_ENABLED(CONFIG_IMA) ? 1 : 0)) /* * These are descriptions of the reasons that can be passed to the @@ -1182,12 +1183,7 @@ int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file * */ int security_bprm_check(struct linux_binprm *bprm) { - int ret; - - ret = call_int_hook(bprm_check_security, 0, bprm); - if (ret) - return ret; - return ima_bprm_check(bprm); + return call_int_hook(bprm_check_security, 0, bprm); } /** @@ -2895,13 +2891,8 @@ static inline unsigned long mmap_prot(struct file *file, unsigned long prot) int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags) { - unsigned long prot_adj = mmap_prot(file, prot); - int ret; - - ret = call_int_hook(mmap_file, 0, file, prot, prot_adj, flags); - if (ret) - return ret; - return ima_file_mmap(file, prot, prot_adj, flags); + return call_int_hook(mmap_file, 0, file, prot, mmap_prot(file, prot), + flags); } /** @@ -2930,12 +2921,7 @@ int security_mmap_addr(unsigned long addr) int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { - int ret; - - ret = call_int_hook(file_mprotect, 0, vma, reqprot, prot); - if (ret) - return ret; - return ima_file_mprotect(vma, reqprot, prot); + return call_int_hook(file_mprotect, 0, vma, reqprot, prot); } /** @@ -3244,12 +3230,7 @@ int security_kernel_create_files_as(struct cred *new, struct inode *inode) */ int security_kernel_module_request(char *kmod_name) { - int ret; - - ret = call_int_hook(kernel_module_request, 0, kmod_name); - if (ret) - return ret; - return ima_kernel_module_request(kmod_name); + return call_int_hook(kernel_module_request, 0, kmod_name); } /** @@ -3265,12 +3246,7 @@ int security_kernel_module_request(char *kmod_name) int security_kernel_read_file(struct file *file, enum kernel_read_file_id id, bool contents) { - int ret; - - ret = call_int_hook(kernel_read_file, 0, file, id, contents); - if (ret) - return ret; - return ima_read_file(file, id, contents); + return call_int_hook(kernel_read_file, 0, file, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_read_file); @@ -3290,12 +3266,7 @@ EXPORT_SYMBOL_GPL(security_kernel_read_file); int security_kernel_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id) { - int ret; - - ret = call_int_hook(kernel_post_read_file, 0, file, buf, size, id); - if (ret) - return ret; - return ima_post_read_file(file, buf, size, id); + return call_int_hook(kernel_post_read_file, 0, file, buf, size, id); } EXPORT_SYMBOL_GPL(security_kernel_post_read_file); @@ -3310,12 +3281,7 @@ EXPORT_SYMBOL_GPL(security_kernel_post_read_file); */ int security_kernel_load_data(enum kernel_load_data_id id, bool contents) { - int ret; - - ret = call_int_hook(kernel_load_data, 0, id, contents); - if (ret) - return ret; - return ima_load_data(id, contents); + return call_int_hook(kernel_load_data, 0, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_load_data); @@ -3337,13 +3303,8 @@ int security_kernel_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description) { - int ret; - - ret = call_int_hook(kernel_post_load_data, 0, buf, size, id, - description); - if (ret) - return ret; - return ima_post_load_data(buf, size, id, description); + return call_int_hook(kernel_post_load_data, 0, buf, size, id, + description); } EXPORT_SYMBOL_GPL(security_kernel_post_load_data); diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 9df29b1e3497..17333787cb2f 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -122,6 +122,9 @@ TEST(correct_lsm_list_modules) case LSM_ID_LANDLOCK: name = "landlock"; break; + case LSM_ID_IMA: + name = "ima"; + break; default: name = "INVALID"; break; From 84594c9ecdca7ca595bc50e315093cb76921fd8e Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:09 +0100 Subject: [PATCH 0426/1038] ima: Move IMA-Appraisal to LSM infrastructure A few additional IMA hooks are needed to reset the cached appraisal status, causing the file's integrity to be re-evaluated on next access. Register these IMA-appraisal only functions separately from the rest of IMA functions, as appraisal is a separate feature not necessarily enabled in the kernel configuration. Reuse the same approach as for other IMA functions, move hardcoded calls from various places in the kernel to the LSM infrastructure. Declare the functions as static and register them as hook implementations in init_ima_appraise_lsm(), called by init_ima_lsm(). Also move the inline function ima_inode_remove_acl() from the public ima.h header to ima_appraise.c. Signed-off-by: Roberto Sassu Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Reviewed-by: Casey Schaufler Acked-by: Christian Brauner Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 - include/linux/ima.h | 55 --------------------------- security/integrity/ima/ima.h | 5 +++ security/integrity/ima/ima_appraise.c | 38 +++++++++++++----- security/integrity/ima/ima_main.c | 1 + security/security.c | 13 ------- 6 files changed, 35 insertions(+), 79 deletions(-) diff --git a/fs/attr.c b/fs/attr.c index 990e1b3a3c91..7e97313e7f70 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -17,7 +17,6 @@ #include #include #include -#include #include "internal.h" @@ -503,7 +502,6 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); security_inode_post_setattr(idmap, dentry, ia_valid); - ima_inode_post_setattr(idmap, dentry, ia_valid); evm_inode_post_setattr(idmap, dentry, ia_valid); } diff --git a/include/linux/ima.h b/include/linux/ima.h index 23ae24b60ecf..0bae61a15b60 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -92,66 +92,11 @@ static inline void ima_add_kexec_buffer(struct kimage *image) #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); -extern void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid); -extern int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len, int flags); -extern int ima_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl); -static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return ima_inode_set_acl(idmap, dentry, acl_name, NULL); -} - -extern int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name); #else static inline bool is_ima_appraise_enabled(void) { return 0; } - -static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid) -{ - return; -} - -static inline int ima_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len, - int flags) -{ - return 0; -} - -static inline int ima_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl) -{ - - return 0; -} - -static inline int ima_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name) -{ - return 0; -} - -static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return 0; -} #endif /* CONFIG_IMA_APPRAISE */ #if defined(CONFIG_IMA_APPRAISE) && defined(CONFIG_INTEGRITY_TRUSTED_KEYRING) diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index c0412100023e..a27fc10f84f7 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -334,6 +334,7 @@ enum hash_algo ima_get_hash_algo(const struct evm_ima_xattr_data *xattr_value, int xattr_len); int ima_read_xattr(struct dentry *dentry, struct evm_ima_xattr_data **xattr_value, int xattr_len); +void __init init_ima_appraise_lsm(const struct lsm_id *lsmid); #else static inline int ima_check_blacklist(struct integrity_iint_cache *iint, @@ -385,6 +386,10 @@ static inline int ima_read_xattr(struct dentry *dentry, return 0; } +static inline void __init init_ima_appraise_lsm(const struct lsm_id *lsmid) +{ +} + #endif /* CONFIG_IMA_APPRAISE */ #ifdef CONFIG_IMA_APPRAISE_MODSIG diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 36abc84ba299..076451109637 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -636,8 +636,8 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid) +static void ima_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { struct inode *inode = d_backing_inode(dentry); struct integrity_iint_cache *iint; @@ -750,9 +750,9 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; } -int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len, int flags) +static int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xvalue = xattr_value; int digsig = 0; @@ -781,8 +781,8 @@ int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, return result; } -int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, - const char *acl_name, struct posix_acl *kacl) +static int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) { if (evm_revalidate_status(acl_name)) ima_reset_appraise_flags(d_backing_inode(dentry), 0); @@ -790,8 +790,8 @@ int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return 0; } -int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name) +static int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { int result; @@ -803,3 +803,23 @@ int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, } return result; } + +static int ima_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + return ima_inode_set_acl(idmap, dentry, acl_name, NULL); +} + +static struct security_hook_list ima_appraise_hooks[] __ro_after_init = { + LSM_HOOK_INIT(inode_post_setattr, ima_inode_post_setattr), + LSM_HOOK_INIT(inode_setxattr, ima_inode_setxattr), + LSM_HOOK_INIT(inode_set_acl, ima_inode_set_acl), + LSM_HOOK_INIT(inode_removexattr, ima_inode_removexattr), + LSM_HOOK_INIT(inode_remove_acl, ima_inode_remove_acl), +}; + +void __init init_ima_appraise_lsm(const struct lsm_id *lsmid) +{ + security_add_hooks(ima_appraise_hooks, ARRAY_SIZE(ima_appraise_hooks), + lsmid); +} diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index d0826e864c29..b8b8891a9379 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -1182,6 +1182,7 @@ static const struct lsm_id ima_lsmid = { static int __init init_ima_lsm(void) { security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid); + init_ima_appraise_lsm(&ima_lsmid); return 0; } diff --git a/security/security.c b/security/security.c index 8495890a5a37..bed659b53e59 100644 --- a/security/security.c +++ b/security/security.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -2308,9 +2307,6 @@ int security_inode_setxattr(struct mnt_idmap *idmap, if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); - if (ret) - return ret; - ret = ima_inode_setxattr(idmap, dentry, name, value, size, flags); if (ret) return ret; return evm_inode_setxattr(idmap, dentry, name, value, size, flags); @@ -2338,9 +2334,6 @@ int security_inode_set_acl(struct mnt_idmap *idmap, return 0; ret = call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, kacl); - if (ret) - return ret; - ret = ima_inode_set_acl(idmap, dentry, acl_name, kacl); if (ret) return ret; return evm_inode_set_acl(idmap, dentry, acl_name, kacl); @@ -2401,9 +2394,6 @@ int security_inode_remove_acl(struct mnt_idmap *idmap, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; ret = call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); - if (ret) - return ret; - ret = ima_inode_remove_acl(idmap, dentry, acl_name); if (ret) return ret; return evm_inode_remove_acl(idmap, dentry, acl_name); @@ -2503,9 +2493,6 @@ int security_inode_removexattr(struct mnt_idmap *idmap, ret = call_int_hook(inode_removexattr, 1, idmap, dentry, name); if (ret == 1) ret = cap_inode_removexattr(idmap, dentry, name); - if (ret) - return ret; - ret = ima_inode_removexattr(idmap, dentry, name); if (ret) return ret; return evm_inode_removexattr(idmap, dentry, name); From 9238311176115aac1b1a86e8e968c04ebec747a1 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:10 +0100 Subject: [PATCH 0427/1038] evm: Move to LSM infrastructure As for IMA, move hardcoded EVM function calls from various places in the kernel to the LSM infrastructure, by introducing a new LSM named 'evm' (last and always enabled like 'ima'). The order in the Makefile ensures that 'evm' hooks are executed after 'ima' ones. Make EVM functions as static (except for evm_inode_init_security(), which is exported), and register them as hook implementations in init_evm_lsm(). Also move the inline functions evm_inode_remove_acl(), evm_inode_post_remove_acl(), and evm_inode_post_set_acl() from the public evm.h header to evm_main.c. Unlike before (see commit to move IMA to the LSM infrastructure), evm_inode_post_setattr(), evm_inode_post_set_acl(), evm_inode_post_remove_acl(), and evm_inode_post_removexattr() are not executed for private inodes. Finally, add the LSM_ID_EVM case in lsm_list_modules_test.c Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Acked-by: Christian Brauner Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- fs/attr.c | 2 - fs/posix_acl.c | 3 - fs/xattr.c | 2 - include/linux/evm.h | 113 ----------------- include/uapi/linux/lsm.h | 1 + security/integrity/evm/evm_main.c | 118 +++++++++++++++--- security/security.c | 43 ++----- .../selftests/lsm/lsm_list_modules_test.c | 3 + 8 files changed, 116 insertions(+), 169 deletions(-) diff --git a/fs/attr.c b/fs/attr.c index 7e97313e7f70..4d0d75953107 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -16,7 +16,6 @@ #include #include #include -#include #include "internal.h" @@ -502,7 +501,6 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); security_inode_post_setattr(idmap, dentry, ia_valid); - evm_inode_post_setattr(idmap, dentry, ia_valid); } return error; diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 0d2371240c1b..5c90239e3f2b 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -1138,7 +1137,6 @@ retry_deleg: if (!error) { fsnotify_xattr(dentry); security_inode_post_set_acl(dentry, acl_name, kacl); - evm_inode_post_set_acl(dentry, acl_name, kacl); } out_inode_unlock: @@ -1247,7 +1245,6 @@ retry_deleg: if (!error) { fsnotify_xattr(dentry); security_inode_post_remove_acl(idmap, dentry, acl_name); - evm_inode_post_remove_acl(idmap, dentry, acl_name); } out_inode_unlock: diff --git a/fs/xattr.c b/fs/xattr.c index f891c260a971..f8b643f91a98 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -557,7 +556,6 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap, fsnotify_xattr(dentry); security_inode_post_removexattr(dentry, name); - evm_inode_post_removexattr(dentry, name); out: return error; diff --git a/include/linux/evm.h b/include/linux/evm.h index 3faabdd47852..cb481eccc967 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -21,45 +21,6 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry, void *xattr_value, size_t xattr_value_len, struct integrity_iint_cache *iint); -extern int evm_inode_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, struct iattr *attr); -extern void evm_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid); -extern int evm_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *name, - const void *value, size_t size, int flags); -extern void evm_inode_post_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len, - int flags); -extern int evm_inode_copy_up_xattr(const char *name); -extern int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *xattr_name); -extern void evm_inode_post_removexattr(struct dentry *dentry, - const char *xattr_name); -static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - evm_inode_post_removexattr(dentry, acl_name); -} -extern int evm_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl); -static inline int evm_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return evm_inode_set_acl(idmap, dentry, acl_name, NULL); -} -static inline void evm_inode_post_set_acl(struct dentry *dentry, - const char *acl_name, - struct posix_acl *kacl) -{ - return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); -} - int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, int *xattr_count); @@ -94,80 +55,6 @@ static inline enum integrity_status evm_verifyxattr(struct dentry *dentry, } #endif -static inline int evm_inode_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, struct iattr *attr) -{ - return 0; -} - -static inline void evm_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, int ia_valid) -{ - return; -} - -static inline int evm_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *name, - const void *value, size_t size, int flags) -{ - return 0; -} - -static inline void evm_inode_post_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len, - int flags) -{ - return; -} - -static inline int evm_inode_copy_up_xattr(const char *name) -{ - return 0; -} - -static inline int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name) -{ - return 0; -} - -static inline void evm_inode_post_removexattr(struct dentry *dentry, - const char *xattr_name) -{ - return; -} - -static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return; -} - -static inline int evm_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl) -{ - return 0; -} - -static inline int evm_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return 0; -} - -static inline void evm_inode_post_set_acl(struct dentry *dentry, - const char *acl_name, - struct posix_acl *kacl) -{ - return; -} - static inline int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h index b3b7fd699b63..33d8c9f4aa6b 100644 --- a/include/uapi/linux/lsm.h +++ b/include/uapi/linux/lsm.h @@ -63,6 +63,7 @@ struct lsm_ctx { #define LSM_ID_BPF 109 #define LSM_ID_LANDLOCK 110 #define LSM_ID_IMA 111 +#define LSM_ID_EVM 112 /* * LSM_ATTR_XXX definitions identify different LSM attributes diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index d35143179699..0a089af83a45 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -589,9 +589,9 @@ out: * userspace from writing HMAC value. Writing 'security.evm' requires * requires CAP_SYS_ADMIN privileges. */ -int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len, int flags) +static int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xattr_data = xattr_value; @@ -621,8 +621,8 @@ int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, * Removing 'security.evm' requires CAP_SYS_ADMIN privileges and that * the current value is valid. */ -int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *xattr_name) +static int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { /* Policy permits modification of the protected xattrs even though * there's no HMAC key loaded @@ -672,9 +672,11 @@ static inline int evm_inode_set_acl_change(struct mnt_idmap *idmap, * Prevent modifying posix acls causing the EVM HMAC to be re-calculated * and 'security.evm' xattr updated, unless the existing 'security.evm' is * valid. + * + * Return: zero on success, -EPERM on failure. */ -int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, - const char *acl_name, struct posix_acl *kacl) +static int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) { enum integrity_status evm_status; @@ -713,6 +715,24 @@ int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; } +/** + * evm_inode_remove_acl - Protect the EVM extended attribute from posix acls + * @idmap: idmap of the mount + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * + * Prevent removing posix acls causing the EVM HMAC to be re-calculated + * and 'security.evm' xattr updated, unless the existing 'security.evm' is + * valid. + * + * Return: zero on success, -EPERM on failure. + */ +static int evm_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + return evm_inode_set_acl(idmap, dentry, acl_name, NULL); +} + static void evm_reset_status(struct inode *inode) { struct integrity_iint_cache *iint; @@ -761,9 +781,11 @@ bool evm_revalidate_status(const char *xattr_name) * __vfs_setxattr_noperm(). The caller of which has taken the inode's * i_mutex lock. */ -void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len, - int flags) +static void evm_inode_post_setxattr(struct dentry *dentry, + const char *xattr_name, + const void *xattr_value, + size_t xattr_value_len, + int flags) { if (!evm_revalidate_status(xattr_name)) return; @@ -782,6 +804,21 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, evm_update_evmxattr(dentry, xattr_name, xattr_value, xattr_value_len); } +/** + * evm_inode_post_set_acl - Update the EVM extended attribute from posix acls + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * @kacl: pointer to the posix acls + * + * Update the 'security.evm' xattr with the EVM HMAC re-calculated after setting + * posix acls. + */ +static void evm_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl) +{ + return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); +} + /** * evm_inode_post_removexattr - update 'security.evm' after removing the xattr * @dentry: pointer to the affected dentry @@ -792,7 +829,8 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, * No need to take the i_mutex lock here, as this function is called from * vfs_removexattr() which takes the i_mutex. */ -void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) +static void evm_inode_post_removexattr(struct dentry *dentry, + const char *xattr_name) { if (!evm_revalidate_status(xattr_name)) return; @@ -808,6 +846,22 @@ void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) evm_update_evmxattr(dentry, xattr_name, NULL, 0); } +/** + * evm_inode_post_remove_acl - Update the EVM extended attribute from posix acls + * @idmap: idmap of the mount + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * + * Update the 'security.evm' xattr with the EVM HMAC re-calculated after + * removing posix acls. + */ +static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name) +{ + evm_inode_post_removexattr(dentry, acl_name); +} + static int evm_attr_change(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -831,8 +885,8 @@ static int evm_attr_change(struct mnt_idmap *idmap, * Permit update of file attributes when files have a valid EVM signature, * except in the case of them having an immutable portable signature. */ -int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, - struct iattr *attr) +static int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; enum integrity_status evm_status; @@ -883,8 +937,8 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, - int ia_valid) +static void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { if (!evm_revalidate_status(NULL)) return; @@ -901,7 +955,7 @@ void evm_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, evm_update_evmxattr(dentry, NULL, NULL, 0); } -int evm_inode_copy_up_xattr(const char *name) +static int evm_inode_copy_up_xattr(const char *name) { if (strcmp(name, XATTR_NAME_EVM) == 0) return 1; /* Discard */ @@ -1004,4 +1058,36 @@ error: return error; } +static struct security_hook_list evm_hooks[] __ro_after_init = { + LSM_HOOK_INIT(inode_setattr, evm_inode_setattr), + LSM_HOOK_INIT(inode_post_setattr, evm_inode_post_setattr), + LSM_HOOK_INIT(inode_copy_up_xattr, evm_inode_copy_up_xattr), + LSM_HOOK_INIT(inode_setxattr, evm_inode_setxattr), + LSM_HOOK_INIT(inode_post_setxattr, evm_inode_post_setxattr), + LSM_HOOK_INIT(inode_set_acl, evm_inode_set_acl), + LSM_HOOK_INIT(inode_post_set_acl, evm_inode_post_set_acl), + LSM_HOOK_INIT(inode_remove_acl, evm_inode_remove_acl), + LSM_HOOK_INIT(inode_post_remove_acl, evm_inode_post_remove_acl), + LSM_HOOK_INIT(inode_removexattr, evm_inode_removexattr), + LSM_HOOK_INIT(inode_post_removexattr, evm_inode_post_removexattr), + LSM_HOOK_INIT(inode_init_security, evm_inode_init_security), +}; + +static const struct lsm_id evm_lsmid = { + .name = "evm", + .id = LSM_ID_EVM, +}; + +static int __init init_evm_lsm(void) +{ + security_add_hooks(evm_hooks, ARRAY_SIZE(evm_hooks), &evm_lsmid); + return 0; +} + +DEFINE_LSM(evm) = { + .name = "evm", + .init = init_evm_lsm, + .order = LSM_ORDER_LAST, +}; + late_initcall(init_evm); diff --git a/security/security.c b/security/security.c index bed659b53e59..59cb0e6cbae3 100644 --- a/security/security.c +++ b/security/security.c @@ -20,13 +20,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include @@ -50,7 +50,8 @@ (IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0) + \ - (IS_ENABLED(CONFIG_IMA) ? 1 : 0)) + (IS_ENABLED(CONFIG_IMA) ? 1 : 0) + \ + (IS_ENABLED(CONFIG_EVM) ? 1 : 0)) /* * These are descriptions of the reasons that can be passed to the @@ -1740,10 +1741,6 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, if (!xattr_count) goto out; - ret = evm_inode_init_security(inode, dir, qstr, new_xattrs, - &xattr_count); - if (ret) - goto out; ret = initxattrs(inode, new_xattrs, fs_data); out: for (; xattr_count > 0; xattr_count--) @@ -2235,14 +2232,9 @@ int security_inode_permission(struct inode *inode, int mask) int security_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_setattr, 0, idmap, dentry, attr); - if (ret) - return ret; - return evm_inode_setattr(idmap, dentry, attr); + return call_int_hook(inode_setattr, 0, idmap, dentry, attr); } EXPORT_SYMBOL_GPL(security_inode_setattr); @@ -2307,9 +2299,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap, if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); - if (ret) - return ret; - return evm_inode_setxattr(idmap, dentry, name, value, size, flags); + return ret; } /** @@ -2328,15 +2318,10 @@ int security_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, - kacl); - if (ret) - return ret; - return evm_inode_set_acl(idmap, dentry, acl_name, kacl); + return call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, + kacl); } /** @@ -2389,14 +2374,9 @@ int security_inode_get_acl(struct mnt_idmap *idmap, int security_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); - if (ret) - return ret; - return evm_inode_remove_acl(idmap, dentry, acl_name); + return call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); } /** @@ -2432,7 +2412,6 @@ void security_inode_post_setxattr(struct dentry *dentry, const char *name, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return; call_void_hook(inode_post_setxattr, dentry, name, value, size, flags); - evm_inode_post_setxattr(dentry, name, value, size, flags); } /** @@ -2493,9 +2472,7 @@ int security_inode_removexattr(struct mnt_idmap *idmap, ret = call_int_hook(inode_removexattr, 1, idmap, dentry, name); if (ret == 1) ret = cap_inode_removexattr(idmap, dentry, name); - if (ret) - return ret; - return evm_inode_removexattr(idmap, dentry, name); + return ret; } /** @@ -2699,7 +2676,7 @@ int security_inode_copy_up_xattr(const char *name) return rc; } - return evm_inode_copy_up_xattr(name); + return LSM_RET_DEFAULT(inode_copy_up_xattr); } EXPORT_SYMBOL(security_inode_copy_up_xattr); diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 17333787cb2f..4d5d4cee2586 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -125,6 +125,9 @@ TEST(correct_lsm_list_modules) case LSM_ID_IMA: name = "ima"; break; + case LSM_ID_EVM: + name = "evm"; + break; default: name = "INVALID"; break; From 75a323e604fc77c50c7ef2af6f0eeef221637642 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:11 +0100 Subject: [PATCH 0428/1038] evm: Make it independent from 'integrity' LSM Define a new structure for EVM-specific metadata, called evm_iint_cache, and embed it in the inode security blob. Introduce evm_iint_inode() to retrieve metadata, and register evm_inode_alloc_security() for the inode_alloc_security LSM hook, to initialize the structure (before splitting metadata, this task was done by iint_init_always()). Keep the non-NULL checks after calling evm_iint_inode() except in evm_inode_alloc_security(), to take into account inodes for which security_inode_alloc() was not called. When using shared metadata, obtaining a NULL pointer from integrity_iint_find() meant that the file wasn't in the IMA policy. Now, because IMA and EVM use disjoint metadata, the EVM status has to be stored for every inode regardless of the IMA policy. Given that from now on EVM relies on its own metadata, remove the iint parameter from evm_verifyxattr(). Also, directly retrieve the iint in evm_verify_hmac(), called by both evm_verifyxattr() and evm_verify_current_integrity(), since now there is no performance penalty in retrieving EVM metadata (constant time). Replicate the management of the IMA_NEW_FILE flag, by introducing evm_post_path_mknod() and evm_file_release() to respectively set and clear the newly introduced flag EVM_NEW_FILE, at the same time IMA does. Like for IMA, select CONFIG_SECURITY_PATH when EVM is enabled, to ensure that files are marked as new. Unlike ima_post_path_mknod(), evm_post_path_mknod() cannot check if a file must be appraised. Thus, it marks all affected files. Also, it does not clear EVM_NEW_FILE depending on i_version, but that is not a problem because IMA_NEW_FILE is always cleared when set in ima_check_last_writer(). Move the EVM-specific flag EVM_IMMUTABLE_DIGSIG to security/integrity/evm/evm.h, since that definition is now unnecessary in the common integrity layer. Finally, switch to the LSM reservation mechanism for the EVM xattr, and consequently decrement by one the number of xattrs to allocate in security_inode_init_security(). Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/evm.h | 8 +-- security/integrity/evm/Kconfig | 1 + security/integrity/evm/evm.h | 19 +++++++ security/integrity/evm/evm_crypto.c | 4 +- security/integrity/evm/evm_main.c | 76 ++++++++++++++++++++------- security/integrity/ima/ima_appraise.c | 2 +- security/integrity/integrity.h | 1 - security/security.c | 4 +- 8 files changed, 83 insertions(+), 32 deletions(-) diff --git a/include/linux/evm.h b/include/linux/evm.h index cb481eccc967..d48d6da32315 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -12,15 +12,12 @@ #include #include -struct integrity_iint_cache; - #ifdef CONFIG_EVM extern int evm_set_key(void *key, size_t keylen); extern enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint); + size_t xattr_value_len); int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, int *xattr_count); @@ -48,8 +45,7 @@ static inline int evm_set_key(void *key, size_t keylen) static inline enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint) + size_t xattr_value_len) { return INTEGRITY_UNKNOWN; } diff --git a/security/integrity/evm/Kconfig b/security/integrity/evm/Kconfig index fba9ee359bc9..861b3bacab82 100644 --- a/security/integrity/evm/Kconfig +++ b/security/integrity/evm/Kconfig @@ -6,6 +6,7 @@ config EVM select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_HASH_INFO + select SECURITY_PATH default n help EVM protects a file's security extended attributes against diff --git a/security/integrity/evm/evm.h b/security/integrity/evm/evm.h index 53bd7fec93fa..eb1a2c343bd7 100644 --- a/security/integrity/evm/evm.h +++ b/security/integrity/evm/evm.h @@ -32,6 +32,25 @@ struct xattr_list { bool enabled; }; +#define EVM_NEW_FILE 0x00000001 +#define EVM_IMMUTABLE_DIGSIG 0x00000002 + +/* EVM integrity metadata associated with an inode */ +struct evm_iint_cache { + unsigned long flags; + enum integrity_status evm_status:4; +}; + +extern struct lsm_blob_sizes evm_blob_sizes; + +static inline struct evm_iint_cache *evm_iint_inode(const struct inode *inode) +{ + if (unlikely(!inode->i_security)) + return NULL; + + return inode->i_security + evm_blob_sizes.lbs_inode; +} + extern int evm_initialized; #define EVM_ATTR_FSUUID 0x0001 diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index b1ffd4cc0b44..7552d49d0725 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -322,10 +322,10 @@ int evm_calc_hash(struct dentry *dentry, const char *req_xattr_name, static int evm_is_immutable(struct dentry *dentry, struct inode *inode) { const struct evm_ima_xattr_data *xattr_data = NULL; - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; int rc = 0; - iint = integrity_iint_find(inode); + iint = evm_iint_inode(inode); if (iint && (iint->flags & EVM_IMMUTABLE_DIGSIG)) return 1; diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 0a089af83a45..81dbade5b9b3 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -178,14 +178,14 @@ static int is_unsupported_fs(struct dentry *dentry) static enum integrity_status evm_verify_hmac(struct dentry *dentry, const char *xattr_name, char *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint) + size_t xattr_value_len) { struct evm_ima_xattr_data *xattr_data = NULL; struct signature_v2_hdr *hdr; enum integrity_status evm_status = INTEGRITY_PASS; struct evm_digest digest; - struct inode *inode; + struct inode *inode = d_backing_inode(dentry); + struct evm_iint_cache *iint = evm_iint_inode(inode); int rc, xattr_len, evm_immutable = 0; if (iint && (iint->evm_status == INTEGRITY_PASS || @@ -254,8 +254,6 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, (const char *)xattr_data, xattr_len, digest.digest, digest.hdr.length); if (!rc) { - inode = d_backing_inode(dentry); - if (xattr_data->type == EVM_XATTR_PORTABLE_DIGSIG) { if (iint) iint->flags |= EVM_IMMUTABLE_DIGSIG; @@ -403,7 +401,6 @@ int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, * @xattr_name: requested xattr * @xattr_value: requested xattr value * @xattr_value_len: requested xattr value length - * @iint: inode integrity metadata * * Calculate the HMAC for the given dentry and verify it against the stored * security.evm xattr. For performance, use the xattr value and length @@ -416,8 +413,7 @@ int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, */ enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, - void *xattr_value, size_t xattr_value_len, - struct integrity_iint_cache *iint) + void *xattr_value, size_t xattr_value_len) { if (!evm_key_loaded() || !evm_protected_xattr(xattr_name)) return INTEGRITY_UNKNOWN; @@ -425,13 +421,8 @@ enum integrity_status evm_verifyxattr(struct dentry *dentry, if (is_unsupported_fs(dentry)) return INTEGRITY_UNKNOWN; - if (!iint) { - iint = integrity_iint_find(d_backing_inode(dentry)); - if (!iint) - return INTEGRITY_UNKNOWN; - } return evm_verify_hmac(dentry, xattr_name, xattr_value, - xattr_value_len, iint); + xattr_value_len); } EXPORT_SYMBOL_GPL(evm_verifyxattr); @@ -448,7 +439,7 @@ static enum integrity_status evm_verify_current_integrity(struct dentry *dentry) if (!evm_key_loaded() || !S_ISREG(inode->i_mode) || evm_fixmode) return INTEGRITY_PASS; - return evm_verify_hmac(dentry, NULL, NULL, 0, NULL); + return evm_verify_hmac(dentry, NULL, NULL, 0); } /* @@ -526,14 +517,14 @@ static int evm_protect_xattr(struct mnt_idmap *idmap, evm_status = evm_verify_current_integrity(dentry); if (evm_status == INTEGRITY_NOXATTRS) { - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; /* Exception if the HMAC is not going to be calculated. */ if (evm_hmac_disabled()) return 0; - iint = integrity_iint_find(d_backing_inode(dentry)); - if (iint && (iint->flags & IMA_NEW_FILE)) + iint = evm_iint_inode(d_backing_inode(dentry)); + if (iint && (iint->flags & EVM_NEW_FILE)) return 0; /* exception for pseudo filesystems */ @@ -735,9 +726,9 @@ static int evm_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, static void evm_reset_status(struct inode *inode) { - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; - iint = integrity_iint_find(inode); + iint = evm_iint_inode(inode); if (iint) iint->evm_status = INTEGRITY_UNKNOWN; } @@ -1019,6 +1010,42 @@ out: } EXPORT_SYMBOL_GPL(evm_inode_init_security); +static int evm_inode_alloc_security(struct inode *inode) +{ + struct evm_iint_cache *iint = evm_iint_inode(inode); + + /* Called by security_inode_alloc(), it cannot be NULL. */ + iint->flags = 0UL; + iint->evm_status = INTEGRITY_UNKNOWN; + + return 0; +} + +static void evm_file_release(struct file *file) +{ + struct inode *inode = file_inode(file); + struct evm_iint_cache *iint = evm_iint_inode(inode); + fmode_t mode = file->f_mode; + + if (!S_ISREG(inode->i_mode) || !(mode & FMODE_WRITE)) + return; + + if (iint && atomic_read(&inode->i_writecount) == 1) + iint->flags &= ~EVM_NEW_FILE; +} + +static void evm_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + struct inode *inode = d_backing_inode(dentry); + struct evm_iint_cache *iint = evm_iint_inode(inode); + + if (!S_ISREG(inode->i_mode)) + return; + + if (iint) + iint->flags |= EVM_NEW_FILE; +} + #ifdef CONFIG_EVM_LOAD_X509 void __init evm_load_x509(void) { @@ -1071,6 +1098,9 @@ static struct security_hook_list evm_hooks[] __ro_after_init = { LSM_HOOK_INIT(inode_removexattr, evm_inode_removexattr), LSM_HOOK_INIT(inode_post_removexattr, evm_inode_post_removexattr), LSM_HOOK_INIT(inode_init_security, evm_inode_init_security), + LSM_HOOK_INIT(inode_alloc_security, evm_inode_alloc_security), + LSM_HOOK_INIT(file_release, evm_file_release), + LSM_HOOK_INIT(path_post_mknod, evm_post_path_mknod), }; static const struct lsm_id evm_lsmid = { @@ -1084,10 +1114,16 @@ static int __init init_evm_lsm(void) return 0; } +struct lsm_blob_sizes evm_blob_sizes __ro_after_init = { + .lbs_inode = sizeof(struct evm_iint_cache), + .lbs_xattr_count = 1, +}; + DEFINE_LSM(evm) = { .name = "evm", .init = init_evm_lsm, .order = LSM_ORDER_LAST, + .blobs = &evm_blob_sizes, }; late_initcall(init_evm); diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 076451109637..1dd6ee72a20a 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -520,7 +520,7 @@ int ima_appraise_measurement(enum ima_hooks func, } status = evm_verifyxattr(dentry, XATTR_NAME_IMA, xattr_value, - rc < 0 ? 0 : rc, iint); + rc < 0 ? 0 : rc); switch (status) { case INTEGRITY_PASS: case INTEGRITY_PASS_IMMUTABLE: diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 59eaddd84434..7a97c269a072 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -37,7 +37,6 @@ #define IMA_DIGSIG_REQUIRED 0x01000000 #define IMA_PERMIT_DIRECTIO 0x02000000 #define IMA_NEW_FILE 0x04000000 -#define EVM_IMMUTABLE_DIGSIG 0x08000000 #define IMA_FAIL_UNVERIFIABLE_SIGS 0x10000000 #define IMA_MODSIG_ALLOWED 0x20000000 #define IMA_CHECK_BLACKLIST 0x40000000 diff --git a/security/security.c b/security/security.c index 59cb0e6cbae3..e08c1aa82edf 100644 --- a/security/security.c +++ b/security/security.c @@ -1716,8 +1716,8 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, return 0; if (initxattrs) { - /* Allocate +1 for EVM and +1 as terminator. */ - new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 2, + /* Allocate +1 as terminator. */ + new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 1, sizeof(*new_xattrs), GFP_NOFS); if (!new_xattrs) return -ENOMEM; From 4de2f084fbff41113d9adec3c9e15ab12bf05e21 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:12 +0100 Subject: [PATCH 0429/1038] ima: Make it independent from 'integrity' LSM Make the 'ima' LSM independent from the 'integrity' LSM by introducing IMA own integrity metadata (ima_iint_cache structure, with IMA-specific fields from the integrity_iint_cache structure), and by managing it directly from the 'ima' LSM. Create ima_iint.c and introduce the same integrity metadata management functions found in iint.c (renamed with ima_). However, instead of putting metadata in an rbtree, reserve space from IMA in the inode security blob for a pointer, and introduce the ima_inode_set_iint()/ima_inode_get_iint() primitives to store/retrieve that pointer. This improves search time from logarithmic to constant. Consequently, don't include the inode pointer as field in the ima_iint_cache structure, since the association with the inode is clear. Since the inode field is missing in ima_iint_cache, pass the extra inode parameter to ima_get_verity_digest(). Prefer storing the pointer instead of the entire ima_iint_cache structure, to avoid too much memory pressure. Use the same mechanism as before, a cache named ima_iint_cache (renamed from iint_cache), to quickly allocate a new ima_iint_cache structure when requested by the IMA policy. Create the new ima_iint_cache in ima_iintcache_init(), called by init_ima_lsm(), during the initialization of the 'ima' LSM. And, register ima_inode_free_security() to free the ima_iint_cache structure, if exists. Replace integrity_iint_cache with ima_iint_cache in various places of the IMA code. Also, replace integrity_inode_get() and integrity_iint_find(), respectively with ima_inode_get() and ima_iint_find(). Finally, move the remaining IMA-specific flags to security/integrity/ima/ima.h, since they are now unnecessary in the common integrity layer. Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- security/integrity/ima/Makefile | 2 +- security/integrity/ima/ima.h | 137 +++++++++++++++++++++---- security/integrity/ima/ima_api.c | 23 +++-- security/integrity/ima/ima_appraise.c | 27 +++-- security/integrity/ima/ima_iint.c | 142 ++++++++++++++++++++++++++ security/integrity/ima/ima_init.c | 2 +- security/integrity/ima/ima_main.c | 36 ++++--- security/integrity/ima/ima_policy.c | 2 +- security/integrity/integrity.h | 53 ---------- 9 files changed, 308 insertions(+), 116 deletions(-) create mode 100644 security/integrity/ima/ima_iint.c diff --git a/security/integrity/ima/Makefile b/security/integrity/ima/Makefile index 2499f2485c04..b376d38b4ee6 100644 --- a/security/integrity/ima/Makefile +++ b/security/integrity/ima/Makefile @@ -4,7 +4,7 @@ # Measurement Architecture(IMA). # -obj-$(CONFIG_IMA) += ima.o +obj-$(CONFIG_IMA) += ima.o ima_iint.o ima-y := ima_fs.o ima_queue.o ima_init.o ima_main.o ima_crypto.o ima_api.o \ ima_policy.o ima_template.o ima_template_lib.o diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index a27fc10f84f7..11d7c0332207 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -60,7 +60,7 @@ extern const char boot_aggregate_name[]; /* IMA event related data */ struct ima_event_data { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; struct file *file; const unsigned char *filename; struct evm_ima_xattr_data *xattr_value; @@ -119,6 +119,107 @@ struct ima_kexec_hdr { u64 count; }; +/* IMA iint action cache flags */ +#define IMA_MEASURE 0x00000001 +#define IMA_MEASURED 0x00000002 +#define IMA_APPRAISE 0x00000004 +#define IMA_APPRAISED 0x00000008 +/*#define IMA_COLLECT 0x00000010 do not use this flag */ +#define IMA_COLLECTED 0x00000020 +#define IMA_AUDIT 0x00000040 +#define IMA_AUDITED 0x00000080 +#define IMA_HASH 0x00000100 +#define IMA_HASHED 0x00000200 + +/* IMA iint policy rule cache flags */ +#define IMA_NONACTION_FLAGS 0xff000000 +#define IMA_DIGSIG_REQUIRED 0x01000000 +#define IMA_PERMIT_DIRECTIO 0x02000000 +#define IMA_NEW_FILE 0x04000000 +#define IMA_FAIL_UNVERIFIABLE_SIGS 0x10000000 +#define IMA_MODSIG_ALLOWED 0x20000000 +#define IMA_CHECK_BLACKLIST 0x40000000 +#define IMA_VERITY_REQUIRED 0x80000000 + +#define IMA_DO_MASK (IMA_MEASURE | IMA_APPRAISE | IMA_AUDIT | \ + IMA_HASH | IMA_APPRAISE_SUBMASK) +#define IMA_DONE_MASK (IMA_MEASURED | IMA_APPRAISED | IMA_AUDITED | \ + IMA_HASHED | IMA_COLLECTED | \ + IMA_APPRAISED_SUBMASK) + +/* IMA iint subaction appraise cache flags */ +#define IMA_FILE_APPRAISE 0x00001000 +#define IMA_FILE_APPRAISED 0x00002000 +#define IMA_MMAP_APPRAISE 0x00004000 +#define IMA_MMAP_APPRAISED 0x00008000 +#define IMA_BPRM_APPRAISE 0x00010000 +#define IMA_BPRM_APPRAISED 0x00020000 +#define IMA_READ_APPRAISE 0x00040000 +#define IMA_READ_APPRAISED 0x00080000 +#define IMA_CREDS_APPRAISE 0x00100000 +#define IMA_CREDS_APPRAISED 0x00200000 +#define IMA_APPRAISE_SUBMASK (IMA_FILE_APPRAISE | IMA_MMAP_APPRAISE | \ + IMA_BPRM_APPRAISE | IMA_READ_APPRAISE | \ + IMA_CREDS_APPRAISE) +#define IMA_APPRAISED_SUBMASK (IMA_FILE_APPRAISED | IMA_MMAP_APPRAISED | \ + IMA_BPRM_APPRAISED | IMA_READ_APPRAISED | \ + IMA_CREDS_APPRAISED) + +/* IMA iint cache atomic_flags */ +#define IMA_CHANGE_XATTR 0 +#define IMA_UPDATE_XATTR 1 +#define IMA_CHANGE_ATTR 2 +#define IMA_DIGSIG 3 +#define IMA_MUST_MEASURE 4 + +/* IMA integrity metadata associated with an inode */ +struct ima_iint_cache { + struct mutex mutex; /* protects: version, flags, digest */ + u64 version; /* track inode changes */ + unsigned long flags; + unsigned long measured_pcrs; + unsigned long atomic_flags; + unsigned long real_ino; + dev_t real_dev; + enum integrity_status ima_file_status:4; + enum integrity_status ima_mmap_status:4; + enum integrity_status ima_bprm_status:4; + enum integrity_status ima_read_status:4; + enum integrity_status ima_creds_status:4; + struct ima_digest_data *ima_hash; +}; + +extern struct lsm_blob_sizes ima_blob_sizes; + +static inline struct ima_iint_cache * +ima_inode_get_iint(const struct inode *inode) +{ + struct ima_iint_cache **iint_sec; + + if (unlikely(!inode->i_security)) + return NULL; + + iint_sec = inode->i_security + ima_blob_sizes.lbs_inode; + return *iint_sec; +} + +static inline void ima_inode_set_iint(const struct inode *inode, + struct ima_iint_cache *iint) +{ + struct ima_iint_cache **iint_sec; + + if (unlikely(!inode->i_security)) + return; + + iint_sec = inode->i_security + ima_blob_sizes.lbs_inode; + *iint_sec = iint; +} + +struct ima_iint_cache *ima_iint_find(struct inode *inode); +struct ima_iint_cache *ima_inode_get(struct inode *inode); +void ima_inode_free(struct inode *inode); +void __init ima_iintcache_init(void); + extern const int read_idmap[]; #ifdef CONFIG_HAVE_IMA_KEXEC @@ -152,8 +253,8 @@ int ima_calc_field_array_hash(struct ima_field_data *field_data, struct ima_template_entry *entry); int ima_calc_boot_aggregate(struct ima_digest_data *hash); void ima_add_violation(struct file *file, const unsigned char *filename, - struct integrity_iint_cache *iint, - const char *op, const char *cause); + struct ima_iint_cache *iint, const char *op, + const char *cause); int ima_init_crypto(void); void ima_putc(struct seq_file *m, void *data, int datalen); void ima_print_digest(struct seq_file *m, u8 *digest, u32 size); @@ -267,10 +368,10 @@ int ima_get_action(struct mnt_idmap *idmap, struct inode *inode, struct ima_template_desc **template_desc, const char *func_data, unsigned int *allowed_algos); int ima_must_measure(struct inode *inode, int mask, enum ima_hooks func); -int ima_collect_measurement(struct integrity_iint_cache *iint, - struct file *file, void *buf, loff_t size, - enum hash_algo algo, struct modsig *modsig); -void ima_store_measurement(struct integrity_iint_cache *iint, struct file *file, +int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file, + void *buf, loff_t size, enum hash_algo algo, + struct modsig *modsig); +void ima_store_measurement(struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig, int pcr, @@ -280,7 +381,7 @@ int process_buffer_measurement(struct mnt_idmap *idmap, const char *eventname, enum ima_hooks func, int pcr, const char *func_data, bool buf_hash, u8 *digest, size_t digest_len); -void ima_audit_measurement(struct integrity_iint_cache *iint, +void ima_audit_measurement(struct ima_iint_cache *iint, const unsigned char *filename); int ima_alloc_init_template(struct ima_event_data *event_data, struct ima_template_entry **entry, @@ -318,17 +419,16 @@ int ima_policy_show(struct seq_file *m, void *v); #define IMA_APPRAISE_KEXEC 0x40 #ifdef CONFIG_IMA_APPRAISE -int ima_check_blacklist(struct integrity_iint_cache *iint, +int ima_check_blacklist(struct ima_iint_cache *iint, const struct modsig *modsig, int pcr); -int ima_appraise_measurement(enum ima_hooks func, - struct integrity_iint_cache *iint, +int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig); int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode, int mask, enum ima_hooks func); -void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file); -enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, +void ima_update_xattr(struct ima_iint_cache *iint, struct file *file); +enum integrity_status ima_get_cache_status(struct ima_iint_cache *iint, enum ima_hooks func); enum hash_algo ima_get_hash_algo(const struct evm_ima_xattr_data *xattr_value, int xattr_len); @@ -337,14 +437,14 @@ int ima_read_xattr(struct dentry *dentry, void __init init_ima_appraise_lsm(const struct lsm_id *lsmid); #else -static inline int ima_check_blacklist(struct integrity_iint_cache *iint, +static inline int ima_check_blacklist(struct ima_iint_cache *iint, const struct modsig *modsig, int pcr) { return 0; } static inline int ima_appraise_measurement(enum ima_hooks func, - struct integrity_iint_cache *iint, + struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, @@ -361,14 +461,13 @@ static inline int ima_must_appraise(struct mnt_idmap *idmap, return 0; } -static inline void ima_update_xattr(struct integrity_iint_cache *iint, +static inline void ima_update_xattr(struct ima_iint_cache *iint, struct file *file) { } -static inline enum integrity_status ima_get_cache_status(struct integrity_iint_cache - *iint, - enum ima_hooks func) +static inline enum integrity_status +ima_get_cache_status(struct ima_iint_cache *iint, enum ima_hooks func) { return INTEGRITY_UNKNOWN; } diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c index 597ea0c4d72f..b37d043d5748 100644 --- a/security/integrity/ima/ima_api.c +++ b/security/integrity/ima/ima_api.c @@ -131,8 +131,8 @@ int ima_store_template(struct ima_template_entry *entry, * value is invalidated. */ void ima_add_violation(struct file *file, const unsigned char *filename, - struct integrity_iint_cache *iint, - const char *op, const char *cause) + struct ima_iint_cache *iint, const char *op, + const char *cause) { struct ima_template_entry *entry; struct inode *inode = file_inode(file); @@ -201,7 +201,8 @@ int ima_get_action(struct mnt_idmap *idmap, struct inode *inode, allowed_algos); } -static bool ima_get_verity_digest(struct integrity_iint_cache *iint, +static bool ima_get_verity_digest(struct ima_iint_cache *iint, + struct inode *inode, struct ima_max_digest_data *hash) { enum hash_algo alg; @@ -211,7 +212,7 @@ static bool ima_get_verity_digest(struct integrity_iint_cache *iint, * On failure, 'measure' policy rules will result in a file data * hash containing 0's. */ - digest_len = fsverity_get_digest(iint->inode, hash->digest, NULL, &alg); + digest_len = fsverity_get_digest(inode, hash->digest, NULL, &alg); if (digest_len == 0) return false; @@ -237,9 +238,9 @@ static bool ima_get_verity_digest(struct integrity_iint_cache *iint, * * Return 0 on success, error code otherwise */ -int ima_collect_measurement(struct integrity_iint_cache *iint, - struct file *file, void *buf, loff_t size, - enum hash_algo algo, struct modsig *modsig) +int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file, + void *buf, loff_t size, enum hash_algo algo, + struct modsig *modsig) { const char *audit_cause = "failed"; struct inode *inode = file_inode(file); @@ -280,7 +281,7 @@ int ima_collect_measurement(struct integrity_iint_cache *iint, memset(&hash.digest, 0, sizeof(hash.digest)); if (iint->flags & IMA_VERITY_REQUIRED) { - if (!ima_get_verity_digest(iint, &hash)) { + if (!ima_get_verity_digest(iint, inode, &hash)) { audit_cause = "no-verity-digest"; result = -ENODATA; } @@ -338,8 +339,8 @@ out: * * Must be called with iint->mutex held. */ -void ima_store_measurement(struct integrity_iint_cache *iint, - struct file *file, const unsigned char *filename, +void ima_store_measurement(struct ima_iint_cache *iint, struct file *file, + const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig, int pcr, struct ima_template_desc *template_desc) @@ -382,7 +383,7 @@ void ima_store_measurement(struct integrity_iint_cache *iint, ima_free_template_entry(entry); } -void ima_audit_measurement(struct integrity_iint_cache *iint, +void ima_audit_measurement(struct ima_iint_cache *iint, const unsigned char *filename) { struct audit_buffer *ab; diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 1dd6ee72a20a..3497741caea9 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -84,8 +84,7 @@ int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode, NULL, NULL, NULL); } -static int ima_fix_xattr(struct dentry *dentry, - struct integrity_iint_cache *iint) +static int ima_fix_xattr(struct dentry *dentry, struct ima_iint_cache *iint) { int rc, offset; u8 algo = iint->ima_hash->algo; @@ -106,7 +105,7 @@ static int ima_fix_xattr(struct dentry *dentry, } /* Return specific func appraised cached result */ -enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, +enum integrity_status ima_get_cache_status(struct ima_iint_cache *iint, enum ima_hooks func) { switch (func) { @@ -126,7 +125,7 @@ enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, } } -static void ima_set_cache_status(struct integrity_iint_cache *iint, +static void ima_set_cache_status(struct ima_iint_cache *iint, enum ima_hooks func, enum integrity_status status) { @@ -152,8 +151,7 @@ static void ima_set_cache_status(struct integrity_iint_cache *iint, } } -static void ima_cache_flags(struct integrity_iint_cache *iint, - enum ima_hooks func) +static void ima_cache_flags(struct ima_iint_cache *iint, enum ima_hooks func) { switch (func) { case MMAP_CHECK: @@ -276,7 +274,7 @@ static int calc_file_id_hash(enum evm_ima_xattr_type type, * * Return 0 on success, error code otherwise. */ -static int xattr_verify(enum ima_hooks func, struct integrity_iint_cache *iint, +static int xattr_verify(enum ima_hooks func, struct ima_iint_cache *iint, struct evm_ima_xattr_data *xattr_value, int xattr_len, enum integrity_status *status, const char **cause) { @@ -443,7 +441,7 @@ static int modsig_verify(enum ima_hooks func, const struct modsig *modsig, * * Returns -EPERM if the hash is blacklisted. */ -int ima_check_blacklist(struct integrity_iint_cache *iint, +int ima_check_blacklist(struct ima_iint_cache *iint, const struct modsig *modsig, int pcr) { enum hash_algo hash_algo; @@ -477,8 +475,7 @@ int ima_check_blacklist(struct integrity_iint_cache *iint, * * Return 0 on success, error code otherwise */ -int ima_appraise_measurement(enum ima_hooks func, - struct integrity_iint_cache *iint, +int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig) @@ -603,7 +600,7 @@ out: /* * ima_update_xattr - update 'security.ima' hash value */ -void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) +void ima_update_xattr(struct ima_iint_cache *iint, struct file *file) { struct dentry *dentry = file_dentry(file); int rc = 0; @@ -640,7 +637,7 @@ static void ima_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid) { struct inode *inode = d_backing_inode(dentry); - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; int action; if (!(ima_policy_flag & IMA_APPRAISE) || !S_ISREG(inode->i_mode) @@ -648,7 +645,7 @@ static void ima_inode_post_setattr(struct mnt_idmap *idmap, return; action = ima_must_appraise(idmap, inode, MAY_ACCESS, POST_SETATTR); - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (iint) { set_bit(IMA_CHANGE_ATTR, &iint->atomic_flags); if (!action) @@ -674,12 +671,12 @@ static int ima_protect_xattr(struct dentry *dentry, const char *xattr_name, static void ima_reset_appraise_flags(struct inode *inode, int digsig) { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; if (!(ima_policy_flag & IMA_APPRAISE) || !S_ISREG(inode->i_mode)) return; - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (!iint) return; iint->measured_pcrs = 0; diff --git a/security/integrity/ima/ima_iint.c b/security/integrity/ima/ima_iint.c new file mode 100644 index 000000000000..e7c9c216c1c6 --- /dev/null +++ b/security/integrity/ima/ima_iint.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2008 IBM Corporation + * + * Authors: + * Mimi Zohar + * + * File: ima_iint.c + * - implements the IMA hook: ima_inode_free + * - cache integrity information in the inode security blob + */ +#include + +#include "ima.h" + +static struct kmem_cache *ima_iint_cache __ro_after_init; + +/** + * ima_iint_find - Return the iint associated with an inode + * @inode: Pointer to the inode + * + * Return the IMA integrity information (iint) associated with an inode, if the + * inode was processed by IMA. + * + * Return: Found iint or NULL. + */ +struct ima_iint_cache *ima_iint_find(struct inode *inode) +{ + if (!IS_IMA(inode)) + return NULL; + + return ima_inode_get_iint(inode); +} + +#define IMA_MAX_NESTING (FILESYSTEM_MAX_STACK_DEPTH + 1) + +/* + * It is not clear that IMA should be nested at all, but as long is it measures + * files both on overlayfs and on underlying fs, we need to annotate the iint + * mutex to avoid lockdep false positives related to IMA + overlayfs. + * See ovl_lockdep_annotate_inode_mutex_key() for more details. + */ +static inline void ima_iint_lockdep_annotate(struct ima_iint_cache *iint, + struct inode *inode) +{ +#ifdef CONFIG_LOCKDEP + static struct lock_class_key ima_iint_mutex_key[IMA_MAX_NESTING]; + + int depth = inode->i_sb->s_stack_depth; + + if (WARN_ON_ONCE(depth < 0 || depth >= IMA_MAX_NESTING)) + depth = 0; + + lockdep_set_class(&iint->mutex, &ima_iint_mutex_key[depth]); +#endif +} + +static void ima_iint_init_always(struct ima_iint_cache *iint, + struct inode *inode) +{ + iint->ima_hash = NULL; + iint->version = 0; + iint->flags = 0UL; + iint->atomic_flags = 0UL; + iint->ima_file_status = INTEGRITY_UNKNOWN; + iint->ima_mmap_status = INTEGRITY_UNKNOWN; + iint->ima_bprm_status = INTEGRITY_UNKNOWN; + iint->ima_read_status = INTEGRITY_UNKNOWN; + iint->ima_creds_status = INTEGRITY_UNKNOWN; + iint->measured_pcrs = 0; + mutex_init(&iint->mutex); + ima_iint_lockdep_annotate(iint, inode); +} + +static void ima_iint_free(struct ima_iint_cache *iint) +{ + kfree(iint->ima_hash); + mutex_destroy(&iint->mutex); + kmem_cache_free(ima_iint_cache, iint); +} + +/** + * ima_inode_get - Find or allocate an iint associated with an inode + * @inode: Pointer to the inode + * + * Find an iint associated with an inode, and allocate a new one if not found. + * Caller must lock i_mutex. + * + * Return: An iint on success, NULL on error. + */ +struct ima_iint_cache *ima_inode_get(struct inode *inode) +{ + struct ima_iint_cache *iint; + + iint = ima_iint_find(inode); + if (iint) + return iint; + + iint = kmem_cache_alloc(ima_iint_cache, GFP_NOFS); + if (!iint) + return NULL; + + ima_iint_init_always(iint, inode); + + inode->i_flags |= S_IMA; + ima_inode_set_iint(inode, iint); + + return iint; +} + +/** + * ima_inode_free - Called on inode free + * @inode: Pointer to the inode + * + * Free the iint associated with an inode. + */ +void ima_inode_free(struct inode *inode) +{ + struct ima_iint_cache *iint; + + if (!IS_IMA(inode)) + return; + + iint = ima_iint_find(inode); + ima_inode_set_iint(inode, NULL); + + ima_iint_free(iint); +} + +static void ima_iint_init_once(void *foo) +{ + struct ima_iint_cache *iint = (struct ima_iint_cache *)foo; + + memset(iint, 0, sizeof(*iint)); +} + +void __init ima_iintcache_init(void) +{ + ima_iint_cache = + kmem_cache_create("ima_iint_cache", sizeof(struct ima_iint_cache), + 0, SLAB_PANIC, ima_iint_init_once); +} diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c index 63979aefc95f..393f5c7912d5 100644 --- a/security/integrity/ima/ima_init.c +++ b/security/integrity/ima/ima_init.c @@ -44,7 +44,7 @@ static int __init ima_add_boot_aggregate(void) static const char op[] = "add_boot_aggregate"; const char *audit_cause = "ENOMEM"; struct ima_template_entry *entry; - struct integrity_iint_cache tmp_iint, *iint = &tmp_iint; + struct ima_iint_cache tmp_iint, *iint = &tmp_iint; struct ima_event_data event_data = { .iint = iint, .filename = boot_aggregate_name }; struct ima_max_digest_data hash; diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index b8b8891a9379..c84e8c55333d 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -114,7 +114,7 @@ static int mmap_violation_check(enum ima_hooks func, struct file *file, * */ static void ima_rdwr_violation_check(struct file *file, - struct integrity_iint_cache *iint, + struct ima_iint_cache *iint, int must_measure, char **pathbuf, const char **pathname, @@ -127,7 +127,7 @@ static void ima_rdwr_violation_check(struct file *file, if (mode & FMODE_WRITE) { if (atomic_read(&inode->i_readcount) && IS_IMA(inode)) { if (!iint) - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); /* IMA_MEASURE is set from reader side */ if (iint && test_bit(IMA_MUST_MEASURE, &iint->atomic_flags)) @@ -153,7 +153,7 @@ static void ima_rdwr_violation_check(struct file *file, "invalid_pcr", "open_writers"); } -static void ima_check_last_writer(struct integrity_iint_cache *iint, +static void ima_check_last_writer(struct ima_iint_cache *iint, struct inode *inode, struct file *file) { fmode_t mode = file->f_mode; @@ -192,12 +192,12 @@ static void ima_check_last_writer(struct integrity_iint_cache *iint, static void ima_file_free(struct file *file) { struct inode *inode = file_inode(file); - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; if (!ima_policy_flag || !S_ISREG(inode->i_mode)) return; - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (!iint) return; @@ -209,7 +209,7 @@ static int process_measurement(struct file *file, const struct cred *cred, enum ima_hooks func) { struct inode *backing_inode, *inode = file_inode(file); - struct integrity_iint_cache *iint = NULL; + struct ima_iint_cache *iint = NULL; struct ima_template_desc *template_desc = NULL; char *pathbuf = NULL; char filename[NAME_MAX]; @@ -248,7 +248,7 @@ static int process_measurement(struct file *file, const struct cred *cred, inode_lock(inode); if (action) { - iint = integrity_inode_get(inode); + iint = ima_inode_get(inode); if (!iint) rc = -ENOMEM; } @@ -564,11 +564,11 @@ static int ima_file_check(struct file *file, int mask) static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, size_t buf_size) { - struct integrity_iint_cache *iint = NULL, tmp_iint; + struct ima_iint_cache *iint = NULL, tmp_iint; int rc, hash_algo; if (ima_policy_flag) { - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (iint) mutex_lock(&iint->mutex); } @@ -578,7 +578,6 @@ static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, mutex_unlock(&iint->mutex); memset(&tmp_iint, 0, sizeof(tmp_iint)); - tmp_iint.inode = inode; mutex_init(&tmp_iint.mutex); rc = ima_collect_measurement(&tmp_iint, file, NULL, 0, @@ -688,7 +687,7 @@ static void ima_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode) { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; int must_appraise; if (!ima_policy_flag || !S_ISREG(inode->i_mode)) @@ -700,7 +699,7 @@ static void ima_post_create_tmpfile(struct mnt_idmap *idmap, return; /* Nothing to do if we can't allocate memory */ - iint = integrity_inode_get(inode); + iint = ima_inode_get(inode); if (!iint) return; @@ -719,7 +718,7 @@ static void ima_post_create_tmpfile(struct mnt_idmap *idmap, */ static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; struct inode *inode = dentry->d_inode; int must_appraise; @@ -732,7 +731,7 @@ static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) return; /* Nothing to do if we can't allocate memory */ - iint = integrity_inode_get(inode); + iint = ima_inode_get(inode); if (!iint) return; @@ -935,7 +934,7 @@ int process_buffer_measurement(struct mnt_idmap *idmap, int ret = 0; const char *audit_cause = "ENOMEM"; struct ima_template_entry *entry = NULL; - struct integrity_iint_cache iint = {}; + struct ima_iint_cache iint = {}; struct ima_event_data event_data = {.iint = &iint, .filename = eventname, .buf = buf, @@ -1172,6 +1171,7 @@ static struct security_hook_list ima_hooks[] __ro_after_init = { #ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS LSM_HOOK_INIT(kernel_module_request, ima_kernel_module_request), #endif + LSM_HOOK_INIT(inode_free_security, ima_inode_free), }; static const struct lsm_id ima_lsmid = { @@ -1181,15 +1181,21 @@ static const struct lsm_id ima_lsmid = { static int __init init_ima_lsm(void) { + ima_iintcache_init(); security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid); init_ima_appraise_lsm(&ima_lsmid); return 0; } +struct lsm_blob_sizes ima_blob_sizes __ro_after_init = { + .lbs_inode = sizeof(struct ima_iint_cache *), +}; + DEFINE_LSM(ima) = { .name = "ima", .init = init_ima_lsm, .order = LSM_ORDER_LAST, + .blobs = &ima_blob_sizes, }; late_initcall(init_ima); /* Start IMA after the TPM is available */ diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index f69062617754..c0556907c2e6 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -49,7 +49,7 @@ #define DONT_HASH 0x0200 #define INVALID_PCR(a) (((a) < 0) || \ - (a) >= (sizeof_field(struct integrity_iint_cache, measured_pcrs) * 8)) + (a) >= (sizeof_field(struct ima_iint_cache, measured_pcrs) * 8)) int ima_policy_flag; static int temp_ima_appraise; diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 7a97c269a072..671fc50255f9 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -20,59 +20,6 @@ #include #include -/* iint action cache flags */ -#define IMA_MEASURE 0x00000001 -#define IMA_MEASURED 0x00000002 -#define IMA_APPRAISE 0x00000004 -#define IMA_APPRAISED 0x00000008 -/*#define IMA_COLLECT 0x00000010 do not use this flag */ -#define IMA_COLLECTED 0x00000020 -#define IMA_AUDIT 0x00000040 -#define IMA_AUDITED 0x00000080 -#define IMA_HASH 0x00000100 -#define IMA_HASHED 0x00000200 - -/* iint policy rule cache flags */ -#define IMA_NONACTION_FLAGS 0xff000000 -#define IMA_DIGSIG_REQUIRED 0x01000000 -#define IMA_PERMIT_DIRECTIO 0x02000000 -#define IMA_NEW_FILE 0x04000000 -#define IMA_FAIL_UNVERIFIABLE_SIGS 0x10000000 -#define IMA_MODSIG_ALLOWED 0x20000000 -#define IMA_CHECK_BLACKLIST 0x40000000 -#define IMA_VERITY_REQUIRED 0x80000000 - -#define IMA_DO_MASK (IMA_MEASURE | IMA_APPRAISE | IMA_AUDIT | \ - IMA_HASH | IMA_APPRAISE_SUBMASK) -#define IMA_DONE_MASK (IMA_MEASURED | IMA_APPRAISED | IMA_AUDITED | \ - IMA_HASHED | IMA_COLLECTED | \ - IMA_APPRAISED_SUBMASK) - -/* iint subaction appraise cache flags */ -#define IMA_FILE_APPRAISE 0x00001000 -#define IMA_FILE_APPRAISED 0x00002000 -#define IMA_MMAP_APPRAISE 0x00004000 -#define IMA_MMAP_APPRAISED 0x00008000 -#define IMA_BPRM_APPRAISE 0x00010000 -#define IMA_BPRM_APPRAISED 0x00020000 -#define IMA_READ_APPRAISE 0x00040000 -#define IMA_READ_APPRAISED 0x00080000 -#define IMA_CREDS_APPRAISE 0x00100000 -#define IMA_CREDS_APPRAISED 0x00200000 -#define IMA_APPRAISE_SUBMASK (IMA_FILE_APPRAISE | IMA_MMAP_APPRAISE | \ - IMA_BPRM_APPRAISE | IMA_READ_APPRAISE | \ - IMA_CREDS_APPRAISE) -#define IMA_APPRAISED_SUBMASK (IMA_FILE_APPRAISED | IMA_MMAP_APPRAISED | \ - IMA_BPRM_APPRAISED | IMA_READ_APPRAISED | \ - IMA_CREDS_APPRAISED) - -/* iint cache atomic_flags */ -#define IMA_CHANGE_XATTR 0 -#define IMA_UPDATE_XATTR 1 -#define IMA_CHANGE_ATTR 2 -#define IMA_DIGSIG 3 -#define IMA_MUST_MEASURE 4 - enum evm_ima_xattr_type { IMA_XATTR_DIGEST = 0x01, EVM_XATTR_HMAC, From b6c0dec9f78bc691166d60347b52b3adf03d5875 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Thu, 15 Feb 2024 11:31:13 +0100 Subject: [PATCH 0430/1038] integrity: Remove LSM Since now IMA and EVM use their own integrity metadata, it is safe to remove the 'integrity' LSM, with its management of integrity metadata. Keep the iint.c file only for loading IMA and EVM keys at boot, and for creating the integrity directory in securityfs (we need to keep it for retrocompatibility reasons). Signed-off-by: Roberto Sassu Reviewed-by: Casey Schaufler Reviewed-by: Stefan Berger Reviewed-by: Mimi Zohar Acked-by: Mimi Zohar Signed-off-by: Paul Moore --- include/linux/integrity.h | 14 --- security/integrity/iint.c | 197 +-------------------------------- security/integrity/integrity.h | 25 ----- security/security.c | 2 - 4 files changed, 2 insertions(+), 236 deletions(-) diff --git a/include/linux/integrity.h b/include/linux/integrity.h index ef0f63ef5ebc..459b79683783 100644 --- a/include/linux/integrity.h +++ b/include/linux/integrity.h @@ -19,24 +19,10 @@ enum integrity_status { INTEGRITY_UNKNOWN, }; -/* List of EVM protected security xattrs */ #ifdef CONFIG_INTEGRITY -extern struct integrity_iint_cache *integrity_inode_get(struct inode *inode); -extern void integrity_inode_free(struct inode *inode); extern void __init integrity_load_keys(void); #else -static inline struct integrity_iint_cache * - integrity_inode_get(struct inode *inode) -{ - return NULL; -} - -static inline void integrity_inode_free(struct inode *inode) -{ - return; -} - static inline void integrity_load_keys(void) { } diff --git a/security/integrity/iint.c b/security/integrity/iint.c index d4419a2a1e24..068ac6c2ae1e 100644 --- a/security/integrity/iint.c +++ b/security/integrity/iint.c @@ -6,207 +6,14 @@ * Mimi Zohar * * File: integrity_iint.c - * - implements the integrity hooks: integrity_inode_alloc, - * integrity_inode_free - * - cache integrity information associated with an inode - * using a rbtree tree. + * - initialize the integrity directory in securityfs + * - load IMA and EVM keys */ -#include -#include -#include -#include -#include -#include #include -#include #include "integrity.h" -static struct rb_root integrity_iint_tree = RB_ROOT; -static DEFINE_RWLOCK(integrity_iint_lock); -static struct kmem_cache *iint_cache __ro_after_init; - struct dentry *integrity_dir; -/* - * __integrity_iint_find - return the iint associated with an inode - */ -static struct integrity_iint_cache *__integrity_iint_find(struct inode *inode) -{ - struct integrity_iint_cache *iint; - struct rb_node *n = integrity_iint_tree.rb_node; - - while (n) { - iint = rb_entry(n, struct integrity_iint_cache, rb_node); - - if (inode < iint->inode) - n = n->rb_left; - else if (inode > iint->inode) - n = n->rb_right; - else - return iint; - } - - return NULL; -} - -/* - * integrity_iint_find - return the iint associated with an inode - */ -struct integrity_iint_cache *integrity_iint_find(struct inode *inode) -{ - struct integrity_iint_cache *iint; - - if (!IS_IMA(inode)) - return NULL; - - read_lock(&integrity_iint_lock); - iint = __integrity_iint_find(inode); - read_unlock(&integrity_iint_lock); - - return iint; -} - -#define IMA_MAX_NESTING (FILESYSTEM_MAX_STACK_DEPTH+1) - -/* - * It is not clear that IMA should be nested at all, but as long is it measures - * files both on overlayfs and on underlying fs, we need to annotate the iint - * mutex to avoid lockdep false positives related to IMA + overlayfs. - * See ovl_lockdep_annotate_inode_mutex_key() for more details. - */ -static inline void iint_lockdep_annotate(struct integrity_iint_cache *iint, - struct inode *inode) -{ -#ifdef CONFIG_LOCKDEP - static struct lock_class_key iint_mutex_key[IMA_MAX_NESTING]; - - int depth = inode->i_sb->s_stack_depth; - - if (WARN_ON_ONCE(depth < 0 || depth >= IMA_MAX_NESTING)) - depth = 0; - - lockdep_set_class(&iint->mutex, &iint_mutex_key[depth]); -#endif -} - -static void iint_init_always(struct integrity_iint_cache *iint, - struct inode *inode) -{ - iint->ima_hash = NULL; - iint->version = 0; - iint->flags = 0UL; - iint->atomic_flags = 0UL; - iint->ima_file_status = INTEGRITY_UNKNOWN; - iint->ima_mmap_status = INTEGRITY_UNKNOWN; - iint->ima_bprm_status = INTEGRITY_UNKNOWN; - iint->ima_read_status = INTEGRITY_UNKNOWN; - iint->ima_creds_status = INTEGRITY_UNKNOWN; - iint->evm_status = INTEGRITY_UNKNOWN; - iint->measured_pcrs = 0; - mutex_init(&iint->mutex); - iint_lockdep_annotate(iint, inode); -} - -static void iint_free(struct integrity_iint_cache *iint) -{ - kfree(iint->ima_hash); - mutex_destroy(&iint->mutex); - kmem_cache_free(iint_cache, iint); -} - -/** - * integrity_inode_get - find or allocate an iint associated with an inode - * @inode: pointer to the inode - * @return: allocated iint - * - * Caller must lock i_mutex - */ -struct integrity_iint_cache *integrity_inode_get(struct inode *inode) -{ - struct rb_node **p; - struct rb_node *node, *parent = NULL; - struct integrity_iint_cache *iint, *test_iint; - - iint = integrity_iint_find(inode); - if (iint) - return iint; - - iint = kmem_cache_alloc(iint_cache, GFP_NOFS); - if (!iint) - return NULL; - - iint_init_always(iint, inode); - - write_lock(&integrity_iint_lock); - - p = &integrity_iint_tree.rb_node; - while (*p) { - parent = *p; - test_iint = rb_entry(parent, struct integrity_iint_cache, - rb_node); - if (inode < test_iint->inode) { - p = &(*p)->rb_left; - } else if (inode > test_iint->inode) { - p = &(*p)->rb_right; - } else { - write_unlock(&integrity_iint_lock); - kmem_cache_free(iint_cache, iint); - return test_iint; - } - } - - iint->inode = inode; - node = &iint->rb_node; - inode->i_flags |= S_IMA; - rb_link_node(node, parent, p); - rb_insert_color(node, &integrity_iint_tree); - - write_unlock(&integrity_iint_lock); - return iint; -} - -/** - * integrity_inode_free - called on security_inode_free - * @inode: pointer to the inode - * - * Free the integrity information(iint) associated with an inode. - */ -void integrity_inode_free(struct inode *inode) -{ - struct integrity_iint_cache *iint; - - if (!IS_IMA(inode)) - return; - - write_lock(&integrity_iint_lock); - iint = __integrity_iint_find(inode); - rb_erase(&iint->rb_node, &integrity_iint_tree); - write_unlock(&integrity_iint_lock); - - iint_free(iint); -} - -static void iint_init_once(void *foo) -{ - struct integrity_iint_cache *iint = (struct integrity_iint_cache *) foo; - - memset(iint, 0, sizeof(*iint)); -} - -static int __init integrity_iintcache_init(void) -{ - iint_cache = - kmem_cache_create("iint_cache", sizeof(struct integrity_iint_cache), - 0, SLAB_PANIC, iint_init_once); - return 0; -} -DEFINE_LSM(integrity) = { - .name = "integrity", - .init = integrity_iintcache_init, - .order = LSM_ORDER_LAST, -}; - - /* * integrity_kernel_read - read data from the file * diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 671fc50255f9..50d6f798e613 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -102,31 +102,6 @@ struct ima_file_id { __u8 hash[HASH_MAX_DIGESTSIZE]; } __packed; -/* integrity data associated with an inode */ -struct integrity_iint_cache { - struct rb_node rb_node; /* rooted in integrity_iint_tree */ - struct mutex mutex; /* protects: version, flags, digest */ - struct inode *inode; /* back pointer to inode in question */ - u64 version; /* track inode changes */ - unsigned long flags; - unsigned long measured_pcrs; - unsigned long atomic_flags; - unsigned long real_ino; - dev_t real_dev; - enum integrity_status ima_file_status:4; - enum integrity_status ima_mmap_status:4; - enum integrity_status ima_bprm_status:4; - enum integrity_status ima_read_status:4; - enum integrity_status ima_creds_status:4; - enum integrity_status evm_status:4; - struct ima_digest_data *ima_hash; -}; - -/* rbtree tree calls to lookup, insert, delete - * integrity data associated with an inode. - */ -struct integrity_iint_cache *integrity_iint_find(struct inode *inode); - int integrity_kernel_read(struct file *file, loff_t offset, void *addr, unsigned long count); diff --git a/security/security.c b/security/security.c index e08c1aa82edf..cbdc9bebe802 100644 --- a/security/security.c +++ b/security/security.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -1597,7 +1596,6 @@ static void inode_free_by_rcu(struct rcu_head *head) */ void security_inode_free(struct inode *inode) { - integrity_inode_free(inode); call_void_hook(inode_free_security, inode); /* * The inode may still be referenced in a path walk and From 91510d5959ad9eac451685e3bfc8385b89c23908 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Fri, 16 Feb 2024 11:59:30 +0100 Subject: [PATCH 0431/1038] gpio: cdev: fix a NULL-pointer dereference with DEBUG enabled We are actually passing the gc pointer to chip_dbg() so we have to srcu_dereference() it. Fixes: 8574b5b47610 ("gpio: cdev: use correct pointer accessors with SRCU") Reported-by: Marek Szyprowski Closes: https://lore.kernel.org/lkml/179caa10-5f86-4707-8bb0-fe1b316326d6@samsung.com/ Signed-off-by: Bartosz Golaszewski Tested-by: Marek Szyprowski --- drivers/gpio/gpiolib-cdev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 85037fa4925e..f384fa278764 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2795,8 +2795,8 @@ int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) return ret; guard(srcu)(&gdev->srcu); - - if (!rcu_access_pointer(gdev->chip)) + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) return -ENODEV; chip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); From 0feda94c868d396fac3b3cb14089d2d989a07c72 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 22 Jan 2024 17:34:00 -0600 Subject: [PATCH 0432/1038] iommu/amd: Mark interrupt as managed On many systems that have an AMD IOMMU the following sequence of warnings is observed during bootup. ``` pci 0000:00:00.2 can't derive routing for PCI INT A pci 0000:00:00.2: PCI INT A: not connected ``` This series of events happens because of the IOMMU initialization sequence order and the lack of _PRT entries for the IOMMU. During initialization the IOMMU driver first enables the PCI device using pci_enable_device(). This will call acpi_pci_irq_enable() which will check if the interrupt is declared in a PCI routing table (_PRT) entry. According to the PCI spec [1] these routing entries are only required under PCI root bridges: The _PRT object is required under all PCI root bridges The IOMMU is directly connected to the root complex, so there is no parent bridge to look for a _PRT entry. The first warning is emitted since no entry could be found in the hierarchy. The second warning is then emitted because the interrupt hasn't yet been configured to any value. The pin was configured in pci_read_irq() but the byte in PCI_INTERRUPT_LINE return 0xff which means "Unknown". After that sequence of events pci_enable_msi() is called and this will allocate an interrupt. That is both of these warnings are totally harmless because the IOMMU uses MSI for interrupts. To avoid even trying to probe for a _PRT entry mark the IOMMU as IRQ managed. This avoids both warnings. Link: https://uefi.org/htmlspecs/ACPI_Spec_6_4_html/06_Device_Configuration/Device_Configuration.html?highlight=_prt#prt-pci-routing-table [1] Signed-off-by: Mario Limonciello Fixes: cffe0a2b5a34 ("x86, irq: Keep balance of IOAPIC pin reference count") Reviewed-by: Vasant Hegde Link: https://lore.kernel.org/r/20240122233400.1802-1-mario.limonciello@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index cb8486c9f8f4..db8d563af444 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2068,6 +2068,9 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) /* Prevent binding other PCI device drivers to IOMMU devices */ iommu->dev->match_driver = false; + /* ACPI _PRT won't have an IRQ for IOMMU */ + iommu->dev->irq_managed = 1; + pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, &iommu->cap); From b07cd3b746cfe9a4524a5a7337fcf4ccb933509b Mon Sep 17 00:00:00 2001 From: Erick Archer Date: Sun, 11 Feb 2024 19:22:50 +0100 Subject: [PATCH 0433/1038] iommu/mtk_iommu: Use devm_kcalloc() instead of devm_kzalloc() This is an effort to get rid of all multiplications from allocation functions in order to prevent integer overflows [1]. Here the multiplication is obviously safe because MTK_PROTECT_PA_ALIGN is defined as a literal value of 256 or 128. For the "mtk_iommu.c" file: 256 For the "mtk_iommu_v1.c" file: 128 However, using devm_kcalloc() is more appropriate [2] and improves readability. This patch has no effect on runtime behavior. Link: https://github.com/KSPP/linux/issues/162 [1] Link: https://www.kernel.org/doc/html/next/process/deprecated.html#open-coded-arithmetic-in-allocator-arguments [2] Signed-off-by: Erick Archer Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Reviewed-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20240211182250.12656-1-erick.archer@gmx.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 2 +- drivers/iommu/mtk_iommu_v1.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7abe9e85a570..9aae6eb604b1 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -1264,7 +1264,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->plat_data = of_device_get_match_data(dev); /* Protect memory. HW will access here while translation fault.*/ - protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); + protect = devm_kcalloc(dev, 2, MTK_PROTECT_PA_ALIGN, GFP_KERNEL); if (!protect) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 25b41222abae..45cd845d153f 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -621,8 +621,8 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) data->dev = dev; /* Protect memory. HW will access here while translation fault.*/ - protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, - GFP_KERNEL | GFP_DMA); + protect = devm_kcalloc(dev, 2, MTK_PROTECT_PA_ALIGN, + GFP_KERNEL | GFP_DMA); if (!protect) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); From 00a9bc6070434814d39118a0de70c1645f64bf60 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:12 +0800 Subject: [PATCH 0434/1038] iommu: Move iommu fault data to linux/iommu.h The iommu fault data is currently defined in uapi/linux/iommu.h, but is only used inside the iommu subsystem. Move it to linux/iommu.h, where it will be more accessible to kernel drivers. With this done, uapi/linux/iommu.h becomes empty and can be removed from the tree. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- MAINTAINERS | 1 - include/linux/iommu.h | 152 +++++++++++++++++++++++++++++++++- include/uapi/linux/iommu.h | 161 ------------------------------------- 3 files changed, 151 insertions(+), 163 deletions(-) delete mode 100644 include/uapi/linux/iommu.h diff --git a/MAINTAINERS b/MAINTAINERS index 960512bec428..1bcdf3144415 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11240,7 +11240,6 @@ F: drivers/iommu/ F: include/linux/iommu.h F: include/linux/iova.h F: include/linux/of_iommu.h -F: include/uapi/linux/iommu.h IOMMUFD M: Jason Gunthorpe diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1ea2a820e1eb..472a8ce029b1 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -14,7 +14,6 @@ #include #include #include -#include #define IOMMU_READ (1 << 0) #define IOMMU_WRITE (1 << 1) @@ -44,6 +43,157 @@ struct iommu_sva; struct iommu_fault_event; struct iommu_dma_cookie; +#define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ +#define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ +#define IOMMU_FAULT_PERM_EXEC (1 << 2) /* exec */ +#define IOMMU_FAULT_PERM_PRIV (1 << 3) /* privileged */ + +/* Generic fault types, can be expanded IRQ remapping fault */ +enum iommu_fault_type { + IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ + IOMMU_FAULT_PAGE_REQ, /* page request fault */ +}; + +enum iommu_fault_reason { + IOMMU_FAULT_REASON_UNKNOWN = 0, + + /* Could not access the PASID table (fetch caused external abort) */ + IOMMU_FAULT_REASON_PASID_FETCH, + + /* PASID entry is invalid or has configuration errors */ + IOMMU_FAULT_REASON_BAD_PASID_ENTRY, + + /* + * PASID is out of range (e.g. exceeds the maximum PASID + * supported by the IOMMU) or disabled. + */ + IOMMU_FAULT_REASON_PASID_INVALID, + + /* + * An external abort occurred fetching (or updating) a translation + * table descriptor + */ + IOMMU_FAULT_REASON_WALK_EABT, + + /* + * Could not access the page table entry (Bad address), + * actual translation fault + */ + IOMMU_FAULT_REASON_PTE_FETCH, + + /* Protection flag check failed */ + IOMMU_FAULT_REASON_PERMISSION, + + /* access flag check failed */ + IOMMU_FAULT_REASON_ACCESS, + + /* Output address of a translation stage caused Address Size fault */ + IOMMU_FAULT_REASON_OOR_ADDRESS, +}; + +/** + * struct iommu_fault_unrecoverable - Unrecoverable fault data + * @reason: reason of the fault, from &enum iommu_fault_reason + * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) + * @pasid: Process Address Space ID + * @perm: requested permission access using by the incoming transaction + * (IOMMU_FAULT_PERM_* values) + * @addr: offending page address + * @fetch_addr: address that caused a fetch abort, if any + */ +struct iommu_fault_unrecoverable { + __u32 reason; +#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) +#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) +#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) + __u32 flags; + __u32 pasid; + __u32 perm; + __u64 addr; + __u64 fetch_addr; +}; + +/** + * struct iommu_fault_page_request - Page Request data + * @flags: encodes whether the corresponding fields are valid and whether this + * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). + * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response + * must have the same PASID value as the page request. When it is clear, + * the page response should not have a PASID. + * @pasid: Process Address Space ID + * @grpid: Page Request Group Index + * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) + * @addr: page address + * @private_data: device-specific private information + */ +struct iommu_fault_page_request { +#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) +#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) +#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) +#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) + __u32 flags; + __u32 pasid; + __u32 grpid; + __u32 perm; + __u64 addr; + __u64 private_data[2]; +}; + +/** + * struct iommu_fault - Generic fault data + * @type: fault type from &enum iommu_fault_type + * @padding: reserved for future use (should be zero) + * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV + * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ + * @padding2: sets the fault size to allow for future extensions + */ +struct iommu_fault { + __u32 type; + __u32 padding; + union { + struct iommu_fault_unrecoverable event; + struct iommu_fault_page_request prm; + __u8 padding2[56]; + }; +}; + +/** + * enum iommu_page_response_code - Return status of fault handlers + * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables + * populated, retry the access. This is "Success" in PCI PRI. + * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from + * this device if possible. This is "Response Failure" in PCI PRI. + * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the + * access. This is "Invalid Request" in PCI PRI. + */ +enum iommu_page_response_code { + IOMMU_PAGE_RESP_SUCCESS = 0, + IOMMU_PAGE_RESP_INVALID, + IOMMU_PAGE_RESP_FAILURE, +}; + +/** + * struct iommu_page_response - Generic page response information + * @argsz: User filled size of this data + * @version: API version of this structure + * @flags: encodes whether the corresponding fields are valid + * (IOMMU_FAULT_PAGE_RESPONSE_* values) + * @pasid: Process Address Space ID + * @grpid: Page Request Group Index + * @code: response code from &enum iommu_page_response_code + */ +struct iommu_page_response { + __u32 argsz; +#define IOMMU_PAGE_RESP_VERSION_1 1 + __u32 version; +#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) + __u32 flags; + __u32 pasid; + __u32 grpid; + __u32 code; +}; + + /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 #define IOMMU_FAULT_WRITE 0x1 diff --git a/include/uapi/linux/iommu.h b/include/uapi/linux/iommu.h deleted file mode 100644 index 65d8b0234f69..000000000000 --- a/include/uapi/linux/iommu.h +++ /dev/null @@ -1,161 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * IOMMU user API definitions - */ - -#ifndef _UAPI_IOMMU_H -#define _UAPI_IOMMU_H - -#include - -#define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ -#define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ -#define IOMMU_FAULT_PERM_EXEC (1 << 2) /* exec */ -#define IOMMU_FAULT_PERM_PRIV (1 << 3) /* privileged */ - -/* Generic fault types, can be expanded IRQ remapping fault */ -enum iommu_fault_type { - IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ - IOMMU_FAULT_PAGE_REQ, /* page request fault */ -}; - -enum iommu_fault_reason { - IOMMU_FAULT_REASON_UNKNOWN = 0, - - /* Could not access the PASID table (fetch caused external abort) */ - IOMMU_FAULT_REASON_PASID_FETCH, - - /* PASID entry is invalid or has configuration errors */ - IOMMU_FAULT_REASON_BAD_PASID_ENTRY, - - /* - * PASID is out of range (e.g. exceeds the maximum PASID - * supported by the IOMMU) or disabled. - */ - IOMMU_FAULT_REASON_PASID_INVALID, - - /* - * An external abort occurred fetching (or updating) a translation - * table descriptor - */ - IOMMU_FAULT_REASON_WALK_EABT, - - /* - * Could not access the page table entry (Bad address), - * actual translation fault - */ - IOMMU_FAULT_REASON_PTE_FETCH, - - /* Protection flag check failed */ - IOMMU_FAULT_REASON_PERMISSION, - - /* access flag check failed */ - IOMMU_FAULT_REASON_ACCESS, - - /* Output address of a translation stage caused Address Size fault */ - IOMMU_FAULT_REASON_OOR_ADDRESS, -}; - -/** - * struct iommu_fault_unrecoverable - Unrecoverable fault data - * @reason: reason of the fault, from &enum iommu_fault_reason - * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) - * @pasid: Process Address Space ID - * @perm: requested permission access using by the incoming transaction - * (IOMMU_FAULT_PERM_* values) - * @addr: offending page address - * @fetch_addr: address that caused a fetch abort, if any - */ -struct iommu_fault_unrecoverable { - __u32 reason; -#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) -#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) -#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) - __u32 flags; - __u32 pasid; - __u32 perm; - __u64 addr; - __u64 fetch_addr; -}; - -/** - * struct iommu_fault_page_request - Page Request data - * @flags: encodes whether the corresponding fields are valid and whether this - * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). - * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response - * must have the same PASID value as the page request. When it is clear, - * the page response should not have a PASID. - * @pasid: Process Address Space ID - * @grpid: Page Request Group Index - * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) - * @addr: page address - * @private_data: device-specific private information - */ -struct iommu_fault_page_request { -#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) -#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) -#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) -#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 perm; - __u64 addr; - __u64 private_data[2]; -}; - -/** - * struct iommu_fault - Generic fault data - * @type: fault type from &enum iommu_fault_type - * @padding: reserved for future use (should be zero) - * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV - * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ - * @padding2: sets the fault size to allow for future extensions - */ -struct iommu_fault { - __u32 type; - __u32 padding; - union { - struct iommu_fault_unrecoverable event; - struct iommu_fault_page_request prm; - __u8 padding2[56]; - }; -}; - -/** - * enum iommu_page_response_code - Return status of fault handlers - * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables - * populated, retry the access. This is "Success" in PCI PRI. - * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from - * this device if possible. This is "Response Failure" in PCI PRI. - * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the - * access. This is "Invalid Request" in PCI PRI. - */ -enum iommu_page_response_code { - IOMMU_PAGE_RESP_SUCCESS = 0, - IOMMU_PAGE_RESP_INVALID, - IOMMU_PAGE_RESP_FAILURE, -}; - -/** - * struct iommu_page_response - Generic page response information - * @argsz: User filled size of this data - * @version: API version of this structure - * @flags: encodes whether the corresponding fields are valid - * (IOMMU_FAULT_PAGE_RESPONSE_* values) - * @pasid: Process Address Space ID - * @grpid: Page Request Group Index - * @code: response code from &enum iommu_page_response_code - */ -struct iommu_page_response { - __u32 argsz; -#define IOMMU_PAGE_RESP_VERSION_1 1 - __u32 version; -#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 code; -}; - -#endif /* _UAPI_IOMMU_H */ From 66014df73b302d326b995178141a150b9a3a52b7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:13 +0800 Subject: [PATCH 0435/1038] iommu/arm-smmu-v3: Remove unrecoverable faults reporting No device driver registers fault handler to handle the reported unrecoveraable faults. Remove it to avoid dead code. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 46 ++++++--------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 0ffb1cf17e0b..4cf1054ed321 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1461,7 +1461,6 @@ arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) { int ret; - u32 reason; u32 perm = 0; struct arm_smmu_master *master; bool ssid_valid = evt[0] & EVTQ_0_SSV; @@ -1471,16 +1470,9 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) switch (FIELD_GET(EVTQ_0_ID, evt[0])) { case EVT_ID_TRANSLATION_FAULT: - reason = IOMMU_FAULT_REASON_PTE_FETCH; - break; case EVT_ID_ADDR_SIZE_FAULT: - reason = IOMMU_FAULT_REASON_OOR_ADDRESS; - break; case EVT_ID_ACCESS_FAULT: - reason = IOMMU_FAULT_REASON_ACCESS; - break; case EVT_ID_PERMISSION_FAULT: - reason = IOMMU_FAULT_REASON_PERMISSION; break; default: return -EOPNOTSUPP; @@ -1490,6 +1482,9 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) if (evt[1] & EVTQ_1_S2) return -EFAULT; + if (!(evt[1] & EVTQ_1_STALL)) + return -EOPNOTSUPP; + if (evt[1] & EVTQ_1_RnW) perm |= IOMMU_FAULT_PERM_READ; else @@ -1501,32 +1496,17 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) if (evt[1] & EVTQ_1_PnU) perm |= IOMMU_FAULT_PERM_PRIV; - if (evt[1] & EVTQ_1_STALL) { - flt->type = IOMMU_FAULT_PAGE_REQ; - flt->prm = (struct iommu_fault_page_request) { - .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE, - .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]), - .perm = perm, - .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), - }; + flt->type = IOMMU_FAULT_PAGE_REQ; + flt->prm = (struct iommu_fault_page_request) { + .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE, + .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]), + .perm = perm, + .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), + }; - if (ssid_valid) { - flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; - flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); - } - } else { - flt->type = IOMMU_FAULT_DMA_UNRECOV; - flt->event = (struct iommu_fault_unrecoverable) { - .reason = reason, - .flags = IOMMU_FAULT_UNRECOV_ADDR_VALID, - .perm = perm, - .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), - }; - - if (ssid_valid) { - flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID; - flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); - } + if (ssid_valid) { + flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); } mutex_lock(&smmu->streams_mutex); From 0edeab66eba88947dabe8634a3efd136cc771750 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:14 +0800 Subject: [PATCH 0436/1038] iommu: Remove unrecoverable fault data The unrecoverable fault data is not used anywhere. Remove it to avoid dead code. Suggested-by: Kevin Tian Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 72 ++----------------------------------------- 1 file changed, 2 insertions(+), 70 deletions(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 472a8ce029b1..c960c4fae3bc 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -50,67 +50,7 @@ struct iommu_dma_cookie; /* Generic fault types, can be expanded IRQ remapping fault */ enum iommu_fault_type { - IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ - IOMMU_FAULT_PAGE_REQ, /* page request fault */ -}; - -enum iommu_fault_reason { - IOMMU_FAULT_REASON_UNKNOWN = 0, - - /* Could not access the PASID table (fetch caused external abort) */ - IOMMU_FAULT_REASON_PASID_FETCH, - - /* PASID entry is invalid or has configuration errors */ - IOMMU_FAULT_REASON_BAD_PASID_ENTRY, - - /* - * PASID is out of range (e.g. exceeds the maximum PASID - * supported by the IOMMU) or disabled. - */ - IOMMU_FAULT_REASON_PASID_INVALID, - - /* - * An external abort occurred fetching (or updating) a translation - * table descriptor - */ - IOMMU_FAULT_REASON_WALK_EABT, - - /* - * Could not access the page table entry (Bad address), - * actual translation fault - */ - IOMMU_FAULT_REASON_PTE_FETCH, - - /* Protection flag check failed */ - IOMMU_FAULT_REASON_PERMISSION, - - /* access flag check failed */ - IOMMU_FAULT_REASON_ACCESS, - - /* Output address of a translation stage caused Address Size fault */ - IOMMU_FAULT_REASON_OOR_ADDRESS, -}; - -/** - * struct iommu_fault_unrecoverable - Unrecoverable fault data - * @reason: reason of the fault, from &enum iommu_fault_reason - * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) - * @pasid: Process Address Space ID - * @perm: requested permission access using by the incoming transaction - * (IOMMU_FAULT_PERM_* values) - * @addr: offending page address - * @fetch_addr: address that caused a fetch abort, if any - */ -struct iommu_fault_unrecoverable { - __u32 reason; -#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) -#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) -#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) - __u32 flags; - __u32 pasid; - __u32 perm; - __u64 addr; - __u64 fetch_addr; + IOMMU_FAULT_PAGE_REQ = 1, /* page request fault */ }; /** @@ -142,19 +82,11 @@ struct iommu_fault_page_request { /** * struct iommu_fault - Generic fault data * @type: fault type from &enum iommu_fault_type - * @padding: reserved for future use (should be zero) - * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ - * @padding2: sets the fault size to allow for future extensions */ struct iommu_fault { __u32 type; - __u32 padding; - union { - struct iommu_fault_unrecoverable event; - struct iommu_fault_page_request prm; - __u8 padding2[56]; - }; + struct iommu_fault_page_request prm; }; /** From 8b32a3bea2629049c484f595af7aad797e24453e Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:15 +0800 Subject: [PATCH 0437/1038] iommu: Cleanup iopf data structure definitions struct iommu_fault_page_request and struct iommu_page_response are not part of uAPI anymore. Convert them to data structures for kAPI. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 1 - drivers/iommu/iommu.c | 4 ---- include/linux/iommu.h | 27 +++++++++++---------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index e5b8b9110c13..24b5545352ae 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -56,7 +56,6 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, enum iommu_page_response_code status) { struct iommu_page_response resp = { - .version = IOMMU_PAGE_RESP_VERSION_1, .pasid = iopf->fault.prm.pasid, .grpid = iopf->fault.prm.grpid, .code = status, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..6a4d9bc0641d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1494,10 +1494,6 @@ int iommu_page_response(struct device *dev, if (!param || !param->fault_param) return -EINVAL; - if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || - msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) - return -EINVAL; - /* Only send response if there is a fault report pending */ mutex_lock(¶m->fault_param->lock); if (list_empty(¶m->fault_param->faults)) { diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c960c4fae3bc..829bcb5a8e23 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -71,12 +71,12 @@ struct iommu_fault_page_request { #define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) #define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) #define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 perm; - __u64 addr; - __u64 private_data[2]; + u32 flags; + u32 pasid; + u32 grpid; + u32 perm; + u64 addr; + u64 private_data[2]; }; /** @@ -85,7 +85,7 @@ struct iommu_fault_page_request { * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ */ struct iommu_fault { - __u32 type; + u32 type; struct iommu_fault_page_request prm; }; @@ -106,8 +106,6 @@ enum iommu_page_response_code { /** * struct iommu_page_response - Generic page response information - * @argsz: User filled size of this data - * @version: API version of this structure * @flags: encodes whether the corresponding fields are valid * (IOMMU_FAULT_PAGE_RESPONSE_* values) * @pasid: Process Address Space ID @@ -115,14 +113,11 @@ enum iommu_page_response_code { * @code: response code from &enum iommu_page_response_code */ struct iommu_page_response { - __u32 argsz; -#define IOMMU_PAGE_RESP_VERSION_1 1 - __u32 version; #define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 code; + u32 flags; + u32 pasid; + u32 grpid; + u32 code; }; From 15fc60cdd2d236a73b32c99d21fc0f7b7ce6cbbb Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:16 +0800 Subject: [PATCH 0438/1038] iommu: Merge iopf_device_param into iommu_fault_param The struct dev_iommu contains two pointers, fault_param and iopf_param. The fault_param pointer points to a data structure that is used to store pending faults that are awaiting responses. The iopf_param pointer points to a data structure that is used to store partial faults that are part of a Page Request Group. The fault_param and iopf_param pointers are essentially duplicate. This causes memory waste. Merge the iopf_device_param pointer into the iommu_fault_param pointer to consolidate the code and save memory. The consolidated pointer would be allocated on demand when the device driver enables the iopf on device, and would be freed after iopf is disabled. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 110 ++++++++++++++++++------------------- drivers/iommu/iommu.c | 34 ++---------- include/linux/iommu.h | 18 ++++-- 3 files changed, 72 insertions(+), 90 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 24b5545352ae..f948303b2a91 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -25,21 +25,6 @@ struct iopf_queue { struct mutex lock; }; -/** - * struct iopf_device_param - IO Page Fault data attached to a device - * @dev: the device that owns this param - * @queue: IOPF queue - * @queue_list: index into queue->devices - * @partial: faults that are part of a Page Request Group for which the last - * request hasn't been submitted yet. - */ -struct iopf_device_param { - struct device *dev; - struct iopf_queue *queue; - struct list_head queue_list; - struct list_head partial; -}; - struct iopf_fault { struct iommu_fault fault; struct list_head list; @@ -144,7 +129,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) int ret; struct iopf_group *group; struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; struct device *dev = cookie; struct dev_iommu *param = dev->iommu; @@ -159,7 +144,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) * As long as we're holding param->lock, the queue can't be unlinked * from the device and therefore cannot disappear. */ - iopf_param = param->iopf_param; + iopf_param = param->fault_param; if (!iopf_param) return -ENODEV; @@ -229,14 +214,14 @@ EXPORT_SYMBOL_GPL(iommu_queue_iopf); int iopf_queue_flush_dev(struct device *dev) { int ret = 0; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; struct dev_iommu *param = dev->iommu; if (!param) return -ENODEV; mutex_lock(¶m->lock); - iopf_param = param->iopf_param; + iopf_param = param->fault_param; if (iopf_param) flush_workqueue(iopf_param->queue->wq); else @@ -260,7 +245,7 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); int iopf_queue_discard_partial(struct iopf_queue *queue) { struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; if (!queue) return -EINVAL; @@ -287,34 +272,36 @@ EXPORT_SYMBOL_GPL(iopf_queue_discard_partial); */ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) { - int ret = -EBUSY; - struct iopf_device_param *iopf_param; + int ret = 0; struct dev_iommu *param = dev->iommu; - - if (!param) - return -ENODEV; - - iopf_param = kzalloc(sizeof(*iopf_param), GFP_KERNEL); - if (!iopf_param) - return -ENOMEM; - - INIT_LIST_HEAD(&iopf_param->partial); - iopf_param->queue = queue; - iopf_param->dev = dev; + struct iommu_fault_param *fault_param; mutex_lock(&queue->lock); mutex_lock(¶m->lock); - if (!param->iopf_param) { - list_add(&iopf_param->queue_list, &queue->devices); - param->iopf_param = iopf_param; - ret = 0; + if (param->fault_param) { + ret = -EBUSY; + goto done_unlock; } + + fault_param = kzalloc(sizeof(*fault_param), GFP_KERNEL); + if (!fault_param) { + ret = -ENOMEM; + goto done_unlock; + } + + mutex_init(&fault_param->lock); + INIT_LIST_HEAD(&fault_param->faults); + INIT_LIST_HEAD(&fault_param->partial); + fault_param->dev = dev; + list_add(&fault_param->queue_list, &queue->devices); + fault_param->queue = queue; + + param->fault_param = fault_param; + +done_unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); - if (ret) - kfree(iopf_param); - return ret; } EXPORT_SYMBOL_GPL(iopf_queue_add_device); @@ -330,34 +317,41 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); */ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - int ret = -EINVAL; + int ret = 0; struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; struct dev_iommu *param = dev->iommu; - - if (!param || !queue) - return -EINVAL; + struct iommu_fault_param *fault_param = param->fault_param; mutex_lock(&queue->lock); mutex_lock(¶m->lock); - iopf_param = param->iopf_param; - if (iopf_param && iopf_param->queue == queue) { - list_del(&iopf_param->queue_list); - param->iopf_param = NULL; - ret = 0; + if (!fault_param) { + ret = -ENODEV; + goto unlock; } - mutex_unlock(¶m->lock); - mutex_unlock(&queue->lock); - if (ret) - return ret; + + if (fault_param->queue != queue) { + ret = -EINVAL; + goto unlock; + } + + if (!list_empty(&fault_param->faults)) { + ret = -EBUSY; + goto unlock; + } + + list_del(&fault_param->queue_list); /* Just in case some faults are still stuck */ - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) + list_for_each_entry_safe(iopf, next, &fault_param->partial, list) kfree(iopf); - kfree(iopf_param); + param->fault_param = NULL; + kfree(fault_param); +unlock: + mutex_unlock(¶m->lock); + mutex_unlock(&queue->lock); - return 0; + return ret; } EXPORT_SYMBOL_GPL(iopf_queue_remove_device); @@ -403,7 +397,7 @@ EXPORT_SYMBOL_GPL(iopf_queue_alloc); */ void iopf_queue_free(struct iopf_queue *queue) { - struct iopf_device_param *iopf_param, *next; + struct iommu_fault_param *iopf_param, *next; if (!queue) return; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 6a4d9bc0641d..9575cb75732a 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1355,27 +1355,18 @@ int iommu_register_device_fault_handler(struct device *dev, struct dev_iommu *param = dev->iommu; int ret = 0; - if (!param) + if (!param || !param->fault_param) return -EINVAL; mutex_lock(¶m->lock); /* Only allow one fault handler registered for each device */ - if (param->fault_param) { + if (param->fault_param->handler) { ret = -EBUSY; goto done_unlock; } - get_device(dev); - param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); - if (!param->fault_param) { - put_device(dev); - ret = -ENOMEM; - goto done_unlock; - } param->fault_param->handler = handler; param->fault_param->data = data; - mutex_init(¶m->fault_param->lock); - INIT_LIST_HEAD(¶m->fault_param->faults); done_unlock: mutex_unlock(¶m->lock); @@ -1396,29 +1387,16 @@ EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); int iommu_unregister_device_fault_handler(struct device *dev) { struct dev_iommu *param = dev->iommu; - int ret = 0; - if (!param) + if (!param || !param->fault_param) return -EINVAL; mutex_lock(¶m->lock); - - if (!param->fault_param) - goto unlock; - - /* we cannot unregister handler if there are pending faults */ - if (!list_empty(¶m->fault_param->faults)) { - ret = -EBUSY; - goto unlock; - } - - kfree(param->fault_param); - param->fault_param = NULL; - put_device(dev); -unlock: + param->fault_param->handler = NULL; + param->fault_param->data = NULL; mutex_unlock(¶m->lock); - return ret; + return 0; } EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 829bcb5a8e23..bbb7c2ad5184 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -42,6 +42,7 @@ struct notifier_block; struct iommu_sva; struct iommu_fault_event; struct iommu_dma_cookie; +struct iopf_queue; #define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ #define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ @@ -672,21 +673,31 @@ struct iommu_fault_event { * struct iommu_fault_param - per-device IOMMU fault data * @handler: Callback function to handle IOMMU faults at device level * @data: handler private data - * @faults: holds the pending faults which needs response * @lock: protect pending faults list + * @dev: the device that owns this param + * @queue: IOPF queue + * @queue_list: index into queue->devices + * @partial: faults that are part of a Page Request Group for which the last + * request hasn't been submitted yet. + * @faults: holds the pending faults which need response */ struct iommu_fault_param { iommu_dev_fault_handler_t handler; void *data; - struct list_head faults; struct mutex lock; + + struct device *dev; + struct iopf_queue *queue; + struct list_head queue_list; + + struct list_head partial; + struct list_head faults; }; /** * struct dev_iommu - Collection of per-device IOMMU data * * @fault_param: IOMMU detected device fault reporting data - * @iopf_param: I/O Page Fault queue and data * @fwspec: IOMMU fwspec data * @iommu_dev: IOMMU device this device is linked to * @priv: IOMMU Driver private data @@ -702,7 +713,6 @@ struct iommu_fault_param { struct dev_iommu { struct mutex lock; struct iommu_fault_param *fault_param; - struct iopf_device_param *iopf_param; struct iommu_fwspec *fwspec; struct iommu_device *iommu_dev; void *priv; From 1ff25d798e52943d037accf15c675a6845d9776f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:17 +0800 Subject: [PATCH 0439/1038] iommu: Remove iommu_[un]register_device_fault_handler() The individual iommu driver reports the iommu page faults by calling iommu_report_device_fault(), where a pre-registered device fault handler is called to route the fault to another fault handler installed on the corresponding iommu domain. The pre-registered device fault handler is static and won't be dynamic as the fault handler is eventually per iommu domain. Replace calling device fault handler with iommu_queue_iopf(). After this replacement, the registering and unregistering fault handler interfaces are not needed anywhere. Remove the interfaces and the related data structures to avoid dead code. Convert cookie parameter of iommu_queue_iopf() into a device pointer that is really passed. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 13 +--- drivers/iommu/intel/iommu.c | 24 ++---- drivers/iommu/io-pgfault.c | 6 +- drivers/iommu/iommu-sva.h | 4 +- drivers/iommu/iommu.c | 76 +------------------ include/linux/iommu.h | 23 ------ 6 files changed, 13 insertions(+), 133 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 05722121f00e..ab2b0a5e4369 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -487,7 +487,6 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) { - int ret; struct device *dev = master->dev; /* @@ -500,16 +499,7 @@ static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) if (!master->iopf_enabled) return -EINVAL; - ret = iopf_queue_add_device(master->smmu->evtq.iopf, dev); - if (ret) - return ret; - - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) { - iopf_queue_remove_device(master->smmu->evtq.iopf, dev); - return ret; - } - return 0; + return iopf_queue_add_device(master->smmu->evtq.iopf, dev); } static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) @@ -519,7 +509,6 @@ static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) if (!master->iopf_enabled) return; - iommu_unregister_device_fault_handler(dev); iopf_queue_remove_device(master->smmu->evtq.iopf, dev); } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 6fb5f6fceea1..df6ceefc09ee 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4427,23 +4427,15 @@ static int intel_iommu_enable_iopf(struct device *dev) if (ret) return ret; - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) - goto iopf_remove_device; - ret = pci_enable_pri(pdev, PRQ_DEPTH); - if (ret) - goto iopf_unregister_handler; + if (ret) { + iopf_queue_remove_device(iommu->iopf_queue, dev); + return ret; + } + info->pri_enabled = 1; return 0; - -iopf_unregister_handler: - iommu_unregister_device_fault_handler(dev); -iopf_remove_device: - iopf_queue_remove_device(iommu->iopf_queue, dev); - - return ret; } static int intel_iommu_disable_iopf(struct device *dev) @@ -4466,11 +4458,9 @@ static int intel_iommu_disable_iopf(struct device *dev) info->pri_enabled = 0; /* - * With PRI disabled and outstanding PRQs drained, unregistering - * fault handler and removing device from iopf queue should never - * fail. + * With PRI disabled and outstanding PRQs drained, removing device + * from iopf queue should never fail. */ - WARN_ON(iommu_unregister_device_fault_handler(dev)); WARN_ON(iopf_queue_remove_device(iommu->iopf_queue, dev)); return 0; diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index f948303b2a91..4fda01de5589 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -87,7 +87,7 @@ static void iopf_handler(struct work_struct *work) /** * iommu_queue_iopf - IO Page Fault handler * @fault: fault event - * @cookie: struct device, passed to iommu_register_device_fault_handler. + * @dev: struct device. * * Add a fault to the device workqueue, to be handled by mm. * @@ -124,14 +124,12 @@ static void iopf_handler(struct work_struct *work) * * Return: 0 on success and <0 on error. */ -int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) { int ret; struct iopf_group *group; struct iopf_fault *iopf, *next; struct iommu_fault_param *iopf_param; - - struct device *dev = cookie; struct dev_iommu *param = dev->iommu; lockdep_assert_held(¶m->lock); diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h index 54946b5a7caf..de7819c796ce 100644 --- a/drivers/iommu/iommu-sva.h +++ b/drivers/iommu/iommu-sva.h @@ -13,7 +13,7 @@ struct iommu_fault; struct iopf_queue; #ifdef CONFIG_IOMMU_SVA -int iommu_queue_iopf(struct iommu_fault *fault, void *cookie); +int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev); int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); int iopf_queue_remove_device(struct iopf_queue *queue, @@ -26,7 +26,7 @@ enum iommu_page_response_code iommu_sva_handle_iopf(struct iommu_fault *fault, void *data); #else /* CONFIG_IOMMU_SVA */ -static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +static inline int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) { return -ENODEV; } diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 9575cb75732a..5cb81dbe8d9d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1330,76 +1330,6 @@ void iommu_group_put(struct iommu_group *group) } EXPORT_SYMBOL_GPL(iommu_group_put); -/** - * iommu_register_device_fault_handler() - Register a device fault handler - * @dev: the device - * @handler: the fault handler - * @data: private data passed as argument to the handler - * - * When an IOMMU fault event is received, this handler gets called with the - * fault event and data as argument. The handler should return 0 on success. If - * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also - * complete the fault by calling iommu_page_response() with one of the following - * response code: - * - IOMMU_PAGE_RESP_SUCCESS: retry the translation - * - IOMMU_PAGE_RESP_INVALID: terminate the fault - * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting - * page faults if possible. - * - * Return 0 if the fault handler was installed successfully, or an error. - */ -int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data) -{ - struct dev_iommu *param = dev->iommu; - int ret = 0; - - if (!param || !param->fault_param) - return -EINVAL; - - mutex_lock(¶m->lock); - /* Only allow one fault handler registered for each device */ - if (param->fault_param->handler) { - ret = -EBUSY; - goto done_unlock; - } - - param->fault_param->handler = handler; - param->fault_param->data = data; - -done_unlock: - mutex_unlock(¶m->lock); - - return ret; -} -EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); - -/** - * iommu_unregister_device_fault_handler() - Unregister the device fault handler - * @dev: the device - * - * Remove the device fault handler installed with - * iommu_register_device_fault_handler(). - * - * Return 0 on success, or an error. - */ -int iommu_unregister_device_fault_handler(struct device *dev) -{ - struct dev_iommu *param = dev->iommu; - - if (!param || !param->fault_param) - return -EINVAL; - - mutex_lock(¶m->lock); - param->fault_param->handler = NULL; - param->fault_param->data = NULL; - mutex_unlock(¶m->lock); - - return 0; -} -EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); - /** * iommu_report_device_fault() - Report fault event to device driver * @dev: the device @@ -1424,10 +1354,6 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) /* we only report device fault if there is a handler registered */ mutex_lock(¶m->lock); fparam = param->fault_param; - if (!fparam || !fparam->handler) { - ret = -EINVAL; - goto done_unlock; - } if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { @@ -1442,7 +1368,7 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) mutex_unlock(&fparam->lock); } - ret = fparam->handler(&evt->fault, fparam->data); + ret = iommu_queue_iopf(&evt->fault, dev); if (ret && evt_pending) { mutex_lock(&fparam->lock); list_del(&evt_pending->list); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index bbb7c2ad5184..70176c1c5573 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -128,7 +128,6 @@ struct iommu_page_response { typedef int (*iommu_fault_handler_t)(struct iommu_domain *, struct device *, unsigned long, int, void *); -typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); struct iommu_domain_geometry { dma_addr_t aperture_start; /* First address that can be mapped */ @@ -671,8 +670,6 @@ struct iommu_fault_event { /** * struct iommu_fault_param - per-device IOMMU fault data - * @handler: Callback function to handle IOMMU faults at device level - * @data: handler private data * @lock: protect pending faults list * @dev: the device that owns this param * @queue: IOPF queue @@ -682,8 +679,6 @@ struct iommu_fault_event { * @faults: holds the pending faults which need response */ struct iommu_fault_param { - iommu_dev_fault_handler_t handler; - void *data; struct mutex lock; struct device *dev; @@ -806,11 +801,6 @@ extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data); - -extern int iommu_unregister_device_fault_handler(struct device *dev); extern int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt); @@ -1222,19 +1212,6 @@ static inline void iommu_group_put(struct iommu_group *group) { } -static inline -int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data) -{ - return -ENODEV; -} - -static inline int iommu_unregister_device_fault_handler(struct device *dev) -{ - return 0; -} - static inline int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) { From 3f02a9dc70007c0e6299fda9c4f7a1e2277ec3d2 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:18 +0800 Subject: [PATCH 0440/1038] iommu: Merge iommu_fault_event and iopf_fault The iommu_fault_event and iopf_fault data structures store the same information about an iopf fault. They are also used in the same way. Merge these two data structures into a single one to make the code more concise and easier to maintain. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 +-- drivers/iommu/intel/iommu.h | 2 +- drivers/iommu/intel/svm.c | 5 ++-- drivers/iommu/io-pgfault.c | 5 ---- drivers/iommu/iommu.c | 8 +++--- include/linux/iommu.h | 27 ++++++--------------- 6 files changed, 17 insertions(+), 34 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4cf1054ed321..ab4f04c7f932 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -922,7 +922,7 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, } static int arm_smmu_page_response(struct device *dev, - struct iommu_fault_event *unused, + struct iopf_fault *unused, struct iommu_page_response *resp) { struct arm_smmu_cmdq_ent cmd = {0}; @@ -1465,7 +1465,7 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) struct arm_smmu_master *master; bool ssid_valid = evt[0] & EVTQ_0_SSV; u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]); - struct iommu_fault_event fault_evt = { }; + struct iopf_fault fault_evt = { }; struct iommu_fault *flt = &fault_evt.fault; switch (FIELD_GET(EVTQ_0_ID, evt[0])) { diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index d02f916d8e59..696d95293a69 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1079,7 +1079,7 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); -int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, +int intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 40edd282903f..9751f037e188 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -565,13 +565,12 @@ static int prq_to_iommu_prot(struct page_req_dsc *req) static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, struct page_req_dsc *desc) { - struct iommu_fault_event event; + struct iopf_fault event = { }; if (!dev || !dev_is_pci(dev)) return -ENODEV; /* Fill in event data for device specific processing */ - memset(&event, 0, sizeof(struct iommu_fault_event)); event.fault.type = IOMMU_FAULT_PAGE_REQ; event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; event.fault.prm.pasid = desc->pasid; @@ -743,7 +742,7 @@ prq_advance: } int intel_svm_page_response(struct device *dev, - struct iommu_fault_event *evt, + struct iopf_fault *evt, struct iommu_page_response *msg) { struct device_domain_info *info = dev_iommu_priv_get(dev); diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 4fda01de5589..10d48eb72608 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -25,11 +25,6 @@ struct iopf_queue { struct mutex lock; }; -struct iopf_fault { - struct iommu_fault fault; - struct list_head list; -}; - struct iopf_group { struct iopf_fault last_fault; struct list_head faults; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 5cb81dbe8d9d..15d7b2914780 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1341,10 +1341,10 @@ EXPORT_SYMBOL_GPL(iommu_group_put); * * Return 0 on success, or an error. */ -int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { struct dev_iommu *param = dev->iommu; - struct iommu_fault_event *evt_pending = NULL; + struct iopf_fault *evt_pending = NULL; struct iommu_fault_param *fparam; int ret = 0; @@ -1357,7 +1357,7 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), + evt_pending = kmemdup(evt, sizeof(struct iopf_fault), GFP_KERNEL); if (!evt_pending) { ret = -ENOMEM; @@ -1386,7 +1386,7 @@ int iommu_page_response(struct device *dev, { bool needs_pasid; int ret = -EINVAL; - struct iommu_fault_event *evt; + struct iopf_fault *evt; struct iommu_fault_page_request *prm; struct dev_iommu *param = dev->iommu; const struct iommu_ops *ops = dev_iommu_ops(dev); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 70176c1c5573..2320548a90f8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -40,7 +40,6 @@ struct iommu_domain_ops; struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; -struct iommu_fault_event; struct iommu_dma_cookie; struct iopf_queue; @@ -121,6 +120,11 @@ struct iommu_page_response { u32 code; }; +struct iopf_fault { + struct iommu_fault fault; + /* node for pending lists */ + struct list_head list; +}; /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 @@ -553,7 +557,7 @@ struct iommu_ops { int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); int (*page_response)(struct device *dev, - struct iommu_fault_event *evt, + struct iopf_fault *evt, struct iommu_page_response *msg); int (*def_domain_type)(struct device *dev); @@ -654,20 +658,6 @@ struct iommu_device { u32 max_pasids; }; -/** - * struct iommu_fault_event - Generic fault event - * - * Can represent recoverable faults such as a page requests or - * unrecoverable faults such as DMA or IRQ remapping faults. - * - * @fault: fault descriptor - * @list: pending fault event list, used for tracking responses - */ -struct iommu_fault_event { - struct iommu_fault fault; - struct list_head list; -}; - /** * struct iommu_fault_param - per-device IOMMU fault data * @lock: protect pending faults list @@ -802,8 +792,7 @@ extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_report_device_fault(struct device *dev, - struct iommu_fault_event *evt); +extern int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); extern int iommu_page_response(struct device *dev, struct iommu_page_response *msg); @@ -1213,7 +1202,7 @@ static inline void iommu_group_put(struct iommu_group *group) } static inline -int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { return -ENODEV; } From 24b5d268b5ab95c12b5ae58a054d04bfa442f58f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:19 +0800 Subject: [PATCH 0441/1038] iommu: Prepare for separating SVA and IOPF Move iopf_group data structure to iommu.h to make it a minimal set of faults that a domain's page fault handler should handle. Add a new function, iopf_free_group(), to free a fault group after all faults in the group are handled. This function will be made global so that it can be called from other files, such as iommu-sva.c. Move iopf_queue data structure to iommu.h to allow the workqueue to be scheduled out of this file. This will simplify the sequential patches. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 37 +++++++++++++------------------------ include/linux/iommu.h | 20 +++++++++++++++++++- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 10d48eb72608..c7e6bbed5c05 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -13,24 +13,17 @@ #include "iommu-sva.h" -/** - * struct iopf_queue - IO Page Fault queue - * @wq: the fault workqueue - * @devices: devices attached to this queue - * @lock: protects the device list - */ -struct iopf_queue { - struct workqueue_struct *wq; - struct list_head devices; - struct mutex lock; -}; +static void iopf_free_group(struct iopf_group *group) +{ + struct iopf_fault *iopf, *next; -struct iopf_group { - struct iopf_fault last_fault; - struct list_head faults; - struct work_struct work; - struct device *dev; -}; + list_for_each_entry_safe(iopf, next, &group->faults, list) { + if (!(iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) + kfree(iopf); + } + + kfree(group); +} static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, enum iommu_page_response_code status) @@ -50,9 +43,9 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, static void iopf_handler(struct work_struct *work) { + struct iopf_fault *iopf; struct iopf_group *group; struct iommu_domain *domain; - struct iopf_fault *iopf, *next; enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; group = container_of(work, struct iopf_group, work); @@ -61,7 +54,7 @@ static void iopf_handler(struct work_struct *work) if (!domain || !domain->iopf_handler) status = IOMMU_PAGE_RESP_INVALID; - list_for_each_entry_safe(iopf, next, &group->faults, list) { + list_for_each_entry(iopf, &group->faults, list) { /* * For the moment, errors are sticky: don't handle subsequent * faults in the group if there is an error. @@ -69,14 +62,10 @@ static void iopf_handler(struct work_struct *work) if (status == IOMMU_PAGE_RESP_SUCCESS) status = domain->iopf_handler(&iopf->fault, domain->fault_data); - - if (!(iopf->fault.prm.flags & - IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) - kfree(iopf); } iopf_complete_group(group->dev, &group->last_fault, status); - kfree(group); + iopf_free_group(group); } /** diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 2320548a90f8..c9d4f175f121 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -41,7 +41,6 @@ struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; struct iommu_dma_cookie; -struct iopf_queue; #define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ #define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ @@ -126,6 +125,25 @@ struct iopf_fault { struct list_head list; }; +struct iopf_group { + struct iopf_fault last_fault; + struct list_head faults; + struct work_struct work; + struct device *dev; +}; + +/** + * struct iopf_queue - IO Page Fault queue + * @wq: the fault workqueue + * @devices: devices attached to this queue + * @lock: protects the device list + */ +struct iopf_queue { + struct workqueue_struct *wq; + struct list_head devices; + struct mutex lock; +}; + /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 #define IOMMU_FAULT_WRITE 0x1 From 351ffcb11ca0ff64e399982e279cfa131e7cb1aa Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:20 +0800 Subject: [PATCH 0442/1038] iommu: Make iommu_queue_iopf() more generic Make iommu_queue_iopf() more generic by making the iopf_group a minimal set of iopf's that an iopf handler of domain should handle and respond to. Add domain parameter to struct iopf_group so that the handler can retrieve and use it directly. Change iommu_queue_iopf() to forward groups of iopf's to the domain's iopf handler. This is also a necessary step to decouple the sva iopf handling code from this interface. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Reviewed-by: Yi Liu Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 68 +++++++++++++++++++++++++++++++------- drivers/iommu/iommu-sva.c | 3 +- drivers/iommu/iommu-sva.h | 6 ++-- include/linux/iommu.h | 4 +-- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index c7e6bbed5c05..13cd0929e766 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -13,6 +13,9 @@ #include "iommu-sva.h" +enum iommu_page_response_code +iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm); + static void iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -45,29 +48,48 @@ static void iopf_handler(struct work_struct *work) { struct iopf_fault *iopf; struct iopf_group *group; - struct iommu_domain *domain; enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; group = container_of(work, struct iopf_group, work); - domain = iommu_get_domain_for_dev_pasid(group->dev, - group->last_fault.fault.prm.pasid, 0); - if (!domain || !domain->iopf_handler) - status = IOMMU_PAGE_RESP_INVALID; - list_for_each_entry(iopf, &group->faults, list) { /* * For the moment, errors are sticky: don't handle subsequent * faults in the group if there is an error. */ - if (status == IOMMU_PAGE_RESP_SUCCESS) - status = domain->iopf_handler(&iopf->fault, - domain->fault_data); + if (status != IOMMU_PAGE_RESP_SUCCESS) + break; + + status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); } iopf_complete_group(group->dev, &group->last_fault, status); iopf_free_group(group); } +static struct iommu_domain *get_domain_for_iopf(struct device *dev, + struct iommu_fault *fault) +{ + struct iommu_domain *domain; + + if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) { + domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0); + if (IS_ERR(domain)) + domain = NULL; + } else { + domain = iommu_get_domain_for_dev(dev); + } + + if (!domain || !domain->iopf_handler) { + dev_warn_ratelimited(dev, + "iopf (pasid %d) without domain attached or handler installed\n", + fault->prm.pasid); + + return NULL; + } + + return domain; +} + /** * iommu_queue_iopf - IO Page Fault handler * @fault: fault event @@ -112,6 +134,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) { int ret; struct iopf_group *group; + struct iommu_domain *domain; struct iopf_fault *iopf, *next; struct iommu_fault_param *iopf_param; struct dev_iommu *param = dev->iommu; @@ -143,6 +166,12 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) return 0; } + domain = get_domain_for_iopf(dev, fault); + if (!domain) { + ret = -EINVAL; + goto cleanup_partial; + } + group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) { /* @@ -157,8 +186,8 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) group->dev = dev; group->last_fault.fault = *fault; INIT_LIST_HEAD(&group->faults); + group->domain = domain; list_add(&group->last_fault.list, &group->faults); - INIT_WORK(&group->work, iopf_handler); /* See if we have partial faults for this group */ list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { @@ -167,9 +196,13 @@ int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) list_move(&iopf->list, &group->faults); } - queue_work(iopf_param->queue->wq, &group->work); - return 0; + mutex_unlock(&iopf_param->lock); + ret = domain->iopf_handler(group); + mutex_lock(&iopf_param->lock); + if (ret) + iopf_free_group(group); + return ret; cleanup_partial: list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { if (iopf->fault.prm.grpid == fault->prm.grpid) { @@ -181,6 +214,17 @@ cleanup_partial: } EXPORT_SYMBOL_GPL(iommu_queue_iopf); +int iommu_sva_handle_iopf(struct iopf_group *group) +{ + struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + + INIT_WORK(&group->work, iopf_handler); + if (!queue_work(fault_param->queue->wq, &group->work)) + return -EBUSY; + + return 0; +} + /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed * @dev: the endpoint whose faults need to be flushed. diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index c3fc9201d0be..fcae7308fcb7 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -163,11 +163,10 @@ EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); * I/O page fault handler for SVA */ enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data) +iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) { vm_fault_t ret; struct vm_area_struct *vma; - struct mm_struct *mm = data; unsigned int access_flags = 0; unsigned int fault_flags = FAULT_FLAG_REMOTE; struct iommu_fault_page_request *prm = &fault->prm; diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h index de7819c796ce..27c8da115b41 100644 --- a/drivers/iommu/iommu-sva.h +++ b/drivers/iommu/iommu-sva.h @@ -22,8 +22,7 @@ int iopf_queue_flush_dev(struct device *dev); struct iopf_queue *iopf_queue_alloc(const char *name); void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); -enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data); +int iommu_sva_handle_iopf(struct iopf_group *group); #else /* CONFIG_IOMMU_SVA */ static inline int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) @@ -62,8 +61,7 @@ static inline int iopf_queue_discard_partial(struct iopf_queue *queue) return -ENODEV; } -static inline enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data) +static inline int iommu_sva_handle_iopf(struct iopf_group *group) { return IOMMU_PAGE_RESP_INVALID; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c9d4f175f121..791f183a988e 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -130,6 +130,7 @@ struct iopf_group { struct list_head faults; struct work_struct work; struct device *dev; + struct iommu_domain *domain; }; /** @@ -209,8 +210,7 @@ struct iommu_domain { unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ struct iommu_domain_geometry geometry; struct iommu_dma_cookie *iova_cookie; - enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault, - void *data); + int (*iopf_handler)(struct iopf_group *group); void *fault_data; union { struct { From 17c51a0ea36b800e7a5998a92d83016c82935dff Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:21 +0800 Subject: [PATCH 0443/1038] iommu: Separate SVA and IOPF Add CONFIG_IOMMU_IOPF for page fault handling framework and select it from its real consumer. Move iopf function declaration from iommu-sva.h to iommu.h and remove iommu-sva.h as it's empty now. Consolidate all SVA related code into iommu-sva.c: - Move iommu_sva_domain_alloc() from iommu.c to iommu-sva.c. - Move sva iopf handling code from io-pgfault.c to iommu-sva.c. Consolidate iommu_report_device_fault() and iommu_page_response() into io-pgfault.c. Export iopf_free_group() and iopf_group_response() for iopf handlers implemented in modules. Some functions are renamed with more meaningful names. No other intentional functionality changes. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-11-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 4 + drivers/iommu/Makefile | 3 +- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 1 - drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 1 - drivers/iommu/intel/Kconfig | 1 + drivers/iommu/intel/iommu.c | 1 - drivers/iommu/intel/svm.c | 1 - drivers/iommu/io-pgfault.c | 188 +++++++++++++----- drivers/iommu/iommu-sva.c | 68 ++++++- drivers/iommu/iommu-sva.h | 69 ------- drivers/iommu/iommu.c | 133 ------------- include/linux/iommu.h | 98 ++++++--- 12 files changed, 277 insertions(+), 291 deletions(-) delete mode 100644 drivers/iommu/iommu-sva.h diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 9a29d742617e..d9ed5ad129f2 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -163,6 +163,9 @@ config IOMMU_SVA select IOMMU_MM_DATA bool +config IOMMU_IOPF + bool + config FSL_PAMU bool "Freescale IOMMU support" depends on PCI @@ -398,6 +401,7 @@ config ARM_SMMU_V3_SVA bool "Shared Virtual Addressing support for the ARM SMMUv3" depends on ARM_SMMU_V3 select IOMMU_SVA + select IOMMU_IOPF select MMU_NOTIFIER help Support for sharing process address spaces with devices using the diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 95ad9dbfbda0..542760d963ec 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o -obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o +obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o +obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index ab2b0a5e4369..6513a98fcb72 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -10,7 +10,6 @@ #include #include "arm-smmu-v3.h" -#include "../../iommu-sva.h" #include "../../io-pgtable-arm.h" struct arm_smmu_mmu_notifier { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index ab4f04c7f932..4e93e845458c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -29,7 +29,6 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -#include "../../iommu-sva.h" static bool disable_bypass = true; module_param(disable_bypass, bool, 0444); diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 012cd2541a68..a4a125666293 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -51,6 +51,7 @@ config INTEL_IOMMU_SVM depends on X86_64 select MMU_NOTIFIER select IOMMU_SVA + select IOMMU_IOPF help Shared Virtual Memory (SVM) provides a facility for devices to access DMA resources through process address space by diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index df6ceefc09ee..29a12f289e2e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -27,7 +27,6 @@ #include "iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" -#include "../iommu-sva.h" #include "pasid.h" #include "cap_audit.h" #include "perfmon.h" diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 9751f037e188..e1cbcb9515f0 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -22,7 +22,6 @@ #include "iommu.h" #include "pasid.h" #include "perf.h" -#include "../iommu-sva.h" #include "trace.h" static irqreturn_t prq_event_thread(int irq, void *d); diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 13cd0929e766..c1e88da973ce 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -11,12 +11,9 @@ #include #include -#include "iommu-sva.h" +#include "iommu-priv.h" -enum iommu_page_response_code -iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm); - -static void iopf_free_group(struct iopf_group *group) +void iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -27,44 +24,7 @@ static void iopf_free_group(struct iopf_group *group) kfree(group); } - -static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, - enum iommu_page_response_code status) -{ - struct iommu_page_response resp = { - .pasid = iopf->fault.prm.pasid, - .grpid = iopf->fault.prm.grpid, - .code = status, - }; - - if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && - (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; - - return iommu_page_response(dev, &resp); -} - -static void iopf_handler(struct work_struct *work) -{ - struct iopf_fault *iopf; - struct iopf_group *group; - enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; - - group = container_of(work, struct iopf_group, work); - list_for_each_entry(iopf, &group->faults, list) { - /* - * For the moment, errors are sticky: don't handle subsequent - * faults in the group if there is an error. - */ - if (status != IOMMU_PAGE_RESP_SUCCESS) - break; - - status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); - } - - iopf_complete_group(group->dev, &group->last_fault, status); - iopf_free_group(group); -} +EXPORT_SYMBOL_GPL(iopf_free_group); static struct iommu_domain *get_domain_for_iopf(struct device *dev, struct iommu_fault *fault) @@ -91,7 +51,7 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, } /** - * iommu_queue_iopf - IO Page Fault handler + * iommu_handle_iopf - IO Page Fault handler * @fault: fault event * @dev: struct device. * @@ -130,7 +90,7 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, * * Return: 0 on success and <0 on error. */ -int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) +static int iommu_handle_iopf(struct iommu_fault *fault, struct device *dev) { int ret; struct iopf_group *group; @@ -212,18 +172,117 @@ cleanup_partial: } return ret; } -EXPORT_SYMBOL_GPL(iommu_queue_iopf); -int iommu_sva_handle_iopf(struct iopf_group *group) +/** + * iommu_report_device_fault() - Report fault event to device driver + * @dev: the device + * @evt: fault event data + * + * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ + * handler. When this function fails and the fault is recoverable, it is the + * caller's responsibility to complete the fault. + * + * Return 0 on success, or an error. + */ +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + struct dev_iommu *param = dev->iommu; + struct iopf_fault *evt_pending = NULL; + struct iommu_fault_param *fparam; + int ret = 0; - INIT_WORK(&group->work, iopf_handler); - if (!queue_work(fault_param->queue->wq, &group->work)) - return -EBUSY; + if (!param || !evt) + return -EINVAL; - return 0; + /* we only report device fault if there is a handler registered */ + mutex_lock(¶m->lock); + fparam = param->fault_param; + + if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && + (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + evt_pending = kmemdup(evt, sizeof(struct iopf_fault), + GFP_KERNEL); + if (!evt_pending) { + ret = -ENOMEM; + goto done_unlock; + } + mutex_lock(&fparam->lock); + list_add_tail(&evt_pending->list, &fparam->faults); + mutex_unlock(&fparam->lock); + } + + ret = iommu_handle_iopf(&evt->fault, dev); + if (ret && evt_pending) { + mutex_lock(&fparam->lock); + list_del(&evt_pending->list); + mutex_unlock(&fparam->lock); + kfree(evt_pending); + } +done_unlock: + mutex_unlock(¶m->lock); + return ret; } +EXPORT_SYMBOL_GPL(iommu_report_device_fault); + +int iommu_page_response(struct device *dev, + struct iommu_page_response *msg) +{ + bool needs_pasid; + int ret = -EINVAL; + struct iopf_fault *evt; + struct iommu_fault_page_request *prm; + struct dev_iommu *param = dev->iommu; + const struct iommu_ops *ops = dev_iommu_ops(dev); + bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; + + if (!ops->page_response) + return -ENODEV; + + if (!param || !param->fault_param) + return -EINVAL; + + /* Only send response if there is a fault report pending */ + mutex_lock(¶m->fault_param->lock); + if (list_empty(¶m->fault_param->faults)) { + dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); + goto done_unlock; + } + /* + * Check if we have a matching page request pending to respond, + * otherwise return -EINVAL + */ + list_for_each_entry(evt, ¶m->fault_param->faults, list) { + prm = &evt->fault.prm; + if (prm->grpid != msg->grpid) + continue; + + /* + * If the PASID is required, the corresponding request is + * matched using the group ID, the PASID valid bit and the PASID + * value. Otherwise only the group ID matches request and + * response. + */ + needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; + if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) + continue; + + if (!needs_pasid && has_pasid) { + /* No big deal, just clear it. */ + msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; + msg->pasid = 0; + } + + ret = ops->page_response(dev, evt, msg); + list_del(&evt->list); + kfree(evt); + break; + } + +done_unlock: + mutex_unlock(¶m->fault_param->lock); + return ret; +} +EXPORT_SYMBOL_GPL(iommu_page_response); /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed @@ -258,6 +317,31 @@ int iopf_queue_flush_dev(struct device *dev) } EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); +/** + * iopf_group_response - Respond a group of page faults + * @group: the group of faults with the same group id + * @status: the response code + * + * Return 0 on success and <0 on error. + */ +int iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) +{ + struct iopf_fault *iopf = &group->last_fault; + struct iommu_page_response resp = { + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = status, + }; + + if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && + (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) + resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + + return iommu_page_response(group->dev, &resp); +} +EXPORT_SYMBOL_GPL(iopf_group_response); + /** * iopf_queue_discard_partial - Remove all pending partial fault * @queue: the queue whose partial faults need to be discarded diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index fcae7308fcb7..9de878e40413 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -7,7 +7,7 @@ #include #include -#include "iommu-sva.h" +#include "iommu-priv.h" static DEFINE_MUTEX(iommu_sva_lock); @@ -159,10 +159,21 @@ u32 iommu_sva_get_pasid(struct iommu_sva *handle) } EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); +void mm_pasid_drop(struct mm_struct *mm) +{ + struct iommu_mm_data *iommu_mm = mm->iommu_mm; + + if (!iommu_mm) + return; + + iommu_free_global_pasid(iommu_mm->pasid); + kfree(iommu_mm); +} + /* * I/O page fault handler for SVA */ -enum iommu_page_response_code +static enum iommu_page_response_code iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) { vm_fault_t ret; @@ -216,13 +227,54 @@ out_put_mm: return status; } -void mm_pasid_drop(struct mm_struct *mm) +static void iommu_sva_handle_iopf(struct work_struct *work) { - struct iommu_mm_data *iommu_mm = mm->iommu_mm; + struct iopf_fault *iopf; + struct iopf_group *group; + enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; - if (!iommu_mm) - return; + group = container_of(work, struct iopf_group, work); + list_for_each_entry(iopf, &group->faults, list) { + /* + * For the moment, errors are sticky: don't handle subsequent + * faults in the group if there is an error. + */ + if (status != IOMMU_PAGE_RESP_SUCCESS) + break; - iommu_free_global_pasid(iommu_mm->pasid); - kfree(iommu_mm); + status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); + } + + iopf_group_response(group, status); + iopf_free_group(group); +} + +static int iommu_sva_iopf_handler(struct iopf_group *group) +{ + struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + + INIT_WORK(&group->work, iommu_sva_handle_iopf); + if (!queue_work(fault_param->queue->wq, &group->work)) + return -EBUSY; + + return 0; +} + +struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, + struct mm_struct *mm) +{ + const struct iommu_ops *ops = dev_iommu_ops(dev); + struct iommu_domain *domain; + + domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); + if (!domain) + return NULL; + + domain->type = IOMMU_DOMAIN_SVA; + mmgrab(mm); + domain->mm = mm; + domain->owner = ops; + domain->iopf_handler = iommu_sva_iopf_handler; + + return domain; } diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h deleted file mode 100644 index 27c8da115b41..000000000000 --- a/drivers/iommu/iommu-sva.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * SVA library for IOMMU drivers - */ -#ifndef _IOMMU_SVA_H -#define _IOMMU_SVA_H - -#include - -/* I/O Page fault */ -struct device; -struct iommu_fault; -struct iopf_queue; - -#ifdef CONFIG_IOMMU_SVA -int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev); - -int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); -int iopf_queue_remove_device(struct iopf_queue *queue, - struct device *dev); -int iopf_queue_flush_dev(struct device *dev); -struct iopf_queue *iopf_queue_alloc(const char *name); -void iopf_queue_free(struct iopf_queue *queue); -int iopf_queue_discard_partial(struct iopf_queue *queue); -int iommu_sva_handle_iopf(struct iopf_group *group); - -#else /* CONFIG_IOMMU_SVA */ -static inline int iommu_queue_iopf(struct iommu_fault *fault, struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_add_device(struct iopf_queue *queue, - struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_remove_device(struct iopf_queue *queue, - struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_flush_dev(struct device *dev) -{ - return -ENODEV; -} - -static inline struct iopf_queue *iopf_queue_alloc(const char *name) -{ - return NULL; -} - -static inline void iopf_queue_free(struct iopf_queue *queue) -{ -} - -static inline int iopf_queue_discard_partial(struct iopf_queue *queue) -{ - return -ENODEV; -} - -static inline int iommu_sva_handle_iopf(struct iopf_group *group) -{ - return IOMMU_PAGE_RESP_INVALID; -} -#endif /* CONFIG_IOMMU_SVA */ -#endif /* _IOMMU_SVA_H */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 15d7b2914780..35dac3e4629a 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -36,8 +36,6 @@ #include "dma-iommu.h" #include "iommu-priv.h" -#include "iommu-sva.h" - static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); static DEFINE_IDA(iommu_global_pasid_ida); @@ -1330,117 +1328,6 @@ void iommu_group_put(struct iommu_group *group) } EXPORT_SYMBOL_GPL(iommu_group_put); -/** - * iommu_report_device_fault() - Report fault event to device driver - * @dev: the device - * @evt: fault event data - * - * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. - * - * Return 0 on success, or an error. - */ -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) -{ - struct dev_iommu *param = dev->iommu; - struct iopf_fault *evt_pending = NULL; - struct iommu_fault_param *fparam; - int ret = 0; - - if (!param || !evt) - return -EINVAL; - - /* we only report device fault if there is a handler registered */ - mutex_lock(¶m->lock); - fparam = param->fault_param; - - if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - evt_pending = kmemdup(evt, sizeof(struct iopf_fault), - GFP_KERNEL); - if (!evt_pending) { - ret = -ENOMEM; - goto done_unlock; - } - mutex_lock(&fparam->lock); - list_add_tail(&evt_pending->list, &fparam->faults); - mutex_unlock(&fparam->lock); - } - - ret = iommu_queue_iopf(&evt->fault, dev); - if (ret && evt_pending) { - mutex_lock(&fparam->lock); - list_del(&evt_pending->list); - mutex_unlock(&fparam->lock); - kfree(evt_pending); - } -done_unlock: - mutex_unlock(¶m->lock); - return ret; -} -EXPORT_SYMBOL_GPL(iommu_report_device_fault); - -int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) -{ - bool needs_pasid; - int ret = -EINVAL; - struct iopf_fault *evt; - struct iommu_fault_page_request *prm; - struct dev_iommu *param = dev->iommu; - const struct iommu_ops *ops = dev_iommu_ops(dev); - bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - - if (!ops->page_response) - return -ENODEV; - - if (!param || !param->fault_param) - return -EINVAL; - - /* Only send response if there is a fault report pending */ - mutex_lock(¶m->fault_param->lock); - if (list_empty(¶m->fault_param->faults)) { - dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); - goto done_unlock; - } - /* - * Check if we have a matching page request pending to respond, - * otherwise return -EINVAL - */ - list_for_each_entry(evt, ¶m->fault_param->faults, list) { - prm = &evt->fault.prm; - if (prm->grpid != msg->grpid) - continue; - - /* - * If the PASID is required, the corresponding request is - * matched using the group ID, the PASID valid bit and the PASID - * value. Otherwise only the group ID matches request and - * response. - */ - needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; - if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) - continue; - - if (!needs_pasid && has_pasid) { - /* No big deal, just clear it. */ - msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; - msg->pasid = 0; - } - - ret = ops->page_response(dev, evt, msg); - list_del(&evt->list); - kfree(evt); - break; - } - -done_unlock: - mutex_unlock(¶m->fault_param->lock); - return ret; -} -EXPORT_SYMBOL_GPL(iommu_page_response); - /** * iommu_group_id - Return ID for a group * @group: the group to ID @@ -3523,26 +3410,6 @@ struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev, } EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid); -struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, - struct mm_struct *mm) -{ - const struct iommu_ops *ops = dev_iommu_ops(dev); - struct iommu_domain *domain; - - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); - if (!domain) - return NULL; - - domain->type = IOMMU_DOMAIN_SVA; - mmgrab(mm); - domain->mm = mm; - domain->owner = ops; - domain->iopf_handler = iommu_sva_handle_iopf; - domain->fault_data = mm; - - return domain; -} - ioasid_t iommu_alloc_global_pasid(struct device *dev) { int ret; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 791f183a988e..fc912aed7886 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -810,10 +810,6 @@ extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); -extern int iommu_page_response(struct device *dev, - struct iommu_page_response *msg); - extern int iommu_group_id(struct iommu_group *group); extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); @@ -1029,8 +1025,6 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group); int iommu_device_claim_dma_owner(struct device *dev, void *owner); void iommu_device_release_dma_owner(struct device *dev); -struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, - struct mm_struct *mm); int iommu_attach_device_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid); void iommu_detach_device_pasid(struct iommu_domain *domain, @@ -1219,18 +1213,6 @@ static inline void iommu_group_put(struct iommu_group *group) { } -static inline -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) -{ - return -ENODEV; -} - -static inline int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) -{ - return -ENODEV; -} - static inline int iommu_group_id(struct iommu_group *group) { return -ENODEV; @@ -1379,12 +1361,6 @@ static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner) return -ENODEV; } -static inline struct iommu_domain * -iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) -{ - return NULL; -} - static inline int iommu_attach_device_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid) { @@ -1524,6 +1500,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm); void iommu_sva_unbind_device(struct iommu_sva *handle); u32 iommu_sva_get_pasid(struct iommu_sva *handle); +struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, + struct mm_struct *mm); #else static inline struct iommu_sva * iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) @@ -1548,6 +1526,78 @@ static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm) } static inline void mm_pasid_drop(struct mm_struct *mm) {} + +static inline struct iommu_domain * +iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) +{ + return NULL; +} #endif /* CONFIG_IOMMU_SVA */ +#ifdef CONFIG_IOMMU_IOPF +int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); +int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); +int iopf_queue_flush_dev(struct device *dev); +struct iopf_queue *iopf_queue_alloc(const char *name); +void iopf_queue_free(struct iopf_queue *queue); +int iopf_queue_discard_partial(struct iopf_queue *queue); +void iopf_free_group(struct iopf_group *group); +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); +int iommu_page_response(struct device *dev, struct iommu_page_response *msg); +int iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status); +#else +static inline int +iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) +{ + return -ENODEV; +} + +static inline int +iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +{ + return -ENODEV; +} + +static inline int iopf_queue_flush_dev(struct device *dev) +{ + return -ENODEV; +} + +static inline struct iopf_queue *iopf_queue_alloc(const char *name) +{ + return NULL; +} + +static inline void iopf_queue_free(struct iopf_queue *queue) +{ +} + +static inline int iopf_queue_discard_partial(struct iopf_queue *queue) +{ + return -ENODEV; +} + +static inline void iopf_free_group(struct iopf_group *group) +{ +} + +static inline int +iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) +{ + return -ENODEV; +} + +static inline int +iommu_page_response(struct device *dev, struct iommu_page_response *msg) +{ + return -ENODEV; +} + +static inline int iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) +{ + return -ENODEV; +} +#endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ From cc7338e9d807e20e60e6720a62956f0e9d46f0f8 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:22 +0800 Subject: [PATCH 0444/1038] iommu: Refine locking for per-device fault data management The per-device fault data is a data structure that is used to store information about faults that occur on a device. This data is allocated when IOPF is enabled on the device and freed when IOPF is disabled. The data is used in the paths of iopf reporting, handling, responding, and draining. The fault data is protected by two locks: - dev->iommu->lock: This lock is used to protect the allocation and freeing of the fault data. - dev->iommu->fault_parameter->lock: This lock is used to protect the fault data itself. Apply the locking mechanism to the fault reporting and responding paths. The fault_parameter->lock is also added in iopf_queue_discard_partial(). It does not fix any real issue, as iopf_queue_discard_partial() is only used in the VT-d driver's prq_event_thread(), which is a single-threaded path that reports the IOPFs. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Tested-by: Yan Zhao Tested-by: Longfang Liu Link: https://lore.kernel.org/r/20240212012227.119381-12-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 61 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index c1e88da973ce..5aea8402be47 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -53,7 +53,7 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, /** * iommu_handle_iopf - IO Page Fault handler * @fault: fault event - * @dev: struct device. + * @iopf_param: the fault parameter of the device. * * Add a fault to the device workqueue, to be handled by mm. * @@ -90,29 +90,21 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, * * Return: 0 on success and <0 on error. */ -static int iommu_handle_iopf(struct iommu_fault *fault, struct device *dev) +static int iommu_handle_iopf(struct iommu_fault *fault, + struct iommu_fault_param *iopf_param) { int ret; struct iopf_group *group; struct iommu_domain *domain; struct iopf_fault *iopf, *next; - struct iommu_fault_param *iopf_param; - struct dev_iommu *param = dev->iommu; + struct device *dev = iopf_param->dev; - lockdep_assert_held(¶m->lock); + lockdep_assert_held(&iopf_param->lock); if (fault->type != IOMMU_FAULT_PAGE_REQ) /* Not a recoverable page fault */ return -EOPNOTSUPP; - /* - * As long as we're holding param->lock, the queue can't be unlinked - * from the device and therefore cannot disappear. - */ - iopf_param = param->fault_param; - if (!iopf_param) - return -ENODEV; - if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); if (!iopf) @@ -186,18 +178,19 @@ cleanup_partial: */ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *fault_param; struct iopf_fault *evt_pending = NULL; - struct iommu_fault_param *fparam; + struct dev_iommu *param = dev->iommu; int ret = 0; - if (!param || !evt) - return -EINVAL; - - /* we only report device fault if there is a handler registered */ mutex_lock(¶m->lock); - fparam = param->fault_param; + fault_param = param->fault_param; + if (!fault_param) { + mutex_unlock(¶m->lock); + return -EINVAL; + } + mutex_lock(&fault_param->lock); if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { evt_pending = kmemdup(evt, sizeof(struct iopf_fault), @@ -206,20 +199,18 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) ret = -ENOMEM; goto done_unlock; } - mutex_lock(&fparam->lock); - list_add_tail(&evt_pending->list, &fparam->faults); - mutex_unlock(&fparam->lock); + list_add_tail(&evt_pending->list, &fault_param->faults); } - ret = iommu_handle_iopf(&evt->fault, dev); + ret = iommu_handle_iopf(&evt->fault, fault_param); if (ret && evt_pending) { - mutex_lock(&fparam->lock); list_del(&evt_pending->list); - mutex_unlock(&fparam->lock); kfree(evt_pending); } done_unlock: + mutex_unlock(&fault_param->lock); mutex_unlock(¶m->lock); + return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); @@ -232,18 +223,23 @@ int iommu_page_response(struct device *dev, struct iopf_fault *evt; struct iommu_fault_page_request *prm; struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *fault_param; const struct iommu_ops *ops = dev_iommu_ops(dev); bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; if (!ops->page_response) return -ENODEV; - if (!param || !param->fault_param) + mutex_lock(¶m->lock); + fault_param = param->fault_param; + if (!fault_param) { + mutex_unlock(¶m->lock); return -EINVAL; + } /* Only send response if there is a fault report pending */ - mutex_lock(¶m->fault_param->lock); - if (list_empty(¶m->fault_param->faults)) { + mutex_lock(&fault_param->lock); + if (list_empty(&fault_param->faults)) { dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); goto done_unlock; } @@ -251,7 +247,7 @@ int iommu_page_response(struct device *dev, * Check if we have a matching page request pending to respond, * otherwise return -EINVAL */ - list_for_each_entry(evt, ¶m->fault_param->faults, list) { + list_for_each_entry(evt, &fault_param->faults, list) { prm = &evt->fault.prm; if (prm->grpid != msg->grpid) continue; @@ -279,7 +275,8 @@ int iommu_page_response(struct device *dev, } done_unlock: - mutex_unlock(¶m->fault_param->lock); + mutex_unlock(&fault_param->lock); + mutex_unlock(¶m->lock); return ret; } EXPORT_SYMBOL_GPL(iommu_page_response); @@ -362,11 +359,13 @@ int iopf_queue_discard_partial(struct iopf_queue *queue) mutex_lock(&queue->lock); list_for_each_entry(iopf_param, &queue->devices, queue_list) { + mutex_lock(&iopf_param->lock); list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { list_del(&iopf->list); kfree(iopf); } + mutex_unlock(&iopf_param->lock); } mutex_unlock(&queue->lock); return 0; From a74c077b9021b36c785095c571336e5b204d3c2d Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:23 +0800 Subject: [PATCH 0445/1038] iommu: Use refcount for fault data access The per-device fault data structure stores information about faults occurring on a device. Its lifetime spans from IOPF enablement to disablement. Multiple paths, including IOPF reporting, handling, and responding, may access it concurrently. Previously, a mutex protected the fault data from use after free. But this is not performance friendly due to the critical nature of IOPF handling paths. Refine this with a refcount-based approach. The fault data pointer is obtained within an RCU read region with a refcount. The fault data pointer is returned for usage only when the pointer is valid and a refcount is successfully obtained. The fault data is freed with kfree_rcu(), ensuring data is only freed after all RCU critical regions complete. An iopf handling work starts once an iopf group is created. The handling work continues until iommu_page_response() is called to respond to the iopf and the iopf group is freed. During this time, the device fault parameter should always be available. Add a pointer to the device fault parameter in the iopf_group structure and hold the reference until the iopf_group is freed. Make iommu_page_response() static as it is only used in io-pgfault.c. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Link: https://lore.kernel.org/r/20240212012227.119381-13-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 127 +++++++++++++++++++++++-------------- drivers/iommu/iommu-sva.c | 2 +- include/linux/iommu.h | 17 +++-- 3 files changed, 88 insertions(+), 58 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 5aea8402be47..ce7058892b59 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -13,6 +13,32 @@ #include "iommu-priv.h" +/* + * Return the fault parameter of a device if it exists. Otherwise, return NULL. + * On a successful return, the caller takes a reference of this parameter and + * should put it after use by calling iopf_put_dev_fault_param(). + */ +static struct iommu_fault_param *iopf_get_dev_fault_param(struct device *dev) +{ + struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *fault_param; + + rcu_read_lock(); + fault_param = rcu_dereference(param->fault_param); + if (fault_param && !refcount_inc_not_zero(&fault_param->users)) + fault_param = NULL; + rcu_read_unlock(); + + return fault_param; +} + +/* Caller must hold a reference of the fault parameter. */ +static void iopf_put_dev_fault_param(struct iommu_fault_param *fault_param) +{ + if (refcount_dec_and_test(&fault_param->users)) + kfree_rcu(fault_param, rcu); +} + void iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -22,6 +48,8 @@ void iopf_free_group(struct iopf_group *group) kfree(iopf); } + /* Pair with iommu_report_device_fault(). */ + iopf_put_dev_fault_param(group->fault_param); kfree(group); } EXPORT_SYMBOL_GPL(iopf_free_group); @@ -135,7 +163,7 @@ static int iommu_handle_iopf(struct iommu_fault *fault, goto cleanup_partial; } - group->dev = dev; + group->fault_param = iopf_param; group->last_fault.fault = *fault; INIT_LIST_HEAD(&group->faults); group->domain = domain; @@ -178,64 +206,61 @@ cleanup_partial: */ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { + bool last_prq = evt->fault.type == IOMMU_FAULT_PAGE_REQ && + (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE); struct iommu_fault_param *fault_param; - struct iopf_fault *evt_pending = NULL; - struct dev_iommu *param = dev->iommu; - int ret = 0; + struct iopf_fault *evt_pending; + int ret; - mutex_lock(¶m->lock); - fault_param = param->fault_param; - if (!fault_param) { - mutex_unlock(¶m->lock); + fault_param = iopf_get_dev_fault_param(dev); + if (!fault_param) return -EINVAL; - } mutex_lock(&fault_param->lock); - if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { + if (last_prq) { evt_pending = kmemdup(evt, sizeof(struct iopf_fault), GFP_KERNEL); if (!evt_pending) { ret = -ENOMEM; - goto done_unlock; + goto err_unlock; } list_add_tail(&evt_pending->list, &fault_param->faults); } ret = iommu_handle_iopf(&evt->fault, fault_param); - if (ret && evt_pending) { + if (ret) + goto err_free; + + mutex_unlock(&fault_param->lock); + /* The reference count of fault_param is now held by iopf_group. */ + if (!last_prq) + iopf_put_dev_fault_param(fault_param); + + return 0; +err_free: + if (last_prq) { list_del(&evt_pending->list); kfree(evt_pending); } -done_unlock: +err_unlock: mutex_unlock(&fault_param->lock); - mutex_unlock(¶m->lock); + iopf_put_dev_fault_param(fault_param); return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); -int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) +static int iommu_page_response(struct iopf_group *group, + struct iommu_page_response *msg) { bool needs_pasid; int ret = -EINVAL; struct iopf_fault *evt; struct iommu_fault_page_request *prm; - struct dev_iommu *param = dev->iommu; - struct iommu_fault_param *fault_param; + struct device *dev = group->fault_param->dev; const struct iommu_ops *ops = dev_iommu_ops(dev); bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - - if (!ops->page_response) - return -ENODEV; - - mutex_lock(¶m->lock); - fault_param = param->fault_param; - if (!fault_param) { - mutex_unlock(¶m->lock); - return -EINVAL; - } + struct iommu_fault_param *fault_param = group->fault_param; /* Only send response if there is a fault report pending */ mutex_lock(&fault_param->lock); @@ -276,10 +301,9 @@ int iommu_page_response(struct device *dev, done_unlock: mutex_unlock(&fault_param->lock); - mutex_unlock(¶m->lock); + return ret; } -EXPORT_SYMBOL_GPL(iommu_page_response); /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed @@ -295,22 +319,20 @@ EXPORT_SYMBOL_GPL(iommu_page_response); */ int iopf_queue_flush_dev(struct device *dev) { - int ret = 0; struct iommu_fault_param *iopf_param; - struct dev_iommu *param = dev->iommu; - if (!param) + /* + * It's a driver bug to be here after iopf_queue_remove_device(). + * Therefore, it's safe to dereference the fault parameter without + * holding the lock. + */ + iopf_param = rcu_dereference_check(dev->iommu->fault_param, true); + if (WARN_ON(!iopf_param)) return -ENODEV; - mutex_lock(¶m->lock); - iopf_param = param->fault_param; - if (iopf_param) - flush_workqueue(iopf_param->queue->wq); - else - ret = -ENODEV; - mutex_unlock(¶m->lock); + flush_workqueue(iopf_param->queue->wq); - return ret; + return 0; } EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); @@ -335,7 +357,7 @@ int iopf_group_response(struct iopf_group *group, (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) resp.flags = IOMMU_PAGE_RESP_PASID_VALID; - return iommu_page_response(group->dev, &resp); + return iommu_page_response(group, &resp); } EXPORT_SYMBOL_GPL(iopf_group_response); @@ -384,10 +406,15 @@ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) int ret = 0; struct dev_iommu *param = dev->iommu; struct iommu_fault_param *fault_param; + const struct iommu_ops *ops = dev_iommu_ops(dev); + + if (!ops->page_response) + return -ENODEV; mutex_lock(&queue->lock); mutex_lock(¶m->lock); - if (param->fault_param) { + if (rcu_dereference_check(param->fault_param, + lockdep_is_held(¶m->lock))) { ret = -EBUSY; goto done_unlock; } @@ -402,10 +429,11 @@ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) INIT_LIST_HEAD(&fault_param->faults); INIT_LIST_HEAD(&fault_param->partial); fault_param->dev = dev; + refcount_set(&fault_param->users, 1); list_add(&fault_param->queue_list, &queue->devices); fault_param->queue = queue; - param->fault_param = fault_param; + rcu_assign_pointer(param->fault_param, fault_param); done_unlock: mutex_unlock(¶m->lock); @@ -429,10 +457,12 @@ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) int ret = 0; struct iopf_fault *iopf, *next; struct dev_iommu *param = dev->iommu; - struct iommu_fault_param *fault_param = param->fault_param; + struct iommu_fault_param *fault_param; mutex_lock(&queue->lock); mutex_lock(¶m->lock); + fault_param = rcu_dereference_check(param->fault_param, + lockdep_is_held(¶m->lock)); if (!fault_param) { ret = -ENODEV; goto unlock; @@ -454,8 +484,9 @@ int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) list_for_each_entry_safe(iopf, next, &fault_param->partial, list) kfree(iopf); - param->fault_param = NULL; - kfree(fault_param); + /* dec the ref owned by iopf_queue_add_device() */ + rcu_assign_pointer(param->fault_param, NULL); + iopf_put_dev_fault_param(fault_param); unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 9de878e40413..b51995b4fe90 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -251,7 +251,7 @@ static void iommu_sva_handle_iopf(struct work_struct *work) static int iommu_sva_iopf_handler(struct iopf_group *group) { - struct iommu_fault_param *fault_param = group->dev->iommu->fault_param; + struct iommu_fault_param *fault_param = group->fault_param; INIT_WORK(&group->work, iommu_sva_handle_iopf); if (!queue_work(fault_param->queue->wq, &group->work)) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index fc912aed7886..1e9161ae95da 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -41,6 +41,7 @@ struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; struct iommu_dma_cookie; +struct iommu_fault_param; #define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ #define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ @@ -129,8 +130,9 @@ struct iopf_group { struct iopf_fault last_fault; struct list_head faults; struct work_struct work; - struct device *dev; struct iommu_domain *domain; + /* The device's fault data parameter. */ + struct iommu_fault_param *fault_param; }; /** @@ -679,6 +681,8 @@ struct iommu_device { /** * struct iommu_fault_param - per-device IOMMU fault data * @lock: protect pending faults list + * @users: user counter to manage the lifetime of the data + * @rcu: rcu head for kfree_rcu() * @dev: the device that owns this param * @queue: IOPF queue * @queue_list: index into queue->devices @@ -688,6 +692,8 @@ struct iommu_device { */ struct iommu_fault_param { struct mutex lock; + refcount_t users; + struct rcu_head rcu; struct device *dev; struct iopf_queue *queue; @@ -715,7 +721,7 @@ struct iommu_fault_param { */ struct dev_iommu { struct mutex lock; - struct iommu_fault_param *fault_param; + struct iommu_fault_param __rcu *fault_param; struct iommu_fwspec *fwspec; struct iommu_device *iommu_dev; void *priv; @@ -1543,7 +1549,6 @@ void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); void iopf_free_group(struct iopf_group *group); int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); -int iommu_page_response(struct device *dev, struct iommu_page_response *msg); int iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status); #else @@ -1588,12 +1593,6 @@ iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) return -ENODEV; } -static inline int -iommu_page_response(struct device *dev, struct iommu_page_response *msg) -{ - return -ENODEV; -} - static inline int iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status) { From 0095bf83554f8e7a681961656608101bdf40e9ef Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:24 +0800 Subject: [PATCH 0446/1038] iommu: Improve iopf_queue_remove_device() Convert iopf_queue_remove_device() to return void instead of an error code, as the return value is never used. This removal helper is designed to be never-failed, so there's no need for error handling. Ack all outstanding page requests from the device with the response code of IOMMU_PAGE_RESP_INVALID, indicating device should not attempt any retry. Add comments to this helper explaining the steps involved in removing a device from the iopf queue and disabling its PRI. The individual drivers are expected to be adjusted accordingly. Here we just define the expected behaviors of the individual iommu driver from the core's perspective. Suggested-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Tested-by: Yan Zhao Link: https://lore.kernel.org/r/20240212012227.119381-14-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 7 +---- drivers/iommu/io-pgfault.c | 61 ++++++++++++++++++++++++------------- include/linux/iommu.h | 5 ++- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 29a12f289e2e..a81a2be9b870 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4455,12 +4455,7 @@ static int intel_iommu_disable_iopf(struct device *dev) */ pci_disable_pri(to_pci_dev(dev)); info->pri_enabled = 0; - - /* - * With PRI disabled and outstanding PRQs drained, removing device - * from iopf queue should never fail. - */ - WARN_ON(iopf_queue_remove_device(iommu->iopf_queue, dev)); + iopf_queue_remove_device(iommu->iopf_queue, dev); return 0; } diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index ce7058892b59..ece09552e5cf 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -448,50 +448,67 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); * @queue: IOPF queue * @dev: device to remove * - * Caller makes sure that no more faults are reported for this device. + * Removing a device from an iopf_queue. It's recommended to follow these + * steps when removing a device: * - * Return: 0 on success and <0 on error. + * - Disable new PRI reception: Turn off PRI generation in the IOMMU hardware + * and flush any hardware page request queues. This should be done before + * calling into this helper. + * - Acknowledge all outstanding PRQs to the device: Respond to all outstanding + * page requests with IOMMU_PAGE_RESP_INVALID, indicating the device should + * not retry. This helper function handles this. + * - Disable PRI on the device: After calling this helper, the caller could + * then disable PRI on the device. + * + * Calling iopf_queue_remove_device() essentially disassociates the device. + * The fault_param might still exist, but iommu_page_response() will do + * nothing. The device fault parameter reference count has been properly + * passed from iommu_report_device_fault() to the fault handling work, and + * will eventually be released after iommu_page_response(). */ -int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - int ret = 0; struct iopf_fault *iopf, *next; + struct iommu_page_response resp; struct dev_iommu *param = dev->iommu; struct iommu_fault_param *fault_param; + const struct iommu_ops *ops = dev_iommu_ops(dev); mutex_lock(&queue->lock); mutex_lock(¶m->lock); fault_param = rcu_dereference_check(param->fault_param, lockdep_is_held(¶m->lock)); - if (!fault_param) { - ret = -ENODEV; + + if (WARN_ON(!fault_param || fault_param->queue != queue)) goto unlock; - } - if (fault_param->queue != queue) { - ret = -EINVAL; - goto unlock; - } - - if (!list_empty(&fault_param->faults)) { - ret = -EBUSY; - goto unlock; - } - - list_del(&fault_param->queue_list); - - /* Just in case some faults are still stuck */ + mutex_lock(&fault_param->lock); list_for_each_entry_safe(iopf, next, &fault_param->partial, list) kfree(iopf); + list_for_each_entry_safe(iopf, next, &fault_param->faults, list) { + memset(&resp, 0, sizeof(struct iommu_page_response)); + resp.pasid = iopf->fault.prm.pasid; + resp.grpid = iopf->fault.prm.grpid; + resp.code = IOMMU_PAGE_RESP_INVALID; + + if (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID) + resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + + ops->page_response(dev, iopf, &resp); + list_del(&iopf->list); + kfree(iopf); + } + mutex_unlock(&fault_param->lock); + + list_del(&fault_param->queue_list); + /* dec the ref owned by iopf_queue_add_device() */ rcu_assign_pointer(param->fault_param, NULL); iopf_put_dev_fault_param(fault_param); unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); - - return ret; } EXPORT_SYMBOL_GPL(iopf_queue_remove_device); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1e9161ae95da..92dfd9b94577 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1542,7 +1542,7 @@ iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) #ifdef CONFIG_IOMMU_IOPF int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); -int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); +void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); int iopf_queue_flush_dev(struct device *dev); struct iopf_queue *iopf_queue_alloc(const char *name); void iopf_queue_free(struct iopf_queue *queue); @@ -1558,10 +1558,9 @@ iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) return -ENODEV; } -static inline int +static inline void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - return -ENODEV; } static inline int iopf_queue_flush_dev(struct device *dev) From 19911232713573a2ebea84a25bd4d71d024ed86b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:25 +0800 Subject: [PATCH 0447/1038] iommu: Track iopf group instead of last fault Previously, before a group of page faults was passed to the domain's iopf handler, the last page fault of the group was kept in the list of iommu_fault_param::faults. In the page fault response path, the group's last page fault was used to look up the list, and the page faults were responded to device only if there was a matched fault. The previous approach seems unnecessarily complex and not performance friendly. Put the page fault group itself to the outstanding fault list. It can be removed in the page fault response path or in the iopf_queue_remove_device() path. The pending list is protected by iommu_fault_param::lock. To allow checking for the group's presence in the list using list_empty(), the iopf group should be removed from the list with list_del_init(). IOMMU_PAGE_RESP_PASID_VALID is set in the code but not used anywhere. Remove it to make the code clean. IOMMU_PAGE_RESP_PASID_VALID is set in the response message indicating that the response message includes a valid PASID value. Actually, we should keep this hardware detail in the individual driver. When the page fault handling framework in IOMMU and IOMMUFD subsystems includes a valid PASID in the fault message, the response message should always contain the same PASID value. Individual drivers should be responsible for deciding whether to include the PASID in the messages they provide for the hardware. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Yan Zhao Link: https://lore.kernel.org/r/20240212012227.119381-15-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgfault.c | 242 +++++++++++++------------------------ include/linux/iommu.h | 6 +- 2 files changed, 86 insertions(+), 162 deletions(-) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index ece09552e5cf..05e49e2e6a52 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -78,12 +78,33 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, return domain; } +/* Non-last request of a group. Postpone until the last one. */ +static int report_partial_fault(struct iommu_fault_param *fault_param, + struct iommu_fault *fault) +{ + struct iopf_fault *iopf; + + iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); + if (!iopf) + return -ENOMEM; + + iopf->fault = *fault; + + mutex_lock(&fault_param->lock); + list_add(&iopf->list, &fault_param->partial); + mutex_unlock(&fault_param->lock); + + return 0; +} + /** - * iommu_handle_iopf - IO Page Fault handler - * @fault: fault event - * @iopf_param: the fault parameter of the device. + * iommu_report_device_fault() - Report fault event to device driver + * @dev: the device + * @evt: fault event data * - * Add a fault to the device workqueue, to be handled by mm. + * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ + * handler. When this function fails and the fault is recoverable, it is the + * caller's responsibility to complete the fault. * * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't @@ -118,34 +139,37 @@ static struct iommu_domain *get_domain_for_iopf(struct device *dev, * * Return: 0 on success and <0 on error. */ -static int iommu_handle_iopf(struct iommu_fault *fault, - struct iommu_fault_param *iopf_param) +int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - int ret; - struct iopf_group *group; - struct iommu_domain *domain; + struct iommu_fault *fault = &evt->fault; + struct iommu_fault_param *iopf_param; struct iopf_fault *iopf, *next; - struct device *dev = iopf_param->dev; - - lockdep_assert_held(&iopf_param->lock); + struct iommu_domain *domain; + struct iopf_group *group; + int ret; if (fault->type != IOMMU_FAULT_PAGE_REQ) - /* Not a recoverable page fault */ return -EOPNOTSUPP; + iopf_param = iopf_get_dev_fault_param(dev); + if (!iopf_param) + return -ENODEV; + if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); - if (!iopf) - return -ENOMEM; + ret = report_partial_fault(iopf_param, fault); + iopf_put_dev_fault_param(iopf_param); - iopf->fault = *fault; - - /* Non-last request of a group. Postpone until the last one */ - list_add(&iopf->list, &iopf_param->partial); - - return 0; + return ret; } + /* + * This is the last page fault of a group. Allocate an iopf group and + * pass it to domain's page fault handler. The group holds a reference + * count of the fault parameter. It will be released after response or + * error path of this function. If an error is returned, the caller + * will send a response to the hardware. We need to clean up before + * leaving, otherwise partial faults will be stuck. + */ domain = get_domain_for_iopf(dev, fault); if (!domain) { ret = -EINVAL; @@ -154,11 +178,6 @@ static int iommu_handle_iopf(struct iommu_fault *fault, group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) { - /* - * The caller will send a response to the hardware. But we do - * need to clean up before leaving, otherwise partial faults - * will be stuck. - */ ret = -ENOMEM; goto cleanup_partial; } @@ -166,145 +185,45 @@ static int iommu_handle_iopf(struct iommu_fault *fault, group->fault_param = iopf_param; group->last_fault.fault = *fault; INIT_LIST_HEAD(&group->faults); + INIT_LIST_HEAD(&group->pending_node); group->domain = domain; list_add(&group->last_fault.list, &group->faults); /* See if we have partial faults for this group */ + mutex_lock(&iopf_param->lock); list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { if (iopf->fault.prm.grpid == fault->prm.grpid) /* Insert *before* the last fault */ list_move(&iopf->list, &group->faults); } - + list_add(&group->pending_node, &iopf_param->faults); mutex_unlock(&iopf_param->lock); + ret = domain->iopf_handler(group); - mutex_lock(&iopf_param->lock); - if (ret) + if (ret) { + mutex_lock(&iopf_param->lock); + list_del_init(&group->pending_node); + mutex_unlock(&iopf_param->lock); iopf_free_group(group); + } return ret; + cleanup_partial: + mutex_lock(&iopf_param->lock); list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { if (iopf->fault.prm.grpid == fault->prm.grpid) { list_del(&iopf->list); kfree(iopf); } } - return ret; -} - -/** - * iommu_report_device_fault() - Report fault event to device driver - * @dev: the device - * @evt: fault event data - * - * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. - * - * Return 0 on success, or an error. - */ -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) -{ - bool last_prq = evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE); - struct iommu_fault_param *fault_param; - struct iopf_fault *evt_pending; - int ret; - - fault_param = iopf_get_dev_fault_param(dev); - if (!fault_param) - return -EINVAL; - - mutex_lock(&fault_param->lock); - if (last_prq) { - evt_pending = kmemdup(evt, sizeof(struct iopf_fault), - GFP_KERNEL); - if (!evt_pending) { - ret = -ENOMEM; - goto err_unlock; - } - list_add_tail(&evt_pending->list, &fault_param->faults); - } - - ret = iommu_handle_iopf(&evt->fault, fault_param); - if (ret) - goto err_free; - - mutex_unlock(&fault_param->lock); - /* The reference count of fault_param is now held by iopf_group. */ - if (!last_prq) - iopf_put_dev_fault_param(fault_param); - - return 0; -err_free: - if (last_prq) { - list_del(&evt_pending->list); - kfree(evt_pending); - } -err_unlock: - mutex_unlock(&fault_param->lock); - iopf_put_dev_fault_param(fault_param); + mutex_unlock(&iopf_param->lock); + iopf_put_dev_fault_param(iopf_param); return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); -static int iommu_page_response(struct iopf_group *group, - struct iommu_page_response *msg) -{ - bool needs_pasid; - int ret = -EINVAL; - struct iopf_fault *evt; - struct iommu_fault_page_request *prm; - struct device *dev = group->fault_param->dev; - const struct iommu_ops *ops = dev_iommu_ops(dev); - bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - struct iommu_fault_param *fault_param = group->fault_param; - - /* Only send response if there is a fault report pending */ - mutex_lock(&fault_param->lock); - if (list_empty(&fault_param->faults)) { - dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); - goto done_unlock; - } - /* - * Check if we have a matching page request pending to respond, - * otherwise return -EINVAL - */ - list_for_each_entry(evt, &fault_param->faults, list) { - prm = &evt->fault.prm; - if (prm->grpid != msg->grpid) - continue; - - /* - * If the PASID is required, the corresponding request is - * matched using the group ID, the PASID valid bit and the PASID - * value. Otherwise only the group ID matches request and - * response. - */ - needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; - if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) - continue; - - if (!needs_pasid && has_pasid) { - /* No big deal, just clear it. */ - msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; - msg->pasid = 0; - } - - ret = ops->page_response(dev, evt, msg); - list_del(&evt->list); - kfree(evt); - break; - } - -done_unlock: - mutex_unlock(&fault_param->lock); - - return ret; -} - /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed * @dev: the endpoint whose faults need to be flushed. @@ -346,18 +265,26 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); int iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status) { + struct iommu_fault_param *fault_param = group->fault_param; struct iopf_fault *iopf = &group->last_fault; + struct device *dev = group->fault_param->dev; + const struct iommu_ops *ops = dev_iommu_ops(dev); struct iommu_page_response resp = { .pasid = iopf->fault.prm.pasid, .grpid = iopf->fault.prm.grpid, .code = status, }; + int ret = -EINVAL; - if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && - (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + /* Only send response if there is a fault report pending */ + mutex_lock(&fault_param->lock); + if (!list_empty(&group->pending_node)) { + ret = ops->page_response(dev, &group->last_fault, &resp); + list_del_init(&group->pending_node); + } + mutex_unlock(&fault_param->lock); - return iommu_page_response(group, &resp); + return ret; } EXPORT_SYMBOL_GPL(iopf_group_response); @@ -468,8 +395,9 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); */ void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - struct iopf_fault *iopf, *next; - struct iommu_page_response resp; + struct iopf_fault *partial_iopf; + struct iopf_fault *next; + struct iopf_group *group, *temp; struct dev_iommu *param = dev->iommu; struct iommu_fault_param *fault_param; const struct iommu_ops *ops = dev_iommu_ops(dev); @@ -483,21 +411,19 @@ void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) goto unlock; mutex_lock(&fault_param->lock); - list_for_each_entry_safe(iopf, next, &fault_param->partial, list) - kfree(iopf); + list_for_each_entry_safe(partial_iopf, next, &fault_param->partial, list) + kfree(partial_iopf); - list_for_each_entry_safe(iopf, next, &fault_param->faults, list) { - memset(&resp, 0, sizeof(struct iommu_page_response)); - resp.pasid = iopf->fault.prm.pasid; - resp.grpid = iopf->fault.prm.grpid; - resp.code = IOMMU_PAGE_RESP_INVALID; - - if (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + list_for_each_entry_safe(group, temp, &fault_param->faults, pending_node) { + struct iopf_fault *iopf = &group->last_fault; + struct iommu_page_response resp = { + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = IOMMU_PAGE_RESP_INVALID + }; ops->page_response(dev, iopf, &resp); - list_del(&iopf->list); - kfree(iopf); + list_del_init(&group->pending_node); } mutex_unlock(&fault_param->lock); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 92dfd9b94577..f8ed1cc7212e 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -106,15 +106,11 @@ enum iommu_page_response_code { /** * struct iommu_page_response - Generic page response information - * @flags: encodes whether the corresponding fields are valid - * (IOMMU_FAULT_PAGE_RESPONSE_* values) * @pasid: Process Address Space ID * @grpid: Page Request Group Index * @code: response code from &enum iommu_page_response_code */ struct iommu_page_response { -#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - u32 flags; u32 pasid; u32 grpid; u32 code; @@ -129,6 +125,8 @@ struct iopf_fault { struct iopf_group { struct iopf_fault last_fault; struct list_head faults; + /* list node for iommu_fault_param::faults */ + struct list_head pending_node; struct work_struct work; struct iommu_domain *domain; /* The device's fault data parameter. */ From b554e396e51ce3d378a560666f85c6836a8323fd Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:26 +0800 Subject: [PATCH 0448/1038] iommu: Make iopf_group_response() return void The iopf_group_response() should return void, as nothing can do anything with the failure. This implies that ops->page_response() must also return void; this is consistent with what the drivers do. The failure paths, which are all integrity validations of the fault, should be WARN_ON'd, not return codes. If the iommu core fails to enqueue the fault, it should respond the fault directly by calling ops->page_response() instead of returning an error number and relying on the iommu drivers to do so. Consolidate the error fault handling code in the core. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240212012227.119381-16-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 50 +++----- drivers/iommu/intel/iommu.h | 4 +- drivers/iommu/intel/svm.c | 18 +-- drivers/iommu/io-pgfault.c | 132 +++++++++++--------- include/linux/iommu.h | 14 +-- 5 files changed, 98 insertions(+), 120 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4e93e845458c..42eb59cb99f4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -920,31 +920,29 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); } -static int arm_smmu_page_response(struct device *dev, - struct iopf_fault *unused, - struct iommu_page_response *resp) +static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused, + struct iommu_page_response *resp) { struct arm_smmu_cmdq_ent cmd = {0}; struct arm_smmu_master *master = dev_iommu_priv_get(dev); int sid = master->streams[0].id; - if (master->stall_enabled) { - cmd.opcode = CMDQ_OP_RESUME; - cmd.resume.sid = sid; - cmd.resume.stag = resp->grpid; - switch (resp->code) { - case IOMMU_PAGE_RESP_INVALID: - case IOMMU_PAGE_RESP_FAILURE: - cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; - break; - case IOMMU_PAGE_RESP_SUCCESS: - cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; - break; - default: - return -EINVAL; - } - } else { - return -ENODEV; + if (WARN_ON(!master->stall_enabled)) + return; + + cmd.opcode = CMDQ_OP_RESUME; + cmd.resume.sid = sid; + cmd.resume.stag = resp->grpid; + switch (resp->code) { + case IOMMU_PAGE_RESP_INVALID: + case IOMMU_PAGE_RESP_FAILURE: + cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; + break; + case IOMMU_PAGE_RESP_SUCCESS: + cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; + break; + default: + break; } arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); @@ -954,8 +952,6 @@ static int arm_smmu_page_response(struct device *dev, * terminated... at some point in the future. PRI_RESP is fire and * forget. */ - - return 0; } /* Context descriptor manipulation functions */ @@ -1516,16 +1512,6 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) } ret = iommu_report_device_fault(master->dev, &fault_evt); - if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) { - /* Nobody cared, abort the access */ - struct iommu_page_response resp = { - .pasid = flt->prm.pasid, - .grpid = flt->prm.grpid, - .code = IOMMU_PAGE_RESP_FAILURE, - }; - arm_smmu_page_response(master->dev, &fault_evt, &resp); - } - out_unlock: mutex_unlock(&smmu->streams_mutex); return ret; diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 696d95293a69..cf9a28c7fab8 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1079,8 +1079,8 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); -int intel_svm_page_response(struct device *dev, struct iopf_fault *evt, - struct iommu_page_response *msg); +void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); void intel_drain_pasid_prq(struct device *dev, u32 pasid); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e1cbcb9515f0..2f8716636dbb 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -740,9 +740,8 @@ prq_advance: return IRQ_RETVAL(handled); } -int intel_svm_page_response(struct device *dev, - struct iopf_fault *evt, - struct iommu_page_response *msg) +void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg) { struct device_domain_info *info = dev_iommu_priv_get(dev); struct intel_iommu *iommu = info->iommu; @@ -751,7 +750,6 @@ int intel_svm_page_response(struct device *dev, bool private_present; bool pasid_present; bool last_page; - int ret = 0; u16 sid; prm = &evt->fault.prm; @@ -760,16 +758,6 @@ int intel_svm_page_response(struct device *dev, private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - if (!pasid_present) { - ret = -EINVAL; - goto out; - } - - if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { - ret = -EINVAL; - goto out; - } - /* * Per VT-d spec. v3.0 ch7.7, system software must respond * with page group response if private data is present (PDP) @@ -798,8 +786,6 @@ int intel_svm_page_response(struct device *dev, qi_submit_sync(iommu, &desc, 1, 0); } -out: - return ret; } static int intel_svm_set_dev_pasid(struct iommu_domain *domain, diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 05e49e2e6a52..6a325bff8164 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -39,7 +39,7 @@ static void iopf_put_dev_fault_param(struct iommu_fault_param *fault_param) kfree_rcu(fault_param, rcu); } -void iopf_free_group(struct iopf_group *group) +static void __iopf_free_group(struct iopf_group *group) { struct iopf_fault *iopf, *next; @@ -50,6 +50,11 @@ void iopf_free_group(struct iopf_group *group) /* Pair with iommu_report_device_fault(). */ iopf_put_dev_fault_param(group->fault_param); +} + +void iopf_free_group(struct iopf_group *group) +{ + __iopf_free_group(group); kfree(group); } EXPORT_SYMBOL_GPL(iopf_free_group); @@ -97,14 +102,49 @@ static int report_partial_fault(struct iommu_fault_param *fault_param, return 0; } +static struct iopf_group *iopf_group_alloc(struct iommu_fault_param *iopf_param, + struct iopf_fault *evt, + struct iopf_group *abort_group) +{ + struct iopf_fault *iopf, *next; + struct iopf_group *group; + + group = kzalloc(sizeof(*group), GFP_KERNEL); + if (!group) { + /* + * We always need to construct the group as we need it to abort + * the request at the driver if it can't be handled. + */ + group = abort_group; + } + + group->fault_param = iopf_param; + group->last_fault.fault = evt->fault; + INIT_LIST_HEAD(&group->faults); + INIT_LIST_HEAD(&group->pending_node); + list_add(&group->last_fault.list, &group->faults); + + /* See if we have partial faults for this group */ + mutex_lock(&iopf_param->lock); + list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { + if (iopf->fault.prm.grpid == evt->fault.prm.grpid) + /* Insert *before* the last fault */ + list_move(&iopf->list, &group->faults); + } + list_add(&group->pending_node, &iopf_param->faults); + mutex_unlock(&iopf_param->lock); + + return group; +} + /** * iommu_report_device_fault() - Report fault event to device driver * @dev: the device * @evt: fault event data * * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. + * handler. If this function fails then ops->page_response() was called to + * complete evt if required. * * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't @@ -143,22 +183,18 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { struct iommu_fault *fault = &evt->fault; struct iommu_fault_param *iopf_param; - struct iopf_fault *iopf, *next; - struct iommu_domain *domain; + struct iopf_group abort_group = {}; struct iopf_group *group; int ret; - if (fault->type != IOMMU_FAULT_PAGE_REQ) - return -EOPNOTSUPP; - iopf_param = iopf_get_dev_fault_param(dev); - if (!iopf_param) + if (WARN_ON(!iopf_param)) return -ENODEV; if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { ret = report_partial_fault(iopf_param, fault); iopf_put_dev_fault_param(iopf_param); - + /* A request that is not the last does not need to be ack'd */ return ret; } @@ -170,56 +206,33 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) * will send a response to the hardware. We need to clean up before * leaving, otherwise partial faults will be stuck. */ - domain = get_domain_for_iopf(dev, fault); - if (!domain) { - ret = -EINVAL; - goto cleanup_partial; - } - - group = kzalloc(sizeof(*group), GFP_KERNEL); - if (!group) { + group = iopf_group_alloc(iopf_param, evt, &abort_group); + if (group == &abort_group) { ret = -ENOMEM; - goto cleanup_partial; + goto err_abort; } - group->fault_param = iopf_param; - group->last_fault.fault = *fault; - INIT_LIST_HEAD(&group->faults); - INIT_LIST_HEAD(&group->pending_node); - group->domain = domain; - list_add(&group->last_fault.list, &group->faults); - - /* See if we have partial faults for this group */ - mutex_lock(&iopf_param->lock); - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { - if (iopf->fault.prm.grpid == fault->prm.grpid) - /* Insert *before* the last fault */ - list_move(&iopf->list, &group->faults); + group->domain = get_domain_for_iopf(dev, fault); + if (!group->domain) { + ret = -EINVAL; + goto err_abort; } - list_add(&group->pending_node, &iopf_param->faults); - mutex_unlock(&iopf_param->lock); - ret = domain->iopf_handler(group); - if (ret) { - mutex_lock(&iopf_param->lock); - list_del_init(&group->pending_node); - mutex_unlock(&iopf_param->lock); + /* + * On success iopf_handler must call iopf_group_response() and + * iopf_free_group() + */ + ret = group->domain->iopf_handler(group); + if (ret) + goto err_abort; + return 0; + +err_abort: + iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE); + if (group == &abort_group) + __iopf_free_group(group); + else iopf_free_group(group); - } - - return ret; - -cleanup_partial: - mutex_lock(&iopf_param->lock); - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { - if (iopf->fault.prm.grpid == fault->prm.grpid) { - list_del(&iopf->list); - kfree(iopf); - } - } - mutex_unlock(&iopf_param->lock); - iopf_put_dev_fault_param(iopf_param); - return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); @@ -259,11 +272,9 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); * iopf_group_response - Respond a group of page faults * @group: the group of faults with the same group id * @status: the response code - * - * Return 0 on success and <0 on error. */ -int iopf_group_response(struct iopf_group *group, - enum iommu_page_response_code status) +void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) { struct iommu_fault_param *fault_param = group->fault_param; struct iopf_fault *iopf = &group->last_fault; @@ -274,17 +285,14 @@ int iopf_group_response(struct iopf_group *group, .grpid = iopf->fault.prm.grpid, .code = status, }; - int ret = -EINVAL; /* Only send response if there is a fault report pending */ mutex_lock(&fault_param->lock); if (!list_empty(&group->pending_node)) { - ret = ops->page_response(dev, &group->last_fault, &resp); + ops->page_response(dev, &group->last_fault, &resp); list_del_init(&group->pending_node); } mutex_unlock(&fault_param->lock); - - return ret; } EXPORT_SYMBOL_GPL(iopf_group_response); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index f8ed1cc7212e..f632775414a5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -574,9 +574,8 @@ struct iommu_ops { int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); - int (*page_response)(struct device *dev, - struct iopf_fault *evt, - struct iommu_page_response *msg); + void (*page_response)(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg); int (*def_domain_type)(struct device *dev); void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid); @@ -1547,8 +1546,8 @@ void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); void iopf_free_group(struct iopf_group *group); int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); -int iopf_group_response(struct iopf_group *group, - enum iommu_page_response_code status); +void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status); #else static inline int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) @@ -1590,10 +1589,9 @@ iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) return -ENODEV; } -static inline int iopf_group_response(struct iopf_group *group, - enum iommu_page_response_code status) +static inline void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) { - return -ENODEV; } #endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ From 3dfa64aecbafc288216b2790438d395add192c30 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Feb 2024 09:22:27 +0800 Subject: [PATCH 0449/1038] iommu: Make iommu_report_device_fault() return void As the iommu_report_device_fault() has been converted to auto-respond a page fault if it fails to enqueue it, there's no need to return a code in any case. Make it return void. Suggested-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240212012227.119381-17-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 ++-- drivers/iommu/intel/svm.c | 19 ++++++---------- drivers/iommu/io-pgfault.c | 25 +++++++-------------- include/linux/iommu.h | 5 ++--- 4 files changed, 19 insertions(+), 34 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 42eb59cb99f4..02580364acda 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1455,7 +1455,7 @@ arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) /* IRQ and event handlers */ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) { - int ret; + int ret = 0; u32 perm = 0; struct arm_smmu_master *master; bool ssid_valid = evt[0] & EVTQ_0_SSV; @@ -1511,7 +1511,7 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) goto out_unlock; } - ret = iommu_report_device_fault(master->dev, &fault_evt); + iommu_report_device_fault(master->dev, &fault_evt); out_unlock: mutex_unlock(&smmu->streams_mutex); return ret; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 2f8716636dbb..b644d57da841 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -561,14 +561,11 @@ static int prq_to_iommu_prot(struct page_req_dsc *req) return prot; } -static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, - struct page_req_dsc *desc) +static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, + struct page_req_dsc *desc) { struct iopf_fault event = { }; - if (!dev || !dev_is_pci(dev)) - return -ENODEV; - /* Fill in event data for device specific processing */ event.fault.type = IOMMU_FAULT_PAGE_REQ; event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; @@ -601,7 +598,7 @@ static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); } - return iommu_report_device_fault(dev, &event); + iommu_report_device_fault(dev, &event); } static void handle_bad_prq_event(struct intel_iommu *iommu, @@ -704,12 +701,10 @@ bad_req: if (!pdev) goto bad_req; - if (intel_svm_prq_report(iommu, &pdev->dev, req)) - handle_bad_prq_event(iommu, req, QI_RESP_INVALID); - else - trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, - req->priv_data[0], req->priv_data[1], - iommu->prq_seq_number++); + intel_svm_prq_report(iommu, &pdev->dev, req); + trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, + req->priv_data[0], req->priv_data[1], + iommu->prq_seq_number++); pci_dev_put(pdev); prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 6a325bff8164..06d78fcc79fd 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -176,26 +176,22 @@ static struct iopf_group *iopf_group_alloc(struct iommu_fault_param *iopf_param, * freed after the device has stopped generating page faults (or the iommu * hardware has been set to block the page faults) and the pending page faults * have been flushed. - * - * Return: 0 on success and <0 on error. */ -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) +void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { struct iommu_fault *fault = &evt->fault; struct iommu_fault_param *iopf_param; struct iopf_group abort_group = {}; struct iopf_group *group; - int ret; iopf_param = iopf_get_dev_fault_param(dev); if (WARN_ON(!iopf_param)) - return -ENODEV; + return; if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - ret = report_partial_fault(iopf_param, fault); + report_partial_fault(iopf_param, fault); iopf_put_dev_fault_param(iopf_param); /* A request that is not the last does not need to be ack'd */ - return ret; } /* @@ -207,25 +203,21 @@ int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) * leaving, otherwise partial faults will be stuck. */ group = iopf_group_alloc(iopf_param, evt, &abort_group); - if (group == &abort_group) { - ret = -ENOMEM; + if (group == &abort_group) goto err_abort; - } group->domain = get_domain_for_iopf(dev, fault); - if (!group->domain) { - ret = -EINVAL; + if (!group->domain) goto err_abort; - } /* * On success iopf_handler must call iopf_group_response() and * iopf_free_group() */ - ret = group->domain->iopf_handler(group); - if (ret) + if (group->domain->iopf_handler(group)) goto err_abort; - return 0; + + return; err_abort: iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE); @@ -233,7 +225,6 @@ err_abort: __iopf_free_group(group); else iopf_free_group(group); - return ret; } EXPORT_SYMBOL_GPL(iommu_report_device_fault); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index f632775414a5..7cc56cfe98dd 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1545,7 +1545,7 @@ struct iopf_queue *iopf_queue_alloc(const char *name); void iopf_queue_free(struct iopf_queue *queue); int iopf_queue_discard_partial(struct iopf_queue *queue); void iopf_free_group(struct iopf_group *group); -int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); +void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); void iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status); #else @@ -1583,10 +1583,9 @@ static inline void iopf_free_group(struct iopf_group *group) { } -static inline int +static inline void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - return -ENODEV; } static inline void iopf_group_response(struct iopf_group *group, From c763aefeeb3ebeb05efff3d204ffe8fa7872da8f Mon Sep 17 00:00:00 2001 From: Onkarnath Date: Tue, 13 Feb 2024 13:14:15 +0530 Subject: [PATCH 0450/1038] ACPI: use %pe for better readability of errors while printing As %pe is already introduced, it's better to use it in place of (%ld) for printing errors in logs. It would enhance readability of logs. Co-developed-by: Maninder Singh Signed-off-by: Maninder Singh Signed-off-by: Onkarnath Reviewed-by: Stanislaw Gruszka Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_processor.c | 2 +- drivers/acpi/acpi_watchdog.c | 2 +- drivers/acpi/pci_slot.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 4fe2ef54088c..2ddd36a21850 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -161,7 +161,7 @@ static void cpufreq_add_device(const char *name) pdev = platform_device_register_simple(name, PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pdev)) - pr_info("%s device creation failed: %ld\n", name, PTR_ERR(pdev)); + pr_info("%s device creation failed: %pe\n", name, pdev); } #ifdef CONFIG_X86 diff --git a/drivers/acpi/acpi_watchdog.c b/drivers/acpi/acpi_watchdog.c index 8e9e001da38f..14b24157799c 100644 --- a/drivers/acpi/acpi_watchdog.c +++ b/drivers/acpi/acpi_watchdog.c @@ -179,7 +179,7 @@ void __init acpi_watchdog_init(void) pdev = platform_device_register_simple("wdat_wdt", PLATFORM_DEVID_NONE, resources, nresources); if (IS_ERR(pdev)) - pr_err("Device creation failed: %ld\n", PTR_ERR(pdev)); + pr_err("Device creation failed: %pe\n", pdev); kfree(resources); diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c index d6cb2c27a23b..741bcc9d6d6a 100644 --- a/drivers/acpi/pci_slot.c +++ b/drivers/acpi/pci_slot.c @@ -111,7 +111,7 @@ register_slot(acpi_handle handle, u32 lvl, void *context, void **rv) snprintf(name, sizeof(name), "%llu", sun); pci_slot = pci_create_slot(pci_bus, device, name, NULL); if (IS_ERR(pci_slot)) { - pr_err("pci_create_slot returned %ld\n", PTR_ERR(pci_slot)); + pr_err("pci_create_slot returned %pe\n", pci_slot); kfree(slot); return AE_OK; } From e23ad54fef186aa66007895be1382c88f1ee2bf7 Mon Sep 17 00:00:00 2001 From: "Alexey I. Froloff" Date: Fri, 16 Feb 2024 12:30:09 +0000 Subject: [PATCH 0451/1038] ACPI: resource: Do IRQ override on Lunnen Ground laptops The Lunnen Ground 15 and 16 needs IRQ overriding for the keyboard to work. Adding an entries for these laptops to the override_table makes the internal keyboard functional. Signed-off-by: Alexey I. Froloff Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index c99b0cf5e22e..cc5b59fbc714 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -602,6 +602,20 @@ static const struct dmi_system_id irq1_edge_low_force_override[] = { DMI_MATCH(DMI_BOARD_NAME, "GM5RGEE0016COM"), }, }, + { + /* Lunnen Ground 15 / AMD Ryzen 5 5500U */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Lunnen"), + DMI_MATCH(DMI_BOARD_NAME, "LLL5DAW"), + }, + }, + { + /* Lunnen Ground 16 / AMD Ryzen 7 5800U */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Lunnen"), + DMI_MATCH(DMI_BOARD_NAME, "LL6FA"), + }, + }, { } }; From b50155cb0d609437236c88201206267835c6f965 Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 13 Feb 2024 23:39:47 +0100 Subject: [PATCH 0452/1038] powercap: dtpm_cpu: Fix error check against freq_qos_add_request() The caller of the function freq_qos_add_request() checks again a non zero value but freq_qos_add_request() can return '1' if the request already exists. Therefore, the setup function fails while the QoS request actually did not failed. Fix that by changing the check against a negative value like all the other callers of the function. Fixes: 0e8f68d7f0485 ("Add CPU energy model based support") Signed-off-by: Daniel Lezcano Signed-off-by: Rafael J. Wysocki --- drivers/powercap/dtpm_cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c index 9193c3b8edeb..ae7ee611978b 100644 --- a/drivers/powercap/dtpm_cpu.c +++ b/drivers/powercap/dtpm_cpu.c @@ -219,7 +219,7 @@ static int __dtpm_cpu_setup(int cpu, struct dtpm *parent) ret = freq_qos_add_request(&policy->constraints, &dtpm_cpu->qos_req, FREQ_QOS_MAX, pd->table[pd->nr_perf_states - 1].frequency); - if (ret) + if (ret < 0) goto out_dtpm_unregister; cpufreq_cpu_put(policy); From c2b28f6806d2a26a8d46c0f02d4852bf9904929d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Mon, 12 Feb 2024 09:50:05 -0500 Subject: [PATCH 0453/1038] firmware: coreboot: Generate modalias uevent for devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate a modalias uevent for devices in the coreboot bus to allow userspace to automatically load the corresponding modules. Acked-by: Brian Norris Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Brian Norris Signed-off-by: Nícolas F. R. A. Prado Link: https://lore.kernel.org/r/20240212-coreboot-mod-defconfig-v4-1-d14172676f6d@collabora.com Signed-off-by: Tzung-Bi Shih --- drivers/firmware/google/coreboot_table.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index fcdc387ae7aa..234cebf376d9 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -53,11 +53,20 @@ static void coreboot_bus_remove(struct device *dev) driver->remove(device); } +static int coreboot_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + struct coreboot_device *device = CB_DEV(dev); + u32 tag = device->entry.tag; + + return add_uevent_var(env, "MODALIAS=coreboot:t%08X", tag); +} + static const struct bus_type coreboot_bus_type = { .name = "coreboot", .match = coreboot_bus_match, .probe = coreboot_bus_probe, .remove = coreboot_bus_remove, + .uevent = coreboot_bus_uevent, }; static void coreboot_device_release(struct device *dev) From f1cebae1dbf85f9de65c13a2d9f5cc3be7e51dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Mon, 12 Feb 2024 09:50:06 -0500 Subject: [PATCH 0454/1038] firmware: coreboot: Generate aliases for coreboot modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate aliases for coreboot modules to allow automatic module probing. Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Brian Norris Signed-off-by: Nícolas F. R. A. Prado Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240212-coreboot-mod-defconfig-v4-2-d14172676f6d@collabora.com Signed-off-by: Tzung-Bi Shih --- include/linux/mod_devicetable.h | 10 ++++++++++ scripts/mod/devicetable-offsets.c | 3 +++ scripts/mod/file2alias.c | 10 ++++++++++ 3 files changed, 23 insertions(+) diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index f458469c5ce5..7a9a07ea451b 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -960,4 +960,14 @@ struct vchiq_device_id { char name[32]; }; +/** + * struct coreboot_device_id - Identifies a coreboot table entry + * @tag: tag ID + * @driver_data: driver specific data + */ +struct coreboot_device_id { + __u32 tag; + kernel_ulong_t driver_data; +}; + #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index e91a3c38143b..518200813d4e 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -274,5 +274,8 @@ int main(void) DEVID(vchiq_device_id); DEVID_FIELD(vchiq_device_id, name); + DEVID(coreboot_device_id); + DEVID_FIELD(coreboot_device_id, tag); + return 0; } diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 4829680a0a6d..5d1c61fa5a55 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1494,6 +1494,15 @@ static int do_vchiq_entry(const char *filename, void *symval, char *alias) return 1; } +/* Looks like: coreboot:tN */ +static int do_coreboot_entry(const char *filename, void *symval, char *alias) +{ + DEF_FIELD(symval, coreboot_device_id, tag); + sprintf(alias, "coreboot:t%08X", tag); + + return 1; +} + /* Does namelen bytes of name exactly match the symbol? */ static bool sym_is(const char *name, unsigned namelen, const char *symbol) { @@ -1575,6 +1584,7 @@ static const struct devtable devtable[] = { {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry}, {"cdx", SIZE_cdx_device_id, do_cdx_entry}, {"vchiq", SIZE_vchiq_device_id, do_vchiq_entry}, + {"coreboot", SIZE_coreboot_device_id, do_coreboot_entry}, }; /* Create MODULE_ALIAS() statements. From 8a0a62941a042612f7487f6c4ff291f9054ff214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Mon, 12 Feb 2024 09:50:07 -0500 Subject: [PATCH 0455/1038] firmware: coreboot: Replace tag with id table in driver struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the plain 'tag' field in struct coreboot_driver for the newly created coreboot_device_id struct, which also contains a tag field and has the benefit of allowing modalias generation, and update all coreboot drivers accordingly. While at it, also add the id table for each driver to the module device table to allow automatically loading the module. Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Brian Norris Signed-off-by: Nícolas F. R. A. Prado Link: https://lore.kernel.org/r/20240212-coreboot-mod-defconfig-v4-3-d14172676f6d@collabora.com Signed-off-by: Tzung-Bi Shih --- drivers/firmware/google/cbmem.c | 8 +++++++- drivers/firmware/google/coreboot_table.c | 11 ++++++++++- drivers/firmware/google/coreboot_table.h | 3 ++- drivers/firmware/google/framebuffer-coreboot.c | 8 +++++++- drivers/firmware/google/memconsole-coreboot.c | 8 +++++++- drivers/firmware/google/vpd.c | 8 +++++++- 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/drivers/firmware/google/cbmem.c b/drivers/firmware/google/cbmem.c index 88e587ba1e0d..c2bffdc352a3 100644 --- a/drivers/firmware/google/cbmem.c +++ b/drivers/firmware/google/cbmem.c @@ -114,6 +114,12 @@ static int cbmem_entry_probe(struct coreboot_device *dev) return 0; } +static const struct coreboot_device_id cbmem_ids[] = { + { .tag = LB_TAG_CBMEM_ENTRY }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, cbmem_ids); + static struct coreboot_driver cbmem_entry_driver = { .probe = cbmem_entry_probe, .drv = { @@ -121,7 +127,7 @@ static struct coreboot_driver cbmem_entry_driver = { .owner = THIS_MODULE, .dev_groups = dev_groups, }, - .tag = LB_TAG_CBMEM_ENTRY, + .id_table = cbmem_ids, }; module_coreboot_driver(cbmem_entry_driver); diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 234cebf376d9..d4b6e581a6c6 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -28,8 +28,17 @@ static int coreboot_bus_match(struct device *dev, struct device_driver *drv) { struct coreboot_device *device = CB_DEV(dev); struct coreboot_driver *driver = CB_DRV(drv); + const struct coreboot_device_id *id; - return device->entry.tag == driver->tag; + if (!driver->id_table) + return 0; + + for (id = driver->id_table; id->tag; id++) { + if (device->entry.tag == id->tag) + return 1; + } + + return 0; } static int coreboot_bus_probe(struct device *dev) diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index d814dca33a08..86427989c57f 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -13,6 +13,7 @@ #define __COREBOOT_TABLE_H #include +#include /* Coreboot table header structure */ struct coreboot_table_header { @@ -93,7 +94,7 @@ struct coreboot_driver { int (*probe)(struct coreboot_device *); void (*remove)(struct coreboot_device *); struct device_driver drv; - u32 tag; + const struct coreboot_device_id *id_table; }; /* Register a driver that uses the data from a coreboot table. */ diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index 5c84bbebfef8..07c458bf64ec 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -80,13 +80,19 @@ static void framebuffer_remove(struct coreboot_device *dev) platform_device_unregister(pdev); } +static const struct coreboot_device_id framebuffer_ids[] = { + { .tag = CB_TAG_FRAMEBUFFER }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, framebuffer_ids); + static struct coreboot_driver framebuffer_driver = { .probe = framebuffer_probe, .remove = framebuffer_remove, .drv = { .name = "framebuffer", }, - .tag = CB_TAG_FRAMEBUFFER, + .id_table = framebuffer_ids, }; module_coreboot_driver(framebuffer_driver); diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c index 74b5286518ee..24c97a70aa80 100644 --- a/drivers/firmware/google/memconsole-coreboot.c +++ b/drivers/firmware/google/memconsole-coreboot.c @@ -96,13 +96,19 @@ static void memconsole_remove(struct coreboot_device *dev) memconsole_exit(); } +static const struct coreboot_device_id memconsole_ids[] = { + { .tag = CB_TAG_CBMEM_CONSOLE }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, memconsole_ids); + static struct coreboot_driver memconsole_driver = { .probe = memconsole_probe, .remove = memconsole_remove, .drv = { .name = "memconsole", }, - .tag = CB_TAG_CBMEM_CONSOLE, + .id_table = memconsole_ids, }; module_coreboot_driver(memconsole_driver); diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c index ee6e08c0592b..8e4216714b29 100644 --- a/drivers/firmware/google/vpd.c +++ b/drivers/firmware/google/vpd.c @@ -306,13 +306,19 @@ static void vpd_remove(struct coreboot_device *dev) kobject_put(vpd_kobj); } +static const struct coreboot_device_id vpd_ids[] = { + { .tag = CB_TAG_VPD }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, vpd_ids); + static struct coreboot_driver vpd_driver = { .probe = vpd_probe, .remove = vpd_remove, .drv = { .name = "vpd", }, - .tag = CB_TAG_VPD, + .id_table = vpd_ids, }; module_coreboot_driver(vpd_driver); From 4a92857d6e8383eca6d661538bb25dc7004fd391 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 14:52:17 +0100 Subject: [PATCH 0456/1038] gpio: constify opaque pointer "data" in gpio_device_find() The opaque pointer "data" in each match function used by gpio_device_find() is a pointer to const, thus the same argument passed to gpio_device_find() can adjusted similarly. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 2 +- include/linux/gpio/driver.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 02be0ba1a402..e4dd13d81b4d 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1171,7 +1171,7 @@ EXPORT_SYMBOL_GPL(gpiochip_remove); * If the function returns non-NULL, the returned reference must be freed by * the caller using gpio_device_put(). */ -struct gpio_device *gpio_device_find(void *data, +struct gpio_device *gpio_device_find(const void *data, int (*match)(struct gpio_chip *gc, const void *data)) { diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 9d0023f83a57..9c1fbfaebaa8 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -628,7 +628,7 @@ int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, void *data, struct lock_class_key *lock_key, struct lock_class_key *request_key); -struct gpio_device *gpio_device_find(void *data, +struct gpio_device *gpio_device_find(const void *data, int (*match)(struct gpio_chip *gc, const void *data)); struct gpio_device *gpio_device_find_by_label(const char *label); From 24ba441d2b06614871b2787d5c2e16e7f1a15462 Mon Sep 17 00:00:00 2001 From: Xiaolei Wang Date: Sat, 17 Feb 2024 21:52:55 +0800 Subject: [PATCH 0457/1038] gpio: fix memory leak in gpiod_request_commit() Since commit 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU"), desc_set_label() already allocates memory for the label, so there is no need to allocate it again. If we do, we leak it. unreferenced object 0xffff0000c3e4d0c0 (size 32): comm "kworker/u16:4", pid 60, jiffies 4294894555 hex dump (first 32 bytes): 72 65 67 75 6c 61 74 6f 72 2d 63 61 6e 32 2d 73 regulator-can2-s 74 62 79 00 00 00 ff ff ff ff ff ff eb db ff ff tby............. backtrace (crc 2c3a0350): [<00000000e93c5cf4>] kmemleak_alloc+0x34/0x40 [<0000000097a2657f>] __kmalloc_node_track_caller+0x2c4/0x524 [<000000000dd1c057>] kstrdup+0x4c/0x98 [<00000000b513a96a>] kstrdup_const+0x34/0x40 [<000000008a7f0feb>] gpiod_request_commit+0xdc/0x358 [<00000000fc71ad64>] gpiod_request+0xd8/0x204 [<00000000fa24b091>] gpiod_find_and_request+0x170/0x780 [<0000000086ecf92d>] gpiod_get_index+0x70/0xe0 [<000000004aef97f9>] gpiod_get_optional+0x18/0x30 [<00000000312f1b25>] reg_fixed_voltage_probe+0x58c/0xad8 [<00000000e6f47635>] platform_probe+0xc4/0x198 [<00000000cf78fbdb>] really_probe+0x204/0x5a8 [<00000000e28d05ec>] __driver_probe_device+0x158/0x2c4 [<00000000e4fe452b>] driver_probe_device+0x60/0x18c [<00000000479fcf5d>] __device_attach_driver+0x168/0x208 [<000000007d389f38>] bus_for_each_drv+0x104/0x190 Fixes: 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU") Signed-off-by: Xiaolei Wang [Bartosz: tweaked the commit message] Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index e4dd13d81b4d..3c22920bd201 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2250,12 +2250,6 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) if (test_and_set_bit(FLAG_REQUESTED, &desc->flags)) return -EBUSY; - if (label) { - label = kstrdup_const(label, GFP_KERNEL); - if (!label) - return -ENOMEM; - } - /* NOTE: gpio_request() can be called in early boot, * before IRQs are enabled, for non-sleeping (SOC) GPIOs. */ From f2b70a264b44a6c3952fadbcc3e6711deed8458f Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Wed, 14 Feb 2024 14:00:09 +0100 Subject: [PATCH 0458/1038] ahci: rename board_ahci_nosntf Commit 7edbb6059274 ("ahci: clean up intel_pcs_quirk") added a new board type (board_ahci_pcs_quirk) which applies the Intel PCS quirk for legacy platforms. However, it also modified board_ahci_avn and board_ahci_nosntf to apply the same quirk. board_ahci_avn is defined under the label: /* board IDs for specific chipsets in alphabetical order */ This is a board for a specific chipset, so the naming is perfectly fine. (The name does not need to be suffixed with _pcs_quirk, since all controllers for this chipset require the quirk to be applied). board_ahci_nosntf is defined under the label: /* board IDs by feature in alphabetical order */ This is a board for a specific feature/quirk. However, it is used to apply two different quirks. Rename board_ahci_nosntf to more clearly highlight that this board ID applies two different quirks. Fixes: 7edbb6059274 ("ahci: clean up intel_pcs_quirk") Reviewed-by: Damien Le Moal Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 41b4c9777c85..a2efe916c6e2 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -53,7 +53,6 @@ enum board_ids { board_ahci_no_debounce_delay, board_ahci_nomsi, board_ahci_noncq, - board_ahci_nosntf, /* * board_ahci_pcs_quirk is for legacy Intel platforms. * Modern Intel platforms should use board_ahci instead. @@ -62,6 +61,7 @@ enum board_ids { * without testing that the platform actually works without the quirk.) */ board_ahci_pcs_quirk, + board_ahci_pcs_quirk_no_sntf, board_ahci_yes_fbs, /* board IDs for specific chipsets in alphabetical order */ @@ -165,16 +165,16 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_nosntf] = { - AHCI_HFLAGS (AHCI_HFLAG_NO_SNTF | - AHCI_HFLAG_INTEL_PCS_QUIRK), + [board_ahci_pcs_quirk] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_pcs_quirk] = { - AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), + [board_ahci_pcs_quirk_no_sntf] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK | + AHCI_HFLAG_NO_SNTF), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, @@ -271,7 +271,7 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x2683), board_ahci_pcs_quirk }, /* ESB2 */ { PCI_VDEVICE(INTEL, 0x27c6), board_ahci_pcs_quirk }, /* ICH7-M DH */ { PCI_VDEVICE(INTEL, 0x2821), board_ahci_pcs_quirk }, /* ICH8 */ - { PCI_VDEVICE(INTEL, 0x2822), board_ahci_nosntf }, /* ICH8/Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0x2822), board_ahci_pcs_quirk_no_sntf }, /* ICH8/Lewisburg RAID*/ { PCI_VDEVICE(INTEL, 0x2824), board_ahci_pcs_quirk }, /* ICH8 */ { PCI_VDEVICE(INTEL, 0x2829), board_ahci_pcs_quirk }, /* ICH8M */ { PCI_VDEVICE(INTEL, 0x282a), board_ahci_pcs_quirk }, /* ICH8M */ From bf6f1581d121d4723daf46ccd8d1a91d99e39b6d Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Wed, 14 Feb 2024 14:00:10 +0100 Subject: [PATCH 0459/1038] ahci: clean up ahci_broken_devslp quirk Most quirks are applied using a specific board type board_ahci_no* (e.g. board_ahci_nomsi, board_ahci_noncq), which then sets a flag representing the specific quirk. ahci_pci_tbl (which is the table of all supported PCI devices), then uses that board type for the PCI vendor and device IDs which need to be quirked. The ahci_broken_devslp quirk is not implemented in this standard way. Modify the ahci_broken_devslp quirk to be implemented like the other quirks. This way, we will not have the same PCI device and vendor ID scattered over ahci.c. It will simply be defined in a single location. Suggested-by: Dan Williams Reviewed-by: Damien Le Moal Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index a2efe916c6e2..d8e5b26524f6 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -61,6 +61,7 @@ enum board_ids { * without testing that the platform actually works without the quirk.) */ board_ahci_pcs_quirk, + board_ahci_pcs_quirk_no_devslp, board_ahci_pcs_quirk_no_sntf, board_ahci_yes_fbs, @@ -172,6 +173,14 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, + [board_ahci_pcs_quirk_no_devslp] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK | + AHCI_HFLAG_NO_DEVSLP), + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_ops, + }, [board_ahci_pcs_quirk_no_sntf] = { AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK | AHCI_HFLAG_NO_SNTF), @@ -420,7 +429,7 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x06d7), board_ahci_pcs_quirk }, /* Comet Lake-H RAID */ { PCI_VDEVICE(INTEL, 0xa386), board_ahci_pcs_quirk }, /* Comet Lake PCH-V RAID */ { PCI_VDEVICE(INTEL, 0x0f22), board_ahci_pcs_quirk }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x0f23), board_ahci_pcs_quirk }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x0f23), board_ahci_pcs_quirk_no_devslp }, /* Bay Trail AHCI */ { PCI_VDEVICE(INTEL, 0x22a3), board_ahci_pcs_quirk }, /* Cherry Tr. AHCI */ { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_pcs_quirk }, /* ApolloLake AHCI */ { PCI_VDEVICE(INTEL, 0x34d3), board_ahci_pcs_quirk }, /* Ice Lake LP AHCI */ @@ -1420,17 +1429,6 @@ static bool ahci_broken_online(struct pci_dev *pdev) return pdev->bus->number == (val >> 8) && pdev->devfn == (val & 0xff); } -static bool ahci_broken_devslp(struct pci_dev *pdev) -{ - /* device with broken DEVSLP but still showing SDS capability */ - static const struct pci_device_id ids[] = { - { PCI_VDEVICE(INTEL, 0x0f23)}, /* Valleyview SoC */ - {} - }; - - return pci_match_id(ids, pdev); -} - #ifdef CONFIG_ATA_ACPI static void ahci_gtf_filter_workaround(struct ata_host *host) { @@ -1823,10 +1821,6 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) &dev_attr_remapped_nvme.attr, NULL); - /* must set flag prior to save config in order to take effect */ - if (ahci_broken_devslp(pdev)) - hpriv->flags |= AHCI_HFLAG_NO_DEVSLP; - #ifdef CONFIG_ARM64 if (pdev->vendor == PCI_VENDOR_ID_HUAWEI && pdev->device == 0xa235 && From 8d6cfedec1c0c308e0dc8b599d58884b3119b172 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Wed, 14 Feb 2024 14:00:11 +0100 Subject: [PATCH 0460/1038] ahci: drop unused board_ahci_noncq Since commit 66a7cbc303f4 ("ahci: disable MSI instead of NCQ on Samsung pci-e SSDs on macbooks") there is not a single entry in ahci_pci_tbl which uses board_ahci_noncq. Since this is dead code, let's remove it. We cannot remove AHCI_HFLAG_NO_NCQ, as this flag is still used by other boards. Reviewed-by: Damien Le Moal Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index d8e5b26524f6..9bff90fb0591 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -52,7 +52,6 @@ enum board_ids { board_ahci_ign_iferr, board_ahci_no_debounce_delay, board_ahci_nomsi, - board_ahci_noncq, /* * board_ahci_pcs_quirk is for legacy Intel platforms. * Modern Intel platforms should use board_ahci instead. @@ -159,13 +158,6 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_noncq] = { - AHCI_HFLAGS (AHCI_HFLAG_NO_NCQ), - .flags = AHCI_FLAG_COMMON, - .pio_mask = ATA_PIO4, - .udma_mask = ATA_UDMA6, - .port_ops = &ahci_ops, - }, [board_ahci_pcs_quirk] = { AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), .flags = AHCI_FLAG_COMMON, From 873121511212c5da3e63e31d4a9a9aded8a2879a Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Wed, 14 Feb 2024 14:00:12 +0100 Subject: [PATCH 0461/1038] ahci: rename board_ahci_nomsi The naming format of the board_ahci_no* boards are: board_ahci_no_debounce_delay board_ahci_pcs_quirk_no_devslp board_ahci_pcs_quirk_no_sntf Rename board_ahci_nomsi to board_ahci_no_msi to match the other boards. Reviewed-by: Damien Le Moal Reviewed-by: Mika Westerberg Signed-off-by: Niklas Cassel --- drivers/ata/ahci.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 9bff90fb0591..006312f5d73c 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -51,7 +51,7 @@ enum board_ids { board_ahci_43bit_dma, board_ahci_ign_iferr, board_ahci_no_debounce_delay, - board_ahci_nomsi, + board_ahci_no_msi, /* * board_ahci_pcs_quirk is for legacy Intel platforms. * Modern Intel platforms should use board_ahci instead. @@ -151,7 +151,7 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_nomsi] = { + [board_ahci_no_msi] = { AHCI_HFLAGS (AHCI_HFLAG_NO_MSI), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, @@ -621,8 +621,8 @@ static const struct pci_device_id ahci_pci_tbl[] = { * Samsung SSDs found on some macbooks. NCQ times out if MSI is * enabled. https://bugzilla.kernel.org/show_bug.cgi?id=60731 */ - { PCI_VDEVICE(SAMSUNG, 0x1600), board_ahci_nomsi }, - { PCI_VDEVICE(SAMSUNG, 0xa800), board_ahci_nomsi }, + { PCI_VDEVICE(SAMSUNG, 0x1600), board_ahci_no_msi }, + { PCI_VDEVICE(SAMSUNG, 0xa800), board_ahci_no_msi }, /* Enmotus */ { PCI_DEVICE(0x1c44, 0x8000), board_ahci }, From aaa3cc29a78e0db72762999d7ad1224d1cf3d45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:19 +0100 Subject: [PATCH 0462/1038] pwm: dwc: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct dwc_pwm. Use the pwm_chip as driver data and return value of dwc_pwm_alloc() instead of the dwc_pwm to get access to the pwm_chip in dwc_pwm_probe() and dwc_pwm_suspend() without using dwc->chip. Thanks to Raag Jadav for providing a hunk of this patch that Uwe missed during creation of this patch. Link: https://lore.kernel.org/r/008ce5ab84b8e3baa3e81ab6d36dbb0e4be5c319.1707900770.git.u.kleine-koenig@pengutronix.de Link: https://lore.kernel.org/r/20240219033835.11369-2-raag.jadav@intel.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc-core.c | 16 +++++++++------- drivers/pwm/pwm-dwc.c | 20 ++++++++++++-------- drivers/pwm/pwm-dwc.h | 2 +- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/drivers/pwm/pwm-dwc-core.c b/drivers/pwm/pwm-dwc-core.c index ea63dd741f5c..ac7a0b2cb8b9 100644 --- a/drivers/pwm/pwm-dwc-core.c +++ b/drivers/pwm/pwm-dwc-core.c @@ -159,21 +159,23 @@ static const struct pwm_ops dwc_pwm_ops = { .get_state = dwc_pwm_get_state, }; -struct dwc_pwm *dwc_pwm_alloc(struct device *dev) +struct pwm_chip *dwc_pwm_alloc(struct device *dev) { + struct pwm_chip *chip; struct dwc_pwm *dwc; dwc = devm_kzalloc(dev, sizeof(*dwc), GFP_KERNEL); if (!dwc) - return NULL; + return ERR_PTR(-ENOMEM); + chip = &dwc->chip; dwc->clk_ns = 10; - dwc->chip.dev = dev; - dwc->chip.ops = &dwc_pwm_ops; - dwc->chip.npwm = DWC_TIMERS_TOTAL; + chip->dev = dev; + chip->ops = &dwc_pwm_ops; + chip->npwm = DWC_TIMERS_TOTAL; - dev_set_drvdata(dev, dwc); - return dwc; + dev_set_drvdata(dev, chip); + return chip; } EXPORT_SYMBOL_GPL(dwc_pwm_alloc); diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index 9a1b156ead26..c0e586688e57 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -28,12 +28,14 @@ static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) { struct device *dev = &pci->dev; + struct pwm_chip *chip; struct dwc_pwm *dwc; int ret; - dwc = dwc_pwm_alloc(dev); - if (!dwc) - return -ENOMEM; + chip = dwc_pwm_alloc(dev); + if (IS_ERR(chip)) + return PTR_ERR(chip); + dwc = to_dwc_pwm(chip); ret = pcim_enable_device(pci); if (ret) { @@ -55,7 +57,7 @@ static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) return -ENOMEM; } - ret = devm_pwmchip_add(dev, &dwc->chip); + ret = devm_pwmchip_add(dev, chip); if (ret) return ret; @@ -73,13 +75,14 @@ static void dwc_pwm_remove(struct pci_dev *pci) static int dwc_pwm_suspend(struct device *dev) { - struct dwc_pwm *dwc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct dwc_pwm *dwc = to_dwc_pwm(chip); int i; for (i = 0; i < DWC_TIMERS_TOTAL; i++) { - if (dwc->chip.pwms[i].state.enabled) { + if (chip->pwms[i].state.enabled) { dev_err(dev, "PWM %u in use by consumer (%s)\n", - i, dwc->chip.pwms[i].label); + i, chip->pwms[i].label); return -EBUSY; } dwc->ctx[i].cnt = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(i)); @@ -92,7 +95,8 @@ static int dwc_pwm_suspend(struct device *dev) static int dwc_pwm_resume(struct device *dev) { - struct dwc_pwm *dwc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct dwc_pwm *dwc = to_dwc_pwm(chip); int i; for (i = 0; i < DWC_TIMERS_TOTAL; i++) { diff --git a/drivers/pwm/pwm-dwc.h b/drivers/pwm/pwm-dwc.h index 64795247c54c..432a3b4c81fd 100644 --- a/drivers/pwm/pwm-dwc.h +++ b/drivers/pwm/pwm-dwc.h @@ -57,4 +57,4 @@ static inline void dwc_pwm_writel(struct dwc_pwm *dwc, u32 value, u32 offset) writel(value, dwc->base + offset); } -extern struct dwc_pwm *dwc_pwm_alloc(struct device *dev); +extern struct pwm_chip *dwc_pwm_alloc(struct device *dev); From 1647e5065b4d67bec47977e650ad8ba3812cd259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:20 +0100 Subject: [PATCH 0463/1038] pwm: dwc: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-dwc driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_dwc_pwm() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/64e64a9a8644600d51b234ad9bcae951b5748998.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc-core.c | 10 ++++------ drivers/pwm/pwm-dwc.h | 7 +++++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-dwc-core.c b/drivers/pwm/pwm-dwc-core.c index ac7a0b2cb8b9..de1d2ac3316c 100644 --- a/drivers/pwm/pwm-dwc-core.c +++ b/drivers/pwm/pwm-dwc-core.c @@ -164,15 +164,13 @@ struct pwm_chip *dwc_pwm_alloc(struct device *dev) struct pwm_chip *chip; struct dwc_pwm *dwc; - dwc = devm_kzalloc(dev, sizeof(*dwc), GFP_KERNEL); - if (!dwc) - return ERR_PTR(-ENOMEM); - chip = &dwc->chip; + chip = devm_pwmchip_alloc(dev, DWC_TIMERS_TOTAL, sizeof(*dwc)); + if (IS_ERR(chip)) + return chip; + dwc = to_dwc_pwm(chip); dwc->clk_ns = 10; - chip->dev = dev; chip->ops = &dwc_pwm_ops; - chip->npwm = DWC_TIMERS_TOTAL; dev_set_drvdata(dev, chip); return chip; diff --git a/drivers/pwm/pwm-dwc.h b/drivers/pwm/pwm-dwc.h index 432a3b4c81fd..5887371803fd 100644 --- a/drivers/pwm/pwm-dwc.h +++ b/drivers/pwm/pwm-dwc.h @@ -40,12 +40,15 @@ struct dwc_pwm_ctx { }; struct dwc_pwm { - struct pwm_chip chip; void __iomem *base; unsigned int clk_ns; struct dwc_pwm_ctx ctx[DWC_TIMERS_TOTAL]; }; -#define to_dwc_pwm(p) (container_of((p), struct dwc_pwm, chip)) + +static inline struct dwc_pwm *to_dwc_pwm(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static inline u32 dwc_pwm_readl(struct dwc_pwm *dwc, u32 offset) { From 4839f5a7c49a13dcfc3aaa3da87b26e9884488a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:21 +0100 Subject: [PATCH 0464/1038] pwm: dwc-core: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/5240ba82ac227e3723678041bd69b9c7a9e3db43.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc-core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-dwc-core.c b/drivers/pwm/pwm-dwc-core.c index de1d2ac3316c..043736972cb9 100644 --- a/drivers/pwm/pwm-dwc-core.c +++ b/drivers/pwm/pwm-dwc-core.c @@ -105,12 +105,12 @@ static int dwc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->enabled) { if (!pwm->state.enabled) - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); return __dwc_pwm_configure_timer(dwc, pwm, state); } else { if (pwm->state.enabled) { __dwc_pwm_set_enable(dwc, pwm->hwpwm, false); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } } @@ -124,7 +124,7 @@ static int dwc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, u64 duty, period; u32 ctrl, ld, ld2; - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); ctrl = dwc_pwm_readl(dwc, DWC_TIM_CTRL(pwm->hwpwm)); ld = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(pwm->hwpwm)); @@ -149,7 +149,7 @@ static int dwc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->period = period; state->duty_cycle = duty; - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } From ecb4ec5ab8b23b48eab24a81cdce2e4ec2bebc79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:22 +0100 Subject: [PATCH 0465/1038] pwm: ep93xx: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/3bf56f20bc7f749e5a353cdcfad11dd1ba573381.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-ep93xx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-ep93xx.c b/drivers/pwm/pwm-ep93xx.c index 51e072572a87..e4c3546a61c8 100644 --- a/drivers/pwm/pwm-ep93xx.c +++ b/drivers/pwm/pwm-ep93xx.c @@ -46,14 +46,14 @@ static inline struct ep93xx_pwm *to_ep93xx_pwm(struct pwm_chip *chip) static int ep93xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { - struct platform_device *pdev = to_platform_device(chip->dev); + struct platform_device *pdev = to_platform_device(pwmchip_parent(chip)); return ep93xx_pwm_acquire_gpio(pdev); } static void ep93xx_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { - struct platform_device *pdev = to_platform_device(chip->dev); + struct platform_device *pdev = to_platform_device(pwmchip_parent(chip)); ep93xx_pwm_release_gpio(pdev); } From 3f681ff7ba799829c918140a42833ca48878d3f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:23 +0100 Subject: [PATCH 0466/1038] pwm: ep93xx: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-ep93xx driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/9420eeca1eb18fada4a15c897f60696b7b22b532.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-ep93xx.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-ep93xx.c b/drivers/pwm/pwm-ep93xx.c index e4c3546a61c8..666f2954133c 100644 --- a/drivers/pwm/pwm-ep93xx.c +++ b/drivers/pwm/pwm-ep93xx.c @@ -36,12 +36,11 @@ struct ep93xx_pwm { void __iomem *base; struct clk *clk; - struct pwm_chip chip; }; static inline struct ep93xx_pwm *to_ep93xx_pwm(struct pwm_chip *chip) { - return container_of(chip, struct ep93xx_pwm, chip); + return pwmchip_get_drvdata(chip); } static int ep93xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -163,12 +162,14 @@ static const struct pwm_ops ep93xx_pwm_ops = { static int ep93xx_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct ep93xx_pwm *ep93xx_pwm; int ret; - ep93xx_pwm = devm_kzalloc(&pdev->dev, sizeof(*ep93xx_pwm), GFP_KERNEL); - if (!ep93xx_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*ep93xx_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + ep93xx_pwm = to_ep93xx_pwm(chip); ep93xx_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ep93xx_pwm->base)) @@ -178,11 +179,9 @@ static int ep93xx_pwm_probe(struct platform_device *pdev) if (IS_ERR(ep93xx_pwm->clk)) return PTR_ERR(ep93xx_pwm->clk); - ep93xx_pwm->chip.dev = &pdev->dev; - ep93xx_pwm->chip.ops = &ep93xx_pwm_ops; - ep93xx_pwm->chip.npwm = 1; + chip->ops = &ep93xx_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &ep93xx_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return ret; From e80c1aa6aedcb8dc434e013f124c527282079bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:24 +0100 Subject: [PATCH 0467/1038] pwm: fsl-ftm: Change prototype of a helper to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given fsl_pwm_chip. To just not have to do that, rework fsl_pwm_apply_config() to take a pwm_chip. Link: https://lore.kernel.org/r/5f0e88afe25d6a218032891a34f963980814b760.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-fsl-ftm.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-fsl-ftm.c b/drivers/pwm/pwm-fsl-ftm.c index d1b6d1aa4773..5792c6ca449f 100644 --- a/drivers/pwm/pwm-fsl-ftm.c +++ b/drivers/pwm/pwm-fsl-ftm.c @@ -221,10 +221,11 @@ static bool fsl_pwm_is_other_pwm_enabled(struct fsl_pwm_chip *fpc, return false; } -static int fsl_pwm_apply_config(struct fsl_pwm_chip *fpc, +static int fsl_pwm_apply_config(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *newstate) { + struct fsl_pwm_chip *fpc = to_fsl_chip(chip); unsigned int duty; u32 reg_polarity; @@ -232,7 +233,7 @@ static int fsl_pwm_apply_config(struct fsl_pwm_chip *fpc, bool do_write_period = false; if (!fsl_pwm_calculate_period(fpc, newstate->period, &periodcfg)) { - dev_err(fpc->chip.dev, "failed to calculate new period\n"); + dev_err(chip->dev, "failed to calculate new period\n"); return -EINVAL; } @@ -246,7 +247,7 @@ static int fsl_pwm_apply_config(struct fsl_pwm_chip *fpc, */ else if (!fsl_pwm_periodcfg_are_equal(&fpc->period, &periodcfg)) { if (fsl_pwm_is_other_pwm_enabled(fpc, pwm)) { - dev_err(fpc->chip.dev, + dev_err(chip->dev, "Cannot change period for PWM %u, disable other PWMs first\n", pwm->hwpwm); return -EBUSY; @@ -322,7 +323,7 @@ static int fsl_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, goto end_mutex; } - ret = fsl_pwm_apply_config(fpc, pwm, newstate); + ret = fsl_pwm_apply_config(chip, pwm, newstate); if (ret) goto end_mutex; From 74836319c24a12138c466f0be3da89adafc3779a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:25 +0100 Subject: [PATCH 0468/1038] pwm: fsl-ftm: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/131f976a8f876a88e2a940f5acf6aa1d81833b89.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-fsl-ftm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-fsl-ftm.c b/drivers/pwm/pwm-fsl-ftm.c index 5792c6ca449f..47f1f5ac39cf 100644 --- a/drivers/pwm/pwm-fsl-ftm.c +++ b/drivers/pwm/pwm-fsl-ftm.c @@ -233,7 +233,7 @@ static int fsl_pwm_apply_config(struct pwm_chip *chip, bool do_write_period = false; if (!fsl_pwm_calculate_period(fpc, newstate->period, &periodcfg)) { - dev_err(chip->dev, "failed to calculate new period\n"); + dev_err(pwmchip_parent(chip), "failed to calculate new period\n"); return -EINVAL; } @@ -247,7 +247,7 @@ static int fsl_pwm_apply_config(struct pwm_chip *chip, */ else if (!fsl_pwm_periodcfg_are_equal(&fpc->period, &periodcfg)) { if (fsl_pwm_is_other_pwm_enabled(fpc, pwm)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "Cannot change period for PWM %u, disable other PWMs first\n", pwm->hwpwm); return -EBUSY; From 097779f1577df28fe5d02193ea8192a4b55d766a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:26 +0100 Subject: [PATCH 0469/1038] pwm: fsl-ftm: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct fsl_pwm_chip. Use the pwm_chip as driver data instead of the fsl_pwm_chip to get access to the pwm_chip in fsl_pwm_suspend() and fsl_pwm_resume() without using fpc->chip. Link: https://lore.kernel.org/r/2cf27e2929505c05f84501d6fce96f23eba03fc3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-fsl-ftm.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/drivers/pwm/pwm-fsl-ftm.c b/drivers/pwm/pwm-fsl-ftm.c index 47f1f5ac39cf..b4f8dff60c50 100644 --- a/drivers/pwm/pwm-fsl-ftm.c +++ b/drivers/pwm/pwm-fsl-ftm.c @@ -393,6 +393,7 @@ static const struct regmap_config fsl_pwm_regmap_config = { static int fsl_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct fsl_pwm_chip *fpc; void __iomem *base; int ret; @@ -400,11 +401,12 @@ static int fsl_pwm_probe(struct platform_device *pdev) fpc = devm_kzalloc(&pdev->dev, sizeof(*fpc), GFP_KERNEL); if (!fpc) return -ENOMEM; + chip = &fpc->chip; mutex_init(&fpc->lock); fpc->soc = of_device_get_match_data(&pdev->dev); - fpc->chip.dev = &pdev->dev; + chip->dev = &pdev->dev; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) @@ -423,16 +425,16 @@ static int fsl_pwm_probe(struct platform_device *pdev) return PTR_ERR(fpc->clk[FSL_PWM_CLK_SYS]); } - fpc->clk[FSL_PWM_CLK_FIX] = devm_clk_get(fpc->chip.dev, "ftm_fix"); + fpc->clk[FSL_PWM_CLK_FIX] = devm_clk_get(&pdev->dev, "ftm_fix"); if (IS_ERR(fpc->clk[FSL_PWM_CLK_FIX])) return PTR_ERR(fpc->clk[FSL_PWM_CLK_FIX]); - fpc->clk[FSL_PWM_CLK_EXT] = devm_clk_get(fpc->chip.dev, "ftm_ext"); + fpc->clk[FSL_PWM_CLK_EXT] = devm_clk_get(&pdev->dev, "ftm_ext"); if (IS_ERR(fpc->clk[FSL_PWM_CLK_EXT])) return PTR_ERR(fpc->clk[FSL_PWM_CLK_EXT]); fpc->clk[FSL_PWM_CLK_CNTEN] = - devm_clk_get(fpc->chip.dev, "ftm_cnt_clk_en"); + devm_clk_get(&pdev->dev, "ftm_cnt_clk_en"); if (IS_ERR(fpc->clk[FSL_PWM_CLK_CNTEN])) return PTR_ERR(fpc->clk[FSL_PWM_CLK_CNTEN]); @@ -445,16 +447,16 @@ static int fsl_pwm_probe(struct platform_device *pdev) fpc->ipg_clk = fpc->clk[FSL_PWM_CLK_SYS]; - fpc->chip.ops = &fsl_pwm_ops; - fpc->chip.npwm = 8; + chip->ops = &fsl_pwm_ops; + chip->npwm = 8; - ret = devm_pwmchip_add(&pdev->dev, &fpc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); return ret; } - platform_set_drvdata(pdev, fpc); + platform_set_drvdata(pdev, chip); return fsl_pwm_init(fpc); } @@ -462,14 +464,15 @@ static int fsl_pwm_probe(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP static int fsl_pwm_suspend(struct device *dev) { - struct fsl_pwm_chip *fpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct fsl_pwm_chip *fpc = to_fsl_chip(chip); int i; regcache_cache_only(fpc->regmap, true); regcache_mark_dirty(fpc->regmap); - for (i = 0; i < fpc->chip.npwm; i++) { - struct pwm_device *pwm = &fpc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!test_bit(PWMF_REQUESTED, &pwm->flags)) continue; @@ -488,11 +491,12 @@ static int fsl_pwm_suspend(struct device *dev) static int fsl_pwm_resume(struct device *dev) { - struct fsl_pwm_chip *fpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct fsl_pwm_chip *fpc = to_fsl_chip(chip); int i; - for (i = 0; i < fpc->chip.npwm; i++) { - struct pwm_device *pwm = &fpc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!test_bit(PWMF_REQUESTED, &pwm->flags)) continue; From 2e8d68c02ed17263310e3c49f542c1d9df57ec74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:27 +0100 Subject: [PATCH 0470/1038] pwm: fsl-ftm: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-fsl-ftm driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/b3c75dea5b2944402b14944fdf71a5db0e26cbd7.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-fsl-ftm.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-fsl-ftm.c b/drivers/pwm/pwm-fsl-ftm.c index b4f8dff60c50..2510c10ca473 100644 --- a/drivers/pwm/pwm-fsl-ftm.c +++ b/drivers/pwm/pwm-fsl-ftm.c @@ -40,7 +40,6 @@ struct fsl_pwm_periodcfg { }; struct fsl_pwm_chip { - struct pwm_chip chip; struct mutex lock; struct regmap *regmap; @@ -55,7 +54,7 @@ struct fsl_pwm_chip { static inline struct fsl_pwm_chip *to_fsl_chip(struct pwm_chip *chip) { - return container_of(chip, struct fsl_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void ftm_clear_write_protection(struct fsl_pwm_chip *fpc) @@ -398,15 +397,14 @@ static int fsl_pwm_probe(struct platform_device *pdev) void __iomem *base; int ret; - fpc = devm_kzalloc(&pdev->dev, sizeof(*fpc), GFP_KERNEL); - if (!fpc) - return -ENOMEM; - chip = &fpc->chip; + chip = devm_pwmchip_alloc(&pdev->dev, 8, sizeof(*fpc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + fpc = to_fsl_chip(chip); mutex_init(&fpc->lock); fpc->soc = of_device_get_match_data(&pdev->dev); - chip->dev = &pdev->dev; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) @@ -446,9 +444,7 @@ static int fsl_pwm_probe(struct platform_device *pdev) if (IS_ERR(fpc->ipg_clk)) fpc->ipg_clk = fpc->clk[FSL_PWM_CLK_SYS]; - chip->ops = &fsl_pwm_ops; - chip->npwm = 8; ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { From c8cf591113a0db63f5043cace0ca7b1ef7ae1c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:28 +0100 Subject: [PATCH 0471/1038] pwm: hibvt: Consistently name driver data hi_pwm_chip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver uses two different names for variables of type hibvt_pwm_chip: $ git grep 'struct hibvt_pwm_chip \*' v6.8-rc1 drivers/pwm/pwm-hibvt.c v6.8-rc1:drivers/pwm/pwm-hibvt.c:static inline struct hibvt_pwm_chip *to_hibvt_pwm_chip(struct pwm_chip *chip) v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *pwm_chip; v6.8-rc1:drivers/pwm/pwm-hibvt.c: struct hibvt_pwm_chip *pwm_chip; Most functions use "hi_pwm_chip" as name. Make use of this in the remaining two functions (that used "pwm_chip" before, which isn't optimal as this is a type name, too). Link: https://lore.kernel.org/r/66ff5a9b00889ff0c576afc783424478fbbb9853.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-hibvt.c | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/pwm/pwm-hibvt.c b/drivers/pwm/pwm-hibvt.c index c435776e2f78..f1039be0f4e4 100644 --- a/drivers/pwm/pwm-hibvt.c +++ b/drivers/pwm/pwm-hibvt.c @@ -191,72 +191,72 @@ static int hibvt_pwm_probe(struct platform_device *pdev) { const struct hibvt_pwm_soc *soc = of_device_get_match_data(&pdev->dev); - struct hibvt_pwm_chip *pwm_chip; + struct hibvt_pwm_chip *hi_pwm_chip; int ret, i; - pwm_chip = devm_kzalloc(&pdev->dev, sizeof(*pwm_chip), GFP_KERNEL); - if (pwm_chip == NULL) + hi_pwm_chip = devm_kzalloc(&pdev->dev, sizeof(*hi_pwm_chip), GFP_KERNEL); + if (hi_pwm_chip == NULL) return -ENOMEM; - pwm_chip->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pwm_chip->clk)) { + hi_pwm_chip->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hi_pwm_chip->clk)) { dev_err(&pdev->dev, "getting clock failed with %ld\n", - PTR_ERR(pwm_chip->clk)); - return PTR_ERR(pwm_chip->clk); + PTR_ERR(hi_pwm_chip->clk)); + return PTR_ERR(hi_pwm_chip->clk); } - pwm_chip->chip.ops = &hibvt_pwm_ops; - pwm_chip->chip.dev = &pdev->dev; - pwm_chip->chip.npwm = soc->num_pwms; - pwm_chip->soc = soc; + hi_pwm_chip->chip.ops = &hibvt_pwm_ops; + hi_pwm_chip->chip.dev = &pdev->dev; + hi_pwm_chip->chip.npwm = soc->num_pwms; + hi_pwm_chip->soc = soc; - pwm_chip->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(pwm_chip->base)) - return PTR_ERR(pwm_chip->base); + hi_pwm_chip->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hi_pwm_chip->base)) + return PTR_ERR(hi_pwm_chip->base); - ret = clk_prepare_enable(pwm_chip->clk); + ret = clk_prepare_enable(hi_pwm_chip->clk); if (ret < 0) return ret; - pwm_chip->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(pwm_chip->rstc)) { - clk_disable_unprepare(pwm_chip->clk); - return PTR_ERR(pwm_chip->rstc); + hi_pwm_chip->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); + if (IS_ERR(hi_pwm_chip->rstc)) { + clk_disable_unprepare(hi_pwm_chip->clk); + return PTR_ERR(hi_pwm_chip->rstc); } - reset_control_assert(pwm_chip->rstc); + reset_control_assert(hi_pwm_chip->rstc); msleep(30); - reset_control_deassert(pwm_chip->rstc); + reset_control_deassert(hi_pwm_chip->rstc); - ret = pwmchip_add(&pwm_chip->chip); + ret = pwmchip_add(&hi_pwm_chip->chip); if (ret < 0) { - clk_disable_unprepare(pwm_chip->clk); + clk_disable_unprepare(hi_pwm_chip->clk); return ret; } - for (i = 0; i < pwm_chip->chip.npwm; i++) { - hibvt_pwm_set_bits(pwm_chip->base, PWM_CTRL_ADDR(i), + for (i = 0; i < hi_pwm_chip->chip.npwm; i++) { + hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(i), PWM_KEEP_MASK, (0x1 << PWM_KEEP_SHIFT)); } - platform_set_drvdata(pdev, pwm_chip); + platform_set_drvdata(pdev, hi_pwm_chip); return 0; } static void hibvt_pwm_remove(struct platform_device *pdev) { - struct hibvt_pwm_chip *pwm_chip; + struct hibvt_pwm_chip *hi_pwm_chip; - pwm_chip = platform_get_drvdata(pdev); + hi_pwm_chip = platform_get_drvdata(pdev); - pwmchip_remove(&pwm_chip->chip); + pwmchip_remove(&hi_pwm_chip->chip); - reset_control_assert(pwm_chip->rstc); + reset_control_assert(hi_pwm_chip->rstc); msleep(30); - reset_control_deassert(pwm_chip->rstc); + reset_control_deassert(hi_pwm_chip->rstc); - clk_disable_unprepare(pwm_chip->clk); + clk_disable_unprepare(hi_pwm_chip->clk); } static const struct of_device_id hibvt_pwm_of_match[] = { From 19248d867bde9a3adab6e5a59cf0988041c1300a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:29 +0100 Subject: [PATCH 0472/1038] pwm: hibvt: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-hibvt driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/7b12504b4128969bdc783bf2bff1ecc63b7c7bd4.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-hibvt.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/drivers/pwm/pwm-hibvt.c b/drivers/pwm/pwm-hibvt.c index f1039be0f4e4..2eb0b13d4e10 100644 --- a/drivers/pwm/pwm-hibvt.c +++ b/drivers/pwm/pwm-hibvt.c @@ -33,7 +33,6 @@ #define PWM_DUTY_MASK GENMASK(31, 0) struct hibvt_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct reset_control *rstc; @@ -65,7 +64,7 @@ static const struct hibvt_pwm_soc hi3559v100_soc_info = { static inline struct hibvt_pwm_chip *to_hibvt_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct hibvt_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void hibvt_pwm_set_bits(void __iomem *base, u32 offset, @@ -191,12 +190,14 @@ static int hibvt_pwm_probe(struct platform_device *pdev) { const struct hibvt_pwm_soc *soc = of_device_get_match_data(&pdev->dev); + struct pwm_chip *chip; struct hibvt_pwm_chip *hi_pwm_chip; int ret, i; - hi_pwm_chip = devm_kzalloc(&pdev->dev, sizeof(*hi_pwm_chip), GFP_KERNEL); - if (hi_pwm_chip == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, soc->num_pwms, sizeof(*hi_pwm_chip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + hi_pwm_chip = to_hibvt_pwm_chip(chip); hi_pwm_chip->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(hi_pwm_chip->clk)) { @@ -205,9 +206,7 @@ static int hibvt_pwm_probe(struct platform_device *pdev) return PTR_ERR(hi_pwm_chip->clk); } - hi_pwm_chip->chip.ops = &hibvt_pwm_ops; - hi_pwm_chip->chip.dev = &pdev->dev; - hi_pwm_chip->chip.npwm = soc->num_pwms; + chip->ops = &hibvt_pwm_ops; hi_pwm_chip->soc = soc; hi_pwm_chip->base = devm_platform_ioremap_resource(pdev, 0); @@ -228,29 +227,28 @@ static int hibvt_pwm_probe(struct platform_device *pdev) msleep(30); reset_control_deassert(hi_pwm_chip->rstc); - ret = pwmchip_add(&hi_pwm_chip->chip); + ret = pwmchip_add(chip); if (ret < 0) { clk_disable_unprepare(hi_pwm_chip->clk); return ret; } - for (i = 0; i < hi_pwm_chip->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(i), PWM_KEEP_MASK, (0x1 << PWM_KEEP_SHIFT)); } - platform_set_drvdata(pdev, hi_pwm_chip); + platform_set_drvdata(pdev, chip); return 0; } static void hibvt_pwm_remove(struct platform_device *pdev) { - struct hibvt_pwm_chip *hi_pwm_chip; + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); - hi_pwm_chip = platform_get_drvdata(pdev); - - pwmchip_remove(&hi_pwm_chip->chip); + pwmchip_remove(chip); reset_control_assert(hi_pwm_chip->rstc); msleep(30); From 9bd3e889d222dfb4254507ae12aaaedaf92419fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:30 +0100 Subject: [PATCH 0473/1038] pwm: img: Drop write-only variable from driver private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct img_pwm_chip::dev is only assigned to, but the member variable is never used. So drop it. Link: https://lore.kernel.org/r/7b7de753caa453c7fe591228c9a45d0baf8b5878.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-img.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index 5965ac35b32e..5f423445873d 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -59,7 +59,6 @@ struct img_pwm_soc_data { }; struct img_pwm_chip { - struct device *dev; struct pwm_chip chip; struct clk *pwm_clk; struct clk *sys_clk; @@ -265,8 +264,6 @@ static int img_pwm_probe(struct platform_device *pdev) if (!imgchip) return -ENOMEM; - imgchip->dev = &pdev->dev; - imgchip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imgchip->base)) return PTR_ERR(imgchip->base); From 2231f6fe8316701a3a10dc7bcaca63b03fd6877b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:31 +0100 Subject: [PATCH 0474/1038] pwm: img: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/bbea7e6e4c5412c6b0773faa9f165b6311cd53b6.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-img.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index 5f423445873d..a4ffe3d71d56 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -98,7 +98,7 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_ns < imgchip->min_period_ns || period_ns > imgchip->max_period_ns) { - dev_err(chip->dev, "configured period not in range\n"); + dev_err(pwmchip_parent(chip), "configured period not in range\n"); return -ERANGE; } @@ -119,14 +119,14 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, div = PWM_CTRL_CFG_SUB_DIV0_DIV1; timebase = DIV_ROUND_UP(mul, 512); } else { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to configure timebase steps/divider value\n"); return -EINVAL; } duty = DIV_ROUND_UP(timebase * duty_ns, period_ns); - ret = pm_runtime_resume_and_get(chip->dev); + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (ret < 0) return ret; @@ -140,8 +140,8 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, (timebase << PWM_CH_CFG_TMBASE_SHIFT); img_pwm_writel(imgchip, PWM_CH_CFG(pwm->hwpwm), val); - pm_runtime_mark_last_busy(chip->dev); - pm_runtime_put_autosuspend(chip->dev); + pm_runtime_mark_last_busy(pwmchip_parent(chip)); + pm_runtime_put_autosuspend(pwmchip_parent(chip)); return 0; } @@ -152,7 +152,7 @@ static int img_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int ret; - ret = pm_runtime_resume_and_get(chip->dev); + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (ret < 0) return ret; @@ -176,8 +176,8 @@ static void img_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) val &= ~BIT(pwm->hwpwm); img_pwm_writel(imgchip, PWM_CTRL_CFG, val); - pm_runtime_mark_last_busy(chip->dev); - pm_runtime_put_autosuspend(chip->dev); + pm_runtime_mark_last_busy(pwmchip_parent(chip)); + pm_runtime_put_autosuspend(pwmchip_parent(chip)); } static int img_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, From b097d28e3319d059173a3b44b4438dd033a23c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:32 +0100 Subject: [PATCH 0475/1038] pwm: img: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct img_pwm_chip. Use the pwm_chip as driver data instead of the img_pwm_chip to get access to the pwm_chip in img_pwm_remove() and the PM callbacks without using imgchip->chip. Link: https://lore.kernel.org/r/2ddf76d127f69eca7d9faa6475091e432e890ac9.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-img.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index a4ffe3d71d56..e99d5bc979c2 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -224,7 +224,8 @@ MODULE_DEVICE_TABLE(of, img_pwm_of_match); static int img_pwm_runtime_suspend(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); clk_disable_unprepare(imgchip->pwm_clk); clk_disable_unprepare(imgchip->sys_clk); @@ -234,7 +235,8 @@ static int img_pwm_runtime_suspend(struct device *dev) static int img_pwm_runtime_resume(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int ret; ret = clk_prepare_enable(imgchip->sys_clk); @@ -258,11 +260,13 @@ static int img_pwm_probe(struct platform_device *pdev) int ret; u64 val; unsigned long clk_rate; + struct pwm_chip *chip; struct img_pwm_chip *imgchip; imgchip = devm_kzalloc(&pdev->dev, sizeof(*imgchip), GFP_KERNEL); if (!imgchip) return -ENOMEM; + chip = &imgchip->chip; imgchip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imgchip->base)) @@ -287,7 +291,7 @@ static int img_pwm_probe(struct platform_device *pdev) return PTR_ERR(imgchip->pwm_clk); } - platform_set_drvdata(pdev, imgchip); + platform_set_drvdata(pdev, chip); pm_runtime_set_autosuspend_delay(&pdev->dev, IMG_PWM_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); @@ -314,11 +318,11 @@ static int img_pwm_probe(struct platform_device *pdev) do_div(val, clk_rate); imgchip->min_period_ns = val; - imgchip->chip.dev = &pdev->dev; - imgchip->chip.ops = &img_pwm_ops; - imgchip->chip.npwm = IMG_PWM_NPWM; + chip->dev = &pdev->dev; + chip->ops = &img_pwm_ops; + chip->npwm = IMG_PWM_NPWM; - ret = pwmchip_add(&imgchip->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add failed: %d\n", ret); goto err_suspend; @@ -337,19 +341,20 @@ err_pm_disable: static void img_pwm_remove(struct platform_device *pdev) { - struct img_pwm_chip *imgchip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) img_pwm_runtime_suspend(&pdev->dev); - pwmchip_remove(&imgchip->chip); + pwmchip_remove(chip); } #ifdef CONFIG_PM_SLEEP static int img_pwm_suspend(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int i, ret; if (pm_runtime_status_suspended(dev)) { @@ -358,7 +363,7 @@ static int img_pwm_suspend(struct device *dev) return ret; } - for (i = 0; i < imgchip->chip.npwm; i++) + for (i = 0; i < chip->npwm; i++) imgchip->suspend_ch_cfg[i] = img_pwm_readl(imgchip, PWM_CH_CFG(i)); @@ -371,7 +376,8 @@ static int img_pwm_suspend(struct device *dev) static int img_pwm_resume(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int ret; int i; @@ -379,13 +385,13 @@ static int img_pwm_resume(struct device *dev) if (ret) return ret; - for (i = 0; i < imgchip->chip.npwm; i++) + for (i = 0; i < chip->npwm; i++) img_pwm_writel(imgchip, PWM_CH_CFG(i), imgchip->suspend_ch_cfg[i]); img_pwm_writel(imgchip, PWM_CTRL_CFG, imgchip->suspend_ctrl_cfg); - for (i = 0; i < imgchip->chip.npwm; i++) + for (i = 0; i < chip->npwm; i++) if (imgchip->suspend_ctrl_cfg & BIT(i)) regmap_clear_bits(imgchip->periph_regs, PERIP_PWM_PDM_CONTROL, From 12ca0c331a5f4d2f412c02f7a5a498e58f8b225f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:33 +0100 Subject: [PATCH 0476/1038] pwm: img: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-img driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/d99e00af2ff09cd3587d9a7c51dc8e0e13f85208.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-img.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index e99d5bc979c2..d79a96679a26 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -59,7 +59,6 @@ struct img_pwm_soc_data { }; struct img_pwm_chip { - struct pwm_chip chip; struct clk *pwm_clk; struct clk *sys_clk; void __iomem *base; @@ -73,7 +72,7 @@ struct img_pwm_chip { static inline struct img_pwm_chip *to_img_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct img_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline void img_pwm_writel(struct img_pwm_chip *imgchip, @@ -263,10 +262,10 @@ static int img_pwm_probe(struct platform_device *pdev) struct pwm_chip *chip; struct img_pwm_chip *imgchip; - imgchip = devm_kzalloc(&pdev->dev, sizeof(*imgchip), GFP_KERNEL); - if (!imgchip) - return -ENOMEM; - chip = &imgchip->chip; + chip = devm_pwmchip_alloc(&pdev->dev, IMG_PWM_NPWM, sizeof(*imgchip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + imgchip = to_img_pwm_chip(chip); imgchip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imgchip->base)) @@ -318,9 +317,7 @@ static int img_pwm_probe(struct platform_device *pdev) do_div(val, clk_rate); imgchip->min_period_ns = val; - chip->dev = &pdev->dev; chip->ops = &img_pwm_ops; - chip->npwm = IMG_PWM_NPWM; ret = pwmchip_add(chip); if (ret < 0) { From 5911fc37c784e917276c3d6640c604e920eaf78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:34 +0100 Subject: [PATCH 0477/1038] pwm: imx1: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-imx1 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_pwm_imx1_chip() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/c6995e0d281255a7514edf9f25a561302dcaf65f.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-imx1.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-imx1.c b/drivers/pwm/pwm-imx1.c index d175d895f22a..1d2aae2d278f 100644 --- a/drivers/pwm/pwm-imx1.c +++ b/drivers/pwm/pwm-imx1.c @@ -28,10 +28,12 @@ struct pwm_imx1_chip { struct clk *clk_ipg; struct clk *clk_per; void __iomem *mmio_base; - struct pwm_chip chip; }; -#define to_pwm_imx1_chip(chip) container_of(chip, struct pwm_imx1_chip, chip) +static inline struct pwm_imx1_chip *to_pwm_imx1_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int pwm_imx1_clk_prepare_enable(struct pwm_chip *chip) { @@ -156,11 +158,13 @@ MODULE_DEVICE_TABLE(of, pwm_imx1_dt_ids); static int pwm_imx1_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_imx1_chip *imx; - imx = devm_kzalloc(&pdev->dev, sizeof(*imx), GFP_KERNEL); - if (!imx) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*imx)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + imx = to_pwm_imx1_chip(chip); imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); if (IS_ERR(imx->clk_ipg)) @@ -172,15 +176,13 @@ static int pwm_imx1_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(imx->clk_per), "failed to get peripheral clock\n"); - imx->chip.ops = &pwm_imx1_ops; - imx->chip.dev = &pdev->dev; - imx->chip.npwm = 1; + chip->ops = &pwm_imx1_ops; imx->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imx->mmio_base)) return PTR_ERR(imx->mmio_base); - return devm_pwmchip_add(&pdev->dev, &imx->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static struct platform_driver pwm_imx1_driver = { From 175f53a78c9d669e84ec53ff85cee9211c0c685c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:35 +0100 Subject: [PATCH 0478/1038] pwm: imx27: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/a8e3e3b3ea0684c9b289999c6f09eedbc2520f54.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-imx27.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index c825d3bcb329..f84c8be7e9ce 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -145,7 +145,7 @@ static int pwm_imx27_get_state(struct pwm_chip *chip, state->polarity = PWM_POLARITY_INVERSED; break; default: - dev_warn(chip->dev, "can't set polarity, output disconnected"); + dev_warn(pwmchip_parent(chip), "can't set polarity, output disconnected"); } prescaler = MX3_PWMCR_PRESCALER_GET(val); @@ -177,7 +177,7 @@ static int pwm_imx27_get_state(struct pwm_chip *chip, static void pwm_imx27_sw_reset(struct pwm_chip *chip) { struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int wait_count = 0; u32 cr; @@ -196,7 +196,7 @@ static void pwm_imx27_wait_fifo_slot(struct pwm_chip *chip, struct pwm_device *pwm) { struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); unsigned int period_ms; int fifoav; u32 sr; From dcef3929e0b5fb4a5b68b18ba6282f9fbb20d0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:36 +0100 Subject: [PATCH 0479/1038] pwm: imx27: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-imx27 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_pwm_imx27_chip() helper macro to a static inline to get some type safety. Reviewed-by: Philipp Zabel Link: https://lore.kernel.org/r/3da748c80e9bbe29e7c119190dbc4bd67d159f22.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-imx27.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index f84c8be7e9ce..e1412116ef65 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -83,7 +83,6 @@ struct pwm_imx27_chip { struct clk *clk_ipg; struct clk *clk_per; void __iomem *mmio_base; - struct pwm_chip chip; /* * The driver cannot read the current duty cycle from the hardware if @@ -93,7 +92,10 @@ struct pwm_imx27_chip { unsigned int duty_cycle; }; -#define to_pwm_imx27_chip(chip) container_of(chip, struct pwm_imx27_chip, chip) +static inline struct pwm_imx27_chip *to_pwm_imx27_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int pwm_imx27_clk_prepare_enable(struct pwm_imx27_chip *imx) { @@ -303,13 +305,15 @@ MODULE_DEVICE_TABLE(of, pwm_imx27_dt_ids); static int pwm_imx27_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_imx27_chip *imx; int ret; u32 pwmcr; - imx = devm_kzalloc(&pdev->dev, sizeof(*imx), GFP_KERNEL); - if (imx == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*imx)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + imx = to_pwm_imx27_chip(chip); imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); if (IS_ERR(imx->clk_ipg)) @@ -321,9 +325,7 @@ static int pwm_imx27_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(imx->clk_per), "failed to get peripheral clock\n"); - imx->chip.ops = &pwm_imx27_ops; - imx->chip.dev = &pdev->dev; - imx->chip.npwm = 1; + chip->ops = &pwm_imx27_ops; imx->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imx->mmio_base)) @@ -338,7 +340,7 @@ static int pwm_imx27_probe(struct platform_device *pdev) if (!(pwmcr & MX3_PWMCR_EN)) pwm_imx27_clk_disable_unprepare(imx); - return devm_pwmchip_add(&pdev->dev, &imx->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static struct platform_driver imx_pwm_driver = { From abf6569d6482ef32fda11d785a80816451a1c69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:37 +0100 Subject: [PATCH 0480/1038] pwm: imx-tpm: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-tmp driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/5de5d53295fa445d58a79f83421dd3406166c3c6.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-imx-tpm.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c index 9fc290e647e1..b04b974c5f32 100644 --- a/drivers/pwm/pwm-imx-tpm.c +++ b/drivers/pwm/pwm-imx-tpm.c @@ -57,7 +57,6 @@ #define PWM_IMX_TPM_MOD_MOD GENMASK(PWM_IMX_TPM_MOD_WIDTH - 1, 0) struct imx_tpm_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct mutex lock; @@ -75,7 +74,7 @@ struct imx_tpm_pwm_param { static inline struct imx_tpm_pwm_chip * to_imx_tpm_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct imx_tpm_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -336,35 +335,40 @@ static const struct pwm_ops imx_tpm_pwm_ops = { static int pwm_imx_tpm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct imx_tpm_pwm_chip *tpm; + void __iomem *base; int ret; + unsigned int npwm; u32 val; - tpm = devm_kzalloc(&pdev->dev, sizeof(*tpm), GFP_KERNEL); - if (!tpm) - return -ENOMEM; + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + /* get number of channels */ + val = readl(base + PWM_IMX_TPM_PARAM); + npwm = FIELD_GET(PWM_IMX_TPM_PARAM_CHAN, val); + + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*tpm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + tpm = to_imx_tpm_pwm_chip(chip); platform_set_drvdata(pdev, tpm); - tpm->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(tpm->base)) - return PTR_ERR(tpm->base); + tpm->base = base; tpm->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(tpm->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(tpm->clk), "failed to get PWM clock\n"); - tpm->chip.dev = &pdev->dev; - tpm->chip.ops = &imx_tpm_pwm_ops; - - /* get number of channels */ - val = readl(tpm->base + PWM_IMX_TPM_PARAM); - tpm->chip.npwm = FIELD_GET(PWM_IMX_TPM_PARAM_CHAN, val); + chip->ops = &imx_tpm_pwm_ops; mutex_init(&tpm->lock); - ret = devm_pwmchip_add(&pdev->dev, &tpm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); From a1d81abc03b3d8d110d99b5efeb64d5f323c1c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:38 +0100 Subject: [PATCH 0481/1038] pwm: intel-lgm: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-intel-lgm driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/3b2be81b046fe2213736ad37153131d5f4c859f5.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-intel-lgm.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-intel-lgm.c b/drivers/pwm/pwm-intel-lgm.c index 54ecae7f937e..f9cc7c17c8f0 100644 --- a/drivers/pwm/pwm-intel-lgm.c +++ b/drivers/pwm/pwm-intel-lgm.c @@ -42,14 +42,13 @@ #define LGM_PWM_PERIOD_2WIRE_NS (40 * NSEC_PER_MSEC) struct lgm_pwm_chip { - struct pwm_chip chip; struct regmap *regmap; u32 period; }; static inline struct lgm_pwm_chip *to_lgm_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct lgm_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int lgm_pwm_enable(struct pwm_chip *chip, bool enable) @@ -168,14 +167,16 @@ static int lgm_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct reset_control *rst; + struct pwm_chip *chip; struct lgm_pwm_chip *pc; void __iomem *io_base; struct clk *clk; int ret; - pc = devm_kzalloc(dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 1, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_lgm_pwm_chip(chip); io_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(io_base)) @@ -203,13 +204,11 @@ static int lgm_pwm_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "cannot deassert reset control\n"); - pc->chip.dev = dev; - pc->chip.ops = &lgm_pwm_ops; - pc->chip.npwm = 1; + chip->ops = &lgm_pwm_ops; lgm_pwm_init(pc); - ret = devm_pwmchip_add(dev, &pc->chip); + ret = devm_pwmchip_add(dev, chip); if (ret < 0) return dev_err_probe(dev, ret, "failed to add PWM chip\n"); From 8a18b9733239f794cb5e53c876760302a21f804d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:39 +0100 Subject: [PATCH 0482/1038] pwm: iqs620a: Create a wrapper for converting a pwm_chip to driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no semantic change. The new function just implements what was open-coded twice in the driver before. Link: https://lore.kernel.org/r/cc7ab547b4c4b79456b548aa72205bcf4476db8c.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-iqs620a.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c index 378ab036edfe..98008cae6d70 100644 --- a/drivers/pwm/pwm-iqs620a.c +++ b/drivers/pwm/pwm-iqs620a.c @@ -40,6 +40,11 @@ struct iqs620_pwm_private { unsigned int duty_scale; }; +static inline struct iqs620_pwm_private *iqs620_pwm_from_chip(struct pwm_chip *chip) +{ + return container_of(chip, struct iqs620_pwm_private, chip); +} + static int iqs620_pwm_init(struct iqs620_pwm_private *iqs620_pwm, unsigned int duty_scale) { @@ -73,7 +78,7 @@ static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->period < IQS620_PWM_PERIOD_NS) return -EINVAL; - iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); + iqs620_pwm = iqs620_pwm_from_chip(chip); /* * The duty cycle generated by the device is calculated as follows: @@ -109,7 +114,7 @@ static int iqs620_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, { struct iqs620_pwm_private *iqs620_pwm; - iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); + iqs620_pwm = iqs620_pwm_from_chip(chip); mutex_lock(&iqs620_pwm->lock); From 07b61160e45d3f6694708e2f9891845713043625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:40 +0100 Subject: [PATCH 0483/1038] pwm: iqs620a: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct iqs620_pwm_private. The only two functions that make use of struct iqs620_pwm_private::chip (apart from the probe function that is handled later when converting to pwmchip_alloc()) only use the chip's device pointer. Introduce such a pointer in driver data and use that instead. Link: https://lore.kernel.org/r/e6b1636c0d26d8bcb02a66b2963e2cc394014d76.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-iqs620a.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c index 98008cae6d70..faf52d3c2720 100644 --- a/drivers/pwm/pwm-iqs620a.c +++ b/drivers/pwm/pwm-iqs620a.c @@ -34,6 +34,7 @@ struct iqs620_pwm_private { struct iqs62x_core *iqs62x; + struct device *dev; struct pwm_chip chip; struct notifier_block notifier; struct mutex lock; @@ -160,7 +161,7 @@ static int iqs620_pwm_notifier(struct notifier_block *notifier, mutex_unlock(&iqs620_pwm->lock); if (ret) { - dev_err(iqs620_pwm->chip.dev, + dev_err(iqs620_pwm->dev, "Failed to re-initialize device: %d\n", ret); return NOTIFY_BAD; } @@ -181,7 +182,7 @@ static void iqs620_pwm_notifier_unregister(void *context) ret = blocking_notifier_chain_unregister(&iqs620_pwm->iqs62x->nh, &iqs620_pwm->notifier); if (ret) - dev_err(iqs620_pwm->chip.dev, + dev_err(iqs620_pwm->dev, "Failed to unregister notifier: %d\n", ret); } @@ -196,6 +197,7 @@ static int iqs620_pwm_probe(struct platform_device *pdev) if (!iqs620_pwm) return -ENOMEM; + iqs620_pwm->dev = &pdev->dev; iqs620_pwm->iqs62x = iqs62x; ret = regmap_read(iqs62x->regmap, IQS620_PWR_SETTINGS, &val); From 1b6691680bcc845276b955bc078e953c86c6ecd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:41 +0100 Subject: [PATCH 0484/1038] pwm: iqs620a: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-iqs620a driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/85cb0e73baadd76ede364abc4b57ee72000880e5.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-iqs620a.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c index faf52d3c2720..13e5e138c8e9 100644 --- a/drivers/pwm/pwm-iqs620a.c +++ b/drivers/pwm/pwm-iqs620a.c @@ -35,7 +35,6 @@ struct iqs620_pwm_private { struct iqs62x_core *iqs62x; struct device *dev; - struct pwm_chip chip; struct notifier_block notifier; struct mutex lock; unsigned int duty_scale; @@ -43,7 +42,7 @@ struct iqs620_pwm_private { static inline struct iqs620_pwm_private *iqs620_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct iqs620_pwm_private, chip); + return pwmchip_get_drvdata(chip); } static int iqs620_pwm_init(struct iqs620_pwm_private *iqs620_pwm, @@ -189,14 +188,16 @@ static void iqs620_pwm_notifier_unregister(void *context) static int iqs620_pwm_probe(struct platform_device *pdev) { struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct iqs620_pwm_private *iqs620_pwm; unsigned int val; int ret; - iqs620_pwm = devm_kzalloc(&pdev->dev, sizeof(*iqs620_pwm), GFP_KERNEL); - if (!iqs620_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*iqs620_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + iqs620_pwm = iqs620_pwm_from_chip(chip); iqs620_pwm->dev = &pdev->dev; iqs620_pwm->iqs62x = iqs62x; @@ -212,9 +213,7 @@ static int iqs620_pwm_probe(struct platform_device *pdev) iqs620_pwm->duty_scale = val + 1; } - iqs620_pwm->chip.dev = &pdev->dev; - iqs620_pwm->chip.ops = &iqs620_pwm_ops; - iqs620_pwm->chip.npwm = 1; + chip->ops = &iqs620_pwm_ops; mutex_init(&iqs620_pwm->lock); @@ -232,7 +231,7 @@ static int iqs620_pwm_probe(struct platform_device *pdev) if (ret) return ret; - ret = devm_pwmchip_add(&pdev->dev, &iqs620_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) dev_err(&pdev->dev, "Failed to add device: %d\n", ret); From cebf107de5c9417cfde9649161d9cc3b43ac1b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:42 +0100 Subject: [PATCH 0485/1038] pwm: jz4740: Change prototype of a helper to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given jz4740_pwm_chip. To just not have to do that, rework jz4740_pwm_can_use_chn() to take a pwm_chip. Link: https://lore.kernel.org/r/83b826f0bf58b93f26d3bc30b1a957f84eedf41a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-jz4740.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index 3933418e551b..de9e66065e48 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -35,13 +35,12 @@ static inline struct jz4740_pwm_chip *to_jz4740(struct pwm_chip *chip) return container_of(chip, struct jz4740_pwm_chip, chip); } -static bool jz4740_pwm_can_use_chn(struct jz4740_pwm_chip *jz, - unsigned int channel) +static bool jz4740_pwm_can_use_chn(struct pwm_chip *chip, unsigned int channel) { /* Enable all TCU channels for PWM use by default except channels 0/1 */ - u32 pwm_channels_mask = GENMASK(jz->chip.npwm - 1, 2); + u32 pwm_channels_mask = GENMASK(chip->npwm - 1, 2); - device_property_read_u32(jz->chip.dev->parent, + device_property_read_u32(chip->dev->parent, "ingenic,pwm-channels-mask", &pwm_channels_mask); @@ -55,7 +54,7 @@ static int jz4740_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) char name[16]; int err; - if (!jz4740_pwm_can_use_chn(jz, pwm->hwpwm)) + if (!jz4740_pwm_can_use_chn(chip, pwm->hwpwm)) return -EBUSY; snprintf(name, sizeof(name), "timer%u", pwm->hwpwm); From 4eeb33229c322a85e45cec084a9faffb961e4e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:43 +0100 Subject: [PATCH 0486/1038] pwm: jz4740: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/86b8a362f1e50c5b95c43e056d03ee5ca66d1641.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-jz4740.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index de9e66065e48..4d39f61b86ff 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -40,7 +40,7 @@ static bool jz4740_pwm_can_use_chn(struct pwm_chip *chip, unsigned int channel) /* Enable all TCU channels for PWM use by default except channels 0/1 */ u32 pwm_channels_mask = GENMASK(chip->npwm - 1, 2); - device_property_read_u32(chip->dev->parent, + device_property_read_u32(pwmchip_parent(chip)->parent, "ingenic,pwm-channels-mask", &pwm_channels_mask); @@ -59,9 +59,10 @@ static int jz4740_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) snprintf(name, sizeof(name), "timer%u", pwm->hwpwm); - clk = clk_get(chip->dev, name); + clk = clk_get(pwmchip_parent(chip), name); if (IS_ERR(clk)) { - dev_err(chip->dev, "error %pe: Failed to get clock\n", clk); + dev_err(pwmchip_parent(chip), + "error %pe: Failed to get clock\n", clk); return PTR_ERR(clk); } @@ -149,7 +150,7 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, */ rate = clk_round_rate(clk, tmp); if (rate < 0) { - dev_err(chip->dev, "Unable to round rate: %ld\n", rate); + dev_err(pwmchip_parent(chip), "Unable to round rate: %ld\n", rate); return rate; } @@ -170,7 +171,7 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, err = clk_set_rate(clk, rate); if (err) { - dev_err(chip->dev, "Unable to set rate: %d\n", err); + dev_err(pwmchip_parent(chip), "Unable to set rate: %d\n", err); return err; } From 5d0237a7b0859fbe5793e45f6c755aa8fca1c602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:44 +0100 Subject: [PATCH 0487/1038] pwm: jz4740: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-jz4740 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Acked-by: Paul Cercueil Link: https://lore.kernel.org/r/14a081c097b4e7c7f346ca6557ece8d16ad5749d.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-jz4740.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index 4d39f61b86ff..da4bf543d357 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -25,14 +25,13 @@ struct soc_info { }; struct jz4740_pwm_chip { - struct pwm_chip chip; struct regmap *map; struct clk *clk[]; }; static inline struct jz4740_pwm_chip *to_jz4740(struct pwm_chip *chip) { - return container_of(chip, struct jz4740_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static bool jz4740_pwm_can_use_chn(struct pwm_chip *chip, unsigned int channel) @@ -224,6 +223,7 @@ static const struct pwm_ops jz4740_pwm_ops = { static int jz4740_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct jz4740_pwm_chip *jz; const struct soc_info *info; @@ -231,10 +231,10 @@ static int jz4740_pwm_probe(struct platform_device *pdev) if (!info) return -EINVAL; - jz = devm_kzalloc(dev, struct_size(jz, clk, info->num_pwms), - GFP_KERNEL); - if (!jz) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, info->num_pwms, struct_size(jz, clk, info->num_pwms)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + jz = to_jz4740(chip); jz->map = device_node_to_regmap(dev->parent->of_node); if (IS_ERR(jz->map)) { @@ -242,11 +242,9 @@ static int jz4740_pwm_probe(struct platform_device *pdev) return PTR_ERR(jz->map); } - jz->chip.dev = dev; - jz->chip.ops = &jz4740_pwm_ops; - jz->chip.npwm = info->num_pwms; + chip->ops = &jz4740_pwm_ops; - return devm_pwmchip_add(dev, &jz->chip); + return devm_pwmchip_add(dev, chip); } static const struct soc_info jz4740_soc_info = { From 15527ec2527b50f653f734d329d9f78d6a0081db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:45 +0100 Subject: [PATCH 0488/1038] pwm: keembay: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-keembay driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/f65c27bf0a1f9ba4e5dbb115ff628860163ff8e0.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-keembay.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-keembay.c b/drivers/pwm/pwm-keembay.c index ac824ecc3f64..35b641f3f6ed 100644 --- a/drivers/pwm/pwm-keembay.c +++ b/drivers/pwm/pwm-keembay.c @@ -36,7 +36,6 @@ #define KMB_PWM_HIGHLOW_OFFSET(ch) (0x20 + 4 * (ch)) struct keembay_pwm { - struct pwm_chip chip; struct device *dev; struct clk *clk; void __iomem *base; @@ -44,7 +43,7 @@ struct keembay_pwm { static inline struct keembay_pwm *to_keembay_pwm_dev(struct pwm_chip *chip) { - return container_of(chip, struct keembay_pwm, chip); + return pwmchip_get_drvdata(chip); } static void keembay_clk_unprepare(void *data) @@ -185,12 +184,14 @@ static const struct pwm_ops keembay_pwm_ops = { static int keembay_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct keembay_pwm *priv; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, KMB_TOTAL_PWM_CHANNELS, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_keembay_pwm_dev(chip); priv->clk = devm_clk_get(dev, NULL); if (IS_ERR(priv->clk)) @@ -204,11 +205,9 @@ static int keembay_pwm_probe(struct platform_device *pdev) if (ret) return ret; - priv->chip.dev = dev; - priv->chip.ops = &keembay_pwm_ops; - priv->chip.npwm = KMB_TOTAL_PWM_CHANNELS; + chip->ops = &keembay_pwm_ops; - ret = devm_pwmchip_add(dev, &priv->chip); + ret = devm_pwmchip_add(dev, chip); if (ret) return dev_err_probe(dev, ret, "Failed to add PWM chip\n"); From e6377139899e2c7cee53a22523eee691d485e998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:46 +0100 Subject: [PATCH 0489/1038] pwm: lp3943: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-lp3943 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/44a2d303bb511e49c6db8aaedd17986c3aedde44.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lp3943.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-lp3943.c b/drivers/pwm/pwm-lp3943.c index 32350a357278..61189cea1046 100644 --- a/drivers/pwm/pwm-lp3943.c +++ b/drivers/pwm/pwm-lp3943.c @@ -20,7 +20,6 @@ #define LP3943_MAX_PERIOD 1600000 struct lp3943_pwm { - struct pwm_chip chip; struct lp3943 *lp3943; struct lp3943_platform_data *pdata; struct lp3943_pwm_map pwm_map[LP3943_NUM_PWMS]; @@ -28,7 +27,7 @@ struct lp3943_pwm { static inline struct lp3943_pwm *to_lp3943_pwm(struct pwm_chip *chip) { - return container_of(chip, struct lp3943_pwm, chip); + return pwmchip_get_drvdata(chip); } static struct lp3943_pwm_map * @@ -273,12 +272,14 @@ static int lp3943_pwm_parse_dt(struct device *dev, static int lp3943_pwm_probe(struct platform_device *pdev) { struct lp3943 *lp3943 = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct lp3943_pwm *lp3943_pwm; int ret; - lp3943_pwm = devm_kzalloc(&pdev->dev, sizeof(*lp3943_pwm), GFP_KERNEL); - if (!lp3943_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, LP3943_NUM_PWMS, sizeof(*lp3943_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + lp3943_pwm = to_lp3943_pwm(chip); lp3943_pwm->pdata = lp3943->pdata; if (!lp3943_pwm->pdata) { @@ -292,11 +293,9 @@ static int lp3943_pwm_probe(struct platform_device *pdev) } lp3943_pwm->lp3943 = lp3943; - lp3943_pwm->chip.dev = &pdev->dev; - lp3943_pwm->chip.ops = &lp3943_pwm_ops; - lp3943_pwm->chip.npwm = LP3943_NUM_PWMS; + chip->ops = &lp3943_pwm_ops; - return devm_pwmchip_add(&pdev->dev, &lp3943_pwm->chip); + return devm_pwmchip_add(&pdev->dev, chip); } #ifdef CONFIG_OF From 8d1cdd2f2d0dca91814fb39fa6415ebb60c69959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:47 +0100 Subject: [PATCH 0490/1038] pwm: lpc18xx-sct: Drop hardly used member from driver private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device pointer is only used once in lpc18xx_pwm_request(). There the pwm_chip is available, too, which has a pointer to the same structure. Replace the usage accordingly and drop the then write-only member variable. Link: https://lore.kernel.org/r/427a719a24dd7ff5ebbac3e260a1773c094c7489.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpc18xx-sct.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c index fe891fa71a1d..754ea2c3f49b 100644 --- a/drivers/pwm/pwm-lpc18xx-sct.c +++ b/drivers/pwm/pwm-lpc18xx-sct.c @@ -92,7 +92,6 @@ struct lpc18xx_pwm_data { }; struct lpc18xx_pwm_chip { - struct device *dev; struct pwm_chip chip; void __iomem *base; struct clk *pwm_clk; @@ -289,7 +288,7 @@ static int lpc18xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) LPC18XX_PWM_EVENT_MAX); if (event >= LPC18XX_PWM_EVENT_MAX) { - dev_err(lpc18xx_pwm->dev, + dev_err(chip->dev, "maximum number of simultaneous channels reached\n"); return -EBUSY; } @@ -358,8 +357,6 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) if (!lpc18xx_pwm) return -ENOMEM; - lpc18xx_pwm->dev = &pdev->dev; - lpc18xx_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpc18xx_pwm->base)) return PTR_ERR(lpc18xx_pwm->base); From c60b92133fc7cefa501f4b077b7eaf3663b99910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:48 +0100 Subject: [PATCH 0491/1038] pwm: lpc18xx-sct: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/e4fb330c8dbf8fecd365afe3f8b33f8c7952eb38.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpc18xx-sct.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c index 754ea2c3f49b..46eeca27d9ab 100644 --- a/drivers/pwm/pwm-lpc18xx-sct.c +++ b/drivers/pwm/pwm-lpc18xx-sct.c @@ -197,7 +197,7 @@ static int lpc18xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_ns < lpc18xx_pwm->min_period_ns || period_ns > lpc18xx_pwm->max_period_ns) { - dev_err(chip->dev, "period %d not in range\n", period_ns); + dev_err(pwmchip_parent(chip), "period %d not in range\n", period_ns); return -ERANGE; } @@ -213,7 +213,7 @@ static int lpc18xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, */ if (requested_events > 2 && lpc18xx_pwm->period_ns != period_ns && lpc18xx_pwm->period_ns) { - dev_err(chip->dev, "conflicting period requested for PWM %u\n", + dev_err(pwmchip_parent(chip), "conflicting period requested for PWM %u\n", pwm->hwpwm); mutex_unlock(&lpc18xx_pwm->period_lock); return -EBUSY; @@ -288,7 +288,7 @@ static int lpc18xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) LPC18XX_PWM_EVENT_MAX); if (event >= LPC18XX_PWM_EVENT_MAX) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "maximum number of simultaneous channels reached\n"); return -EBUSY; } From 6a79dc8342d40d6039279fba995bd2017656a445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:49 +0100 Subject: [PATCH 0492/1038] pwm: lpc18xx-sct: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct lpc18xx_pwm_chip. Use the pwm_chip as driver data instead of the lpc18xx_pwm_chip to get access to the pwm_chip in lpc18xx_pwm_remove() without using lpc18xx_pwm->chip. Link: https://lore.kernel.org/r/b7bf8c421ff754ec3e985ec6235f4ddd6e5e27a4.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpc18xx-sct.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c index 46eeca27d9ab..9b1e8141aec7 100644 --- a/drivers/pwm/pwm-lpc18xx-sct.c +++ b/drivers/pwm/pwm-lpc18xx-sct.c @@ -348,6 +348,7 @@ MODULE_DEVICE_TABLE(of, lpc18xx_pwm_of_match); static int lpc18xx_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct lpc18xx_pwm_chip *lpc18xx_pwm; int ret; u64 val; @@ -356,6 +357,7 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) GFP_KERNEL); if (!lpc18xx_pwm) return -ENOMEM; + chip = &lpc18xx_pwm->chip; lpc18xx_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpc18xx_pwm->base)) @@ -386,9 +388,9 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) lpc18xx_pwm->min_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, lpc18xx_pwm->clk_rate); - lpc18xx_pwm->chip.dev = &pdev->dev; - lpc18xx_pwm->chip.ops = &lpc18xx_pwm_ops; - lpc18xx_pwm->chip.npwm = LPC18XX_NUM_PWMS; + chip->dev = &pdev->dev; + chip->ops = &lpc18xx_pwm_ops; + chip->npwm = LPC18XX_NUM_PWMS; /* SCT counter must be in unify (32 bit) mode */ lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CONFIG, @@ -420,21 +422,22 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) val |= LPC18XX_PWM_PRE(0); lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CTRL, val); - ret = pwmchip_add(&lpc18xx_pwm->chip); + ret = pwmchip_add(chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add failed\n"); - platform_set_drvdata(pdev, lpc18xx_pwm); + platform_set_drvdata(pdev, chip); return 0; } static void lpc18xx_pwm_remove(struct platform_device *pdev) { - struct lpc18xx_pwm_chip *lpc18xx_pwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct lpc18xx_pwm_chip *lpc18xx_pwm = to_lpc18xx_pwm_chip(chip); u32 val; - pwmchip_remove(&lpc18xx_pwm->chip); + pwmchip_remove(chip); val = lpc18xx_pwm_readl(lpc18xx_pwm, LPC18XX_PWM_CTRL); lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CTRL, From 1dc7dcba0b2d6148cd23d5b0880dfdeb73fb4b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:50 +0100 Subject: [PATCH 0493/1038] pwm: lpc18xx-sct: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-lpc18xx-sct driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/d480891340e927705843cd09e5777d4daaa6c9cc.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpc18xx-sct.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c index 9b1e8141aec7..04b76d257fd8 100644 --- a/drivers/pwm/pwm-lpc18xx-sct.c +++ b/drivers/pwm/pwm-lpc18xx-sct.c @@ -92,7 +92,6 @@ struct lpc18xx_pwm_data { }; struct lpc18xx_pwm_chip { - struct pwm_chip chip; void __iomem *base; struct clk *pwm_clk; unsigned long clk_rate; @@ -109,7 +108,7 @@ struct lpc18xx_pwm_chip { static inline struct lpc18xx_pwm_chip * to_lpc18xx_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct lpc18xx_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline void lpc18xx_pwm_writel(struct lpc18xx_pwm_chip *lpc18xx_pwm, @@ -353,11 +352,10 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) int ret; u64 val; - lpc18xx_pwm = devm_kzalloc(&pdev->dev, sizeof(*lpc18xx_pwm), - GFP_KERNEL); - if (!lpc18xx_pwm) - return -ENOMEM; - chip = &lpc18xx_pwm->chip; + chip = devm_pwmchip_alloc(&pdev->dev, LPC18XX_NUM_PWMS, sizeof(*lpc18xx_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + lpc18xx_pwm = to_lpc18xx_pwm_chip(chip); lpc18xx_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpc18xx_pwm->base)) @@ -388,9 +386,7 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) lpc18xx_pwm->min_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, lpc18xx_pwm->clk_rate); - chip->dev = &pdev->dev; chip->ops = &lpc18xx_pwm_ops; - chip->npwm = LPC18XX_NUM_PWMS; /* SCT counter must be in unify (32 bit) mode */ lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CONFIG, From ac4b44cac51c9e3d3235aa12aadf2141e7029b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:51 +0100 Subject: [PATCH 0494/1038] pwm: lpc32xx: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-lpc32xx driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_lpc32xx_pwm_chip() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/ef26238772ee9ca455a49e9b976a4f66654b26f7.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpc32xx.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-lpc32xx.c b/drivers/pwm/pwm-lpc32xx.c index 1d9f3e7a2434..c748537e57d1 100644 --- a/drivers/pwm/pwm-lpc32xx.c +++ b/drivers/pwm/pwm-lpc32xx.c @@ -15,7 +15,6 @@ #include struct lpc32xx_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; }; @@ -23,8 +22,10 @@ struct lpc32xx_pwm_chip { #define PWM_ENABLE BIT(31) #define PWM_PIN_LEVEL BIT(30) -#define to_lpc32xx_pwm_chip(_chip) \ - container_of(_chip, struct lpc32xx_pwm_chip, chip) +static inline struct lpc32xx_pwm_chip *to_lpc32xx_pwm_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int lpc32xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, int duty_ns, int period_ns) @@ -119,13 +120,15 @@ static const struct pwm_ops lpc32xx_pwm_ops = { static int lpc32xx_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct lpc32xx_pwm_chip *lpc32xx; int ret; u32 val; - lpc32xx = devm_kzalloc(&pdev->dev, sizeof(*lpc32xx), GFP_KERNEL); - if (!lpc32xx) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*lpc32xx)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + lpc32xx = to_lpc32xx_pwm_chip(chip); lpc32xx->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpc32xx->base)) @@ -135,16 +138,14 @@ static int lpc32xx_pwm_probe(struct platform_device *pdev) if (IS_ERR(lpc32xx->clk)) return PTR_ERR(lpc32xx->clk); - lpc32xx->chip.dev = &pdev->dev; - lpc32xx->chip.ops = &lpc32xx_pwm_ops; - lpc32xx->chip.npwm = 1; + chip->ops = &lpc32xx_pwm_ops; /* If PWM is disabled, configure the output to the default value */ val = readl(lpc32xx->base); val &= ~PWM_PIN_LEVEL; writel(val, lpc32xx->base); - ret = devm_pwmchip_add(&pdev->dev, &lpc32xx->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip, error %d\n", ret); return ret; From 0f83bfd80d8671d24a67c655ab10991b7d768796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:52 +0100 Subject: [PATCH 0495/1038] pwm: lpss: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/77617ba84bed7807a05779109432a292dd79f72f.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpss.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c index a6ea3ce7e019..394c768f5a5f 100644 --- a/drivers/pwm/pwm-lpss.c +++ b/drivers/pwm/pwm-lpss.c @@ -106,7 +106,7 @@ static int pwm_lpss_wait_for_update(struct pwm_device *pwm) */ err = readl_poll_timeout(addr, val, !(val & PWM_SW_UPDATE), 40, ms); if (err) - dev_err(pwm->chip->dev, "PWM_SW_UPDATE was not cleared\n"); + dev_err(pwmchip_parent(pwm->chip), "PWM_SW_UPDATE was not cleared\n"); return err; } @@ -114,7 +114,7 @@ static int pwm_lpss_wait_for_update(struct pwm_device *pwm) static inline int pwm_lpss_is_updating(struct pwm_device *pwm) { if (pwm_lpss_read(pwm) & PWM_SW_UPDATE) { - dev_err(pwm->chip->dev, "PWM_SW_UPDATE is still set, skipping update\n"); + dev_err(pwmchip_parent(pwm->chip), "PWM_SW_UPDATE is still set, skipping update\n"); return -EBUSY; } @@ -190,16 +190,16 @@ static int pwm_lpss_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->enabled) { if (!pwm_is_enabled(pwm)) { - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); ret = pwm_lpss_prepare_enable(lpwm, pwm, state); if (ret) - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); } else { ret = pwm_lpss_prepare_enable(lpwm, pwm, state); } } else if (pwm_is_enabled(pwm)) { pwm_lpss_write(pwm, pwm_lpss_read(pwm) & ~PWM_ENABLE); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); } return ret; @@ -213,7 +213,7 @@ static int pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, unsigned long long base_unit, freq, on_time_div; u32 ctrl; - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); base_unit_range = BIT(lpwm->info->base_unit_bits); @@ -235,7 +235,7 @@ static int pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->polarity = PWM_POLARITY_NORMAL; state->enabled = !!(ctrl & PWM_ENABLE); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } From 11265c637e7b196aae64428b2ec60eb21b34ecc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:53 +0100 Subject: [PATCH 0496/1038] pwm: lpss-*: Don't set driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drivers don't make use of driver data, neither in the variant pci_get_drvdata() nor as platform_get_drvdata() nor as dev_get_drvdata(). Drop setting driver data accordingly. Link: https://lore.kernel.org/r/25671b7562d96b67fd05b3fa6fc76538ddf5914f.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-lpss-pci.c | 2 -- drivers/pwm/pwm-lpss-platform.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c index b4134bee2863..34acfe99b74f 100644 --- a/drivers/pwm/pwm-lpss-pci.c +++ b/drivers/pwm/pwm-lpss-pci.c @@ -34,8 +34,6 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, if (IS_ERR(lpwm)) return PTR_ERR(lpwm); - pci_set_drvdata(pdev, lpwm); - pm_runtime_put(&pdev->dev); pm_runtime_allow(&pdev->dev); diff --git a/drivers/pwm/pwm-lpss-platform.c b/drivers/pwm/pwm-lpss-platform.c index 319809aac2c4..5f6ee300e342 100644 --- a/drivers/pwm/pwm-lpss-platform.c +++ b/drivers/pwm/pwm-lpss-platform.c @@ -35,8 +35,6 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) if (IS_ERR(lpwm)) return PTR_ERR(lpwm); - platform_set_drvdata(pdev, lpwm); - /* * On Cherry Trail devices the GFX0._PS0 AML checks if the controller * is on and if it is not on it turns it on and restores what it From 05013062a89fa4c2d5913dfb81a8ae0268e0a9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:54 +0100 Subject: [PATCH 0497/1038] pwm: lpss-*: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-lpss drivers to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/b567ab5dd992e361eb884fa6c2cac11be9c7dde3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pinctrl/intel/pinctrl-intel.c | 6 +++--- drivers/pwm/pwm-lpss-pci.c | 8 ++++---- drivers/pwm/pwm-lpss-platform.c | 8 ++++---- drivers/pwm/pwm-lpss.c | 24 +++++++++++----------- drivers/pwm/pwm-lpss.h | 1 - include/linux/platform_data/x86/pwm-lpss.h | 4 ++-- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index d6f29e6faab7..89bd7ce6711a 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -1492,7 +1492,7 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl, .base_unit_bits = 22, .bypass = true, }; - struct pwm_lpss_chip *pwm; + struct pwm_chip *chip; if (!(community->features & PINCTRL_FEATURE_PWM)) return 0; @@ -1500,8 +1500,8 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl, if (!IS_REACHABLE(CONFIG_PWM_LPSS)) return 0; - pwm = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info); - return PTR_ERR_OR_ZERO(pwm); + chip = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info); + return PTR_ERR_OR_ZERO(chip); } int intel_pinctrl_probe(struct platform_device *pdev, diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c index 34acfe99b74f..25045c229520 100644 --- a/drivers/pwm/pwm-lpss-pci.c +++ b/drivers/pwm/pwm-lpss-pci.c @@ -18,7 +18,7 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, const struct pci_device_id *id) { const struct pwm_lpss_boardinfo *info; - struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; int err; err = pcim_enable_device(pdev); @@ -30,9 +30,9 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, return err; info = (struct pwm_lpss_boardinfo *)id->driver_data; - lpwm = devm_pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); - if (IS_ERR(lpwm)) - return PTR_ERR(lpwm); + chip = devm_pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); + if (IS_ERR(chip)) + return PTR_ERR(chip); pm_runtime_put(&pdev->dev); pm_runtime_allow(&pdev->dev); diff --git a/drivers/pwm/pwm-lpss-platform.c b/drivers/pwm/pwm-lpss-platform.c index 5f6ee300e342..dbc9f5b17bdc 100644 --- a/drivers/pwm/pwm-lpss-platform.c +++ b/drivers/pwm/pwm-lpss-platform.c @@ -20,7 +20,7 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) { const struct pwm_lpss_boardinfo *info; - struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; void __iomem *base; info = device_get_match_data(&pdev->dev); @@ -31,9 +31,9 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - lpwm = devm_pwm_lpss_probe(&pdev->dev, base, info); - if (IS_ERR(lpwm)) - return PTR_ERR(lpwm); + chip = devm_pwm_lpss_probe(&pdev->dev, base, info); + if (IS_ERR(chip)) + return PTR_ERR(chip); /* * On Cherry Trail devices the GFX0._PS0 AML checks if the controller diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c index 394c768f5a5f..867e2bc8c601 100644 --- a/drivers/pwm/pwm-lpss.c +++ b/drivers/pwm/pwm-lpss.c @@ -68,7 +68,7 @@ EXPORT_SYMBOL_GPL(pwm_lpss_tng_info); static inline struct pwm_lpss_chip *to_lpwm(struct pwm_chip *chip) { - return container_of(chip, struct pwm_lpss_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 pwm_lpss_read(const struct pwm_device *pwm) @@ -245,10 +245,11 @@ static const struct pwm_ops pwm_lpss_ops = { .get_state = pwm_lpss_get_state, }; -struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, - const struct pwm_lpss_boardinfo *info) +struct pwm_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, + const struct pwm_lpss_boardinfo *info) { struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; unsigned long c; int i, ret; u32 ctrl; @@ -256,9 +257,10 @@ struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base if (WARN_ON(info->npwm > LPSS_MAX_PWMS)) return ERR_PTR(-ENODEV); - lpwm = devm_kzalloc(dev, sizeof(*lpwm), GFP_KERNEL); - if (!lpwm) - return ERR_PTR(-ENOMEM); + chip = devm_pwmchip_alloc(dev, info->npwm, sizeof(*lpwm)); + if (IS_ERR(chip)) + return chip; + lpwm = to_lpwm(chip); lpwm->regs = base; lpwm->info = info; @@ -267,23 +269,21 @@ struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base if (!c) return ERR_PTR(-EINVAL); - lpwm->chip.dev = dev; - lpwm->chip.ops = &pwm_lpss_ops; - lpwm->chip.npwm = info->npwm; + chip->ops = &pwm_lpss_ops; - ret = devm_pwmchip_add(dev, &lpwm->chip); + ret = devm_pwmchip_add(dev, chip); if (ret) { dev_err(dev, "failed to add PWM chip: %d\n", ret); return ERR_PTR(ret); } for (i = 0; i < lpwm->info->npwm; i++) { - ctrl = pwm_lpss_read(&lpwm->chip.pwms[i]); + ctrl = pwm_lpss_read(&chip->pwms[i]); if (ctrl & PWM_ENABLE) pm_runtime_get(dev); } - return lpwm; + return chip; } EXPORT_SYMBOL_GPL(devm_pwm_lpss_probe); diff --git a/drivers/pwm/pwm-lpss.h b/drivers/pwm/pwm-lpss.h index bf841250385f..b5267ab5193b 100644 --- a/drivers/pwm/pwm-lpss.h +++ b/drivers/pwm/pwm-lpss.h @@ -18,7 +18,6 @@ #define LPSS_MAX_PWMS 4 struct pwm_lpss_chip { - struct pwm_chip chip; void __iomem *regs; const struct pwm_lpss_boardinfo *info; }; diff --git a/include/linux/platform_data/x86/pwm-lpss.h b/include/linux/platform_data/x86/pwm-lpss.h index c852fe24fe2a..752c06b47cc8 100644 --- a/include/linux/platform_data/x86/pwm-lpss.h +++ b/include/linux/platform_data/x86/pwm-lpss.h @@ -27,7 +27,7 @@ struct pwm_lpss_boardinfo { bool other_devices_aml_touches_pwm_regs; }; -struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, - const struct pwm_lpss_boardinfo *info); +struct pwm_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, + const struct pwm_lpss_boardinfo *info); #endif /* __PLATFORM_DATA_X86_PWM_LPSS_H */ From f1b1e74731a80a61c400148f973b0f2d9c1733e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:55 +0100 Subject: [PATCH 0498/1038] pwm: mediatek: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/819996ee175bec0ab71acc385c78e7233617e136.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mediatek.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index 292c8bd5b343..d8bdadb7b338 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -149,7 +149,7 @@ static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, if (clkdiv > PWM_CLK_DIV_MAX) { pwm_mediatek_clk_disable(chip, pwm); - dev_err(chip->dev, "period of %d ns not supported\n", period_ns); + dev_err(pwmchip_parent(chip), "period of %d ns not supported\n", period_ns); return -EINVAL; } From 1c8090d7b3babd2e8ae79ecac285b8c64eb4fe67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:56 +0100 Subject: [PATCH 0499/1038] pwm: mediatek: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-mediatek driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/513b2037c00ebf2f3a69472f4b99adaf4fb40d83.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mediatek.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index d8bdadb7b338..19a87873ad60 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -42,7 +42,6 @@ struct pwm_mediatek_of_data { /** * struct pwm_mediatek_chip - struct representing PWM chip - * @chip: linux PWM chip representation * @regs: base address of PWM chip * @clk_top: the top clock generator * @clk_main: the clock used by PWM core @@ -50,7 +49,6 @@ struct pwm_mediatek_of_data { * @soc: pointer to chip's platform data */ struct pwm_mediatek_chip { - struct pwm_chip chip; void __iomem *regs; struct clk *clk_top; struct clk *clk_main; @@ -69,7 +67,7 @@ static const unsigned int mtk_pwm_reg_offset_v2[] = { static inline struct pwm_mediatek_chip * to_pwm_mediatek_chip(struct pwm_chip *chip) { - return container_of(chip, struct pwm_mediatek_chip, chip); + return pwmchip_get_drvdata(chip); } static int pwm_mediatek_clk_enable(struct pwm_chip *chip, @@ -232,21 +230,26 @@ static const struct pwm_ops pwm_mediatek_ops = { static int pwm_mediatek_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_mediatek_chip *pc; + const struct pwm_mediatek_of_data *soc; unsigned int i; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + soc = of_device_get_match_data(&pdev->dev); - pc->soc = of_device_get_match_data(&pdev->dev); + chip = devm_pwmchip_alloc(&pdev->dev, soc->num_pwms, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_pwm_mediatek_chip(chip); + + pc->soc = soc; pc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->regs)) return PTR_ERR(pc->regs); - pc->clk_pwms = devm_kmalloc_array(&pdev->dev, pc->soc->num_pwms, + pc->clk_pwms = devm_kmalloc_array(&pdev->dev, soc->num_pwms, sizeof(*pc->clk_pwms), GFP_KERNEL); if (!pc->clk_pwms) return -ENOMEM; @@ -261,7 +264,7 @@ static int pwm_mediatek_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk_main), "Failed to get main clock\n"); - for (i = 0; i < pc->soc->num_pwms; i++) { + for (i = 0; i < soc->num_pwms; i++) { char name[8]; snprintf(name, sizeof(name), "pwm%d", i + 1); @@ -272,11 +275,9 @@ static int pwm_mediatek_probe(struct platform_device *pdev) "Failed to get %s clock\n", name); } - pc->chip.dev = &pdev->dev; - pc->chip.ops = &pwm_mediatek_ops; - pc->chip.npwm = pc->soc->num_pwms; + chip->ops = &pwm_mediatek_ops; - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); From b647dcfde1b5683abf7a939897888ca2786ac6c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:57 +0100 Subject: [PATCH 0500/1038] pwm: meson: Change prototype of a few helpers to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given meson_pwm. To just not have to do that, rework meson_pwm_calc(), meson_pwm_enable(), meson_pwm_disable() and meson_pwm_init_channels() to take a pwm_chip. Link: https://lore.kernel.org/r/157084e1dd464da0097edba880240868c1c0b27e.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-meson.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c index 2971bbf3b5e7..7ce41811537d 100644 --- a/drivers/pwm/pwm-meson.c +++ b/drivers/pwm/pwm-meson.c @@ -143,9 +143,10 @@ static void meson_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) clk_disable_unprepare(channel->clk); } -static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, +static int meson_pwm_calc(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { + struct meson_pwm *meson = to_meson_pwm(chip); struct meson_pwm_channel *channel = &meson->channels[pwm->hwpwm]; unsigned int cnt, duty_cnt; unsigned long fin_freq; @@ -169,19 +170,19 @@ static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, fin_freq = clk_round_rate(channel->clk, freq); if (fin_freq == 0) { - dev_err(meson->chip.dev, "invalid source clock frequency\n"); + dev_err(chip->dev, "invalid source clock frequency\n"); return -EINVAL; } - dev_dbg(meson->chip.dev, "fin_freq: %lu Hz\n", fin_freq); + dev_dbg(chip->dev, "fin_freq: %lu Hz\n", fin_freq); cnt = div_u64(fin_freq * period, NSEC_PER_SEC); if (cnt > 0xffff) { - dev_err(meson->chip.dev, "unable to get period cnt\n"); + dev_err(chip->dev, "unable to get period cnt\n"); return -EINVAL; } - dev_dbg(meson->chip.dev, "period=%llu cnt=%u\n", period, cnt); + dev_dbg(chip->dev, "period=%llu cnt=%u\n", period, cnt); if (duty == period) { channel->hi = cnt; @@ -192,7 +193,7 @@ static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, } else { duty_cnt = div_u64(fin_freq * duty, NSEC_PER_SEC); - dev_dbg(meson->chip.dev, "duty=%llu duty_cnt=%u\n", duty, duty_cnt); + dev_dbg(chip->dev, "duty=%llu duty_cnt=%u\n", duty, duty_cnt); channel->hi = duty_cnt; channel->lo = cnt - duty_cnt; @@ -203,8 +204,9 @@ static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, return 0; } -static void meson_pwm_enable(struct meson_pwm *meson, struct pwm_device *pwm) +static void meson_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct meson_pwm *meson = to_meson_pwm(chip); struct meson_pwm_channel *channel = &meson->channels[pwm->hwpwm]; struct meson_pwm_channel_data *channel_data; unsigned long flags; @@ -215,7 +217,7 @@ static void meson_pwm_enable(struct meson_pwm *meson, struct pwm_device *pwm) err = clk_set_rate(channel->clk, channel->rate); if (err) - dev_err(meson->chip.dev, "setting clock rate failed\n"); + dev_err(chip->dev, "setting clock rate failed\n"); spin_lock_irqsave(&meson->lock, flags); @@ -230,8 +232,9 @@ static void meson_pwm_enable(struct meson_pwm *meson, struct pwm_device *pwm) spin_unlock_irqrestore(&meson->lock, flags); } -static void meson_pwm_disable(struct meson_pwm *meson, struct pwm_device *pwm) +static void meson_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct meson_pwm *meson = to_meson_pwm(chip); unsigned long flags; u32 value; @@ -269,16 +272,16 @@ static int meson_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, channel->hi = ~0; channel->lo = 0; - meson_pwm_enable(meson, pwm); + meson_pwm_enable(chip, pwm); } else { - meson_pwm_disable(meson, pwm); + meson_pwm_disable(chip, pwm); } } else { - err = meson_pwm_calc(meson, pwm, state); + err = meson_pwm_calc(chip, pwm, state); if (err < 0) return err; - meson_pwm_enable(meson, pwm); + meson_pwm_enable(chip, pwm); } return 0; @@ -432,10 +435,11 @@ static const struct of_device_id meson_pwm_matches[] = { }; MODULE_DEVICE_TABLE(of, meson_pwm_matches); -static int meson_pwm_init_channels(struct meson_pwm *meson) +static int meson_pwm_init_channels(struct pwm_chip *chip) { + struct meson_pwm *meson = to_meson_pwm(chip); struct clk_parent_data mux_parent_data[MESON_MAX_MUX_PARENTS] = {}; - struct device *dev = meson->chip.dev; + struct device *dev = chip->dev; unsigned int i; char name[255]; int err; @@ -445,7 +449,7 @@ static int meson_pwm_init_channels(struct meson_pwm *meson) mux_parent_data[i].name = meson->data->parent_names[i]; } - for (i = 0; i < meson->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { struct meson_pwm_channel *channel = &meson->channels[i]; struct clk_parent_data div_parent = {}, gate_parent = {}; struct clk_init_data init = {}; @@ -543,7 +547,7 @@ static int meson_pwm_probe(struct platform_device *pdev) meson->data = of_device_get_match_data(&pdev->dev); - err = meson_pwm_init_channels(meson); + err = meson_pwm_init_channels(&meson->chip); if (err < 0) return err; From e369035a98894a7327e78ebd01ec859c6654fc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:58 +0100 Subject: [PATCH 0501/1038] pwm: meson: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/02d5f32001c4d0817fa79f5831cfd16b7e345f00.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-meson.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c index 7ce41811537d..8f67d6ba443d 100644 --- a/drivers/pwm/pwm-meson.c +++ b/drivers/pwm/pwm-meson.c @@ -122,7 +122,7 @@ static int meson_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { struct meson_pwm *meson = to_meson_pwm(chip); struct meson_pwm_channel *channel = &meson->channels[pwm->hwpwm]; - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int err; err = clk_prepare_enable(channel->clk); @@ -170,19 +170,19 @@ static int meson_pwm_calc(struct pwm_chip *chip, struct pwm_device *pwm, fin_freq = clk_round_rate(channel->clk, freq); if (fin_freq == 0) { - dev_err(chip->dev, "invalid source clock frequency\n"); + dev_err(pwmchip_parent(chip), "invalid source clock frequency\n"); return -EINVAL; } - dev_dbg(chip->dev, "fin_freq: %lu Hz\n", fin_freq); + dev_dbg(pwmchip_parent(chip), "fin_freq: %lu Hz\n", fin_freq); cnt = div_u64(fin_freq * period, NSEC_PER_SEC); if (cnt > 0xffff) { - dev_err(chip->dev, "unable to get period cnt\n"); + dev_err(pwmchip_parent(chip), "unable to get period cnt\n"); return -EINVAL; } - dev_dbg(chip->dev, "period=%llu cnt=%u\n", period, cnt); + dev_dbg(pwmchip_parent(chip), "period=%llu cnt=%u\n", period, cnt); if (duty == period) { channel->hi = cnt; @@ -193,7 +193,7 @@ static int meson_pwm_calc(struct pwm_chip *chip, struct pwm_device *pwm, } else { duty_cnt = div_u64(fin_freq * duty, NSEC_PER_SEC); - dev_dbg(chip->dev, "duty=%llu duty_cnt=%u\n", duty, duty_cnt); + dev_dbg(pwmchip_parent(chip), "duty=%llu duty_cnt=%u\n", duty, duty_cnt); channel->hi = duty_cnt; channel->lo = cnt - duty_cnt; @@ -217,7 +217,7 @@ static void meson_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) err = clk_set_rate(channel->clk, channel->rate); if (err) - dev_err(chip->dev, "setting clock rate failed\n"); + dev_err(pwmchip_parent(chip), "setting clock rate failed\n"); spin_lock_irqsave(&meson->lock, flags); @@ -439,7 +439,7 @@ static int meson_pwm_init_channels(struct pwm_chip *chip) { struct meson_pwm *meson = to_meson_pwm(chip); struct clk_parent_data mux_parent_data[MESON_MAX_MUX_PARENTS] = {}; - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); unsigned int i; char name[255]; int err; From 28ecf9bd5ea24ca5ede01657afbd2551fa455f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:31:59 +0100 Subject: [PATCH 0502/1038] pwm: meson: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-meson driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/6cc57880e910e6ffd8df15bc4a41c42fe9523293.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-meson.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c index 8f67d6ba443d..40a5b64c26f5 100644 --- a/drivers/pwm/pwm-meson.c +++ b/drivers/pwm/pwm-meson.c @@ -102,7 +102,6 @@ struct meson_pwm_data { }; struct meson_pwm { - struct pwm_chip chip; const struct meson_pwm_data *data; struct meson_pwm_channel channels[MESON_NUM_PWMS]; void __iomem *base; @@ -115,7 +114,7 @@ struct meson_pwm { static inline struct meson_pwm *to_meson_pwm(struct pwm_chip *chip) { - return container_of(chip, struct meson_pwm, chip); + return pwmchip_get_drvdata(chip); } static int meson_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -529,29 +528,29 @@ static int meson_pwm_init_channels(struct pwm_chip *chip) static int meson_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct meson_pwm *meson; int err; - meson = devm_kzalloc(&pdev->dev, sizeof(*meson), GFP_KERNEL); - if (!meson) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, MESON_NUM_PWMS, sizeof(*meson)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + meson = to_meson_pwm(chip); meson->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(meson->base)) return PTR_ERR(meson->base); spin_lock_init(&meson->lock); - meson->chip.dev = &pdev->dev; - meson->chip.ops = &meson_pwm_ops; - meson->chip.npwm = MESON_NUM_PWMS; + chip->ops = &meson_pwm_ops; meson->data = of_device_get_match_data(&pdev->dev); - err = meson_pwm_init_channels(&meson->chip); + err = meson_pwm_init_channels(chip); if (err < 0) return err; - err = devm_pwmchip_add(&pdev->dev, &meson->chip); + err = devm_pwmchip_add(&pdev->dev, chip); if (err < 0) return dev_err_probe(&pdev->dev, err, "failed to register PWM chip\n"); From f962b190fe20d626d4b9fdf782efcb93584c6164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:00 +0100 Subject: [PATCH 0503/1038] pwm: microchip-core: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-microchip-core driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Acked-by: Conor Dooley Link: https://lore.kernel.org/r/c6660b655c1d4c9d79280030e24090a4a4c5dd8b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-microchip-core.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-microchip-core.c b/drivers/pwm/pwm-microchip-core.c index c0c53968f3e9..c1f2287b8e97 100644 --- a/drivers/pwm/pwm-microchip-core.c +++ b/drivers/pwm/pwm-microchip-core.c @@ -54,7 +54,6 @@ #define MCHPCOREPWM_TIMEOUT_MS 100u struct mchp_core_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct mutex lock; /* protects the shared period */ @@ -65,7 +64,7 @@ struct mchp_core_pwm_chip { static inline struct mchp_core_pwm_chip *to_mchp_core_pwm(struct pwm_chip *chip) { - return container_of(chip, struct mchp_core_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void mchp_core_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm, @@ -447,13 +446,15 @@ MODULE_DEVICE_TABLE(of, mchp_core_of_match); static int mchp_core_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct mchp_core_pwm_chip *mchp_core_pwm; struct resource *regs; int ret; - mchp_core_pwm = devm_kzalloc(&pdev->dev, sizeof(*mchp_core_pwm), GFP_KERNEL); - if (!mchp_core_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 16, sizeof(*mchp_core_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mchp_core_pwm = to_mchp_core_pwm(chip); mchp_core_pwm->base = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); if (IS_ERR(mchp_core_pwm->base)) @@ -470,9 +471,7 @@ static int mchp_core_pwm_probe(struct platform_device *pdev) mutex_init(&mchp_core_pwm->lock); - mchp_core_pwm->chip.dev = &pdev->dev; - mchp_core_pwm->chip.ops = &mchp_core_pwm_ops; - mchp_core_pwm->chip.npwm = 16; + chip->ops = &mchp_core_pwm_ops; mchp_core_pwm->channel_enabled = readb_relaxed(mchp_core_pwm->base + MCHPCOREPWM_EN(0)); mchp_core_pwm->channel_enabled |= @@ -485,7 +484,7 @@ static int mchp_core_pwm_probe(struct platform_device *pdev) writel_relaxed(1U, mchp_core_pwm->base + MCHPCOREPWM_SYNC_UPD); mchp_core_pwm->update_timestamp = ktime_get(); - ret = devm_pwmchip_add(&pdev->dev, &mchp_core_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "Failed to add pwmchip\n"); From f580fa6c842b95e328edd9639e5f0567ff4fdf19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:01 +0100 Subject: [PATCH 0504/1038] pwm: mtk-disp: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/93ba7ef22f8d26f17e64326249d5da54ab8014ea.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mtk-disp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-mtk-disp.c b/drivers/pwm/pwm-mtk-disp.c index a72f7be36996..186aa5c04435 100644 --- a/drivers/pwm/pwm-mtk-disp.c +++ b/drivers/pwm/pwm-mtk-disp.c @@ -91,14 +91,14 @@ static int mtk_disp_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!mdp->enabled) { err = clk_prepare_enable(mdp->clk_main); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_main: %pe\n", + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_main: %pe\n", ERR_PTR(err)); return err; } err = clk_prepare_enable(mdp->clk_mm); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_mm: %pe\n", + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_mm: %pe\n", ERR_PTR(err)); clk_disable_unprepare(mdp->clk_main); return err; @@ -181,13 +181,13 @@ static int mtk_disp_pwm_get_state(struct pwm_chip *chip, err = clk_prepare_enable(mdp->clk_main); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_main: %pe\n", ERR_PTR(err)); + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_main: %pe\n", ERR_PTR(err)); return err; } err = clk_prepare_enable(mdp->clk_mm); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_mm: %pe\n", ERR_PTR(err)); + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_mm: %pe\n", ERR_PTR(err)); clk_disable_unprepare(mdp->clk_main); return err; } From ad0828825e71cfe249cc76136171b7f42fd22f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:02 +0100 Subject: [PATCH 0505/1038] pwm: mtk-disp: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-mtk-disp driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/a70bf253dc41c61d9b901423710c97f235e372c3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mtk-disp.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-mtk-disp.c b/drivers/pwm/pwm-mtk-disp.c index 186aa5c04435..bafd6b6195f6 100644 --- a/drivers/pwm/pwm-mtk-disp.c +++ b/drivers/pwm/pwm-mtk-disp.c @@ -42,7 +42,6 @@ struct mtk_pwm_data { }; struct mtk_disp_pwm { - struct pwm_chip chip; const struct mtk_pwm_data *data; struct clk *clk_main; struct clk *clk_mm; @@ -52,7 +51,7 @@ struct mtk_disp_pwm { static inline struct mtk_disp_pwm *to_mtk_disp_pwm(struct pwm_chip *chip) { - return container_of(chip, struct mtk_disp_pwm, chip); + return pwmchip_get_drvdata(chip); } static void mtk_disp_pwm_update_bits(struct mtk_disp_pwm *mdp, u32 offset, @@ -231,12 +230,14 @@ static const struct pwm_ops mtk_disp_pwm_ops = { static int mtk_disp_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct mtk_disp_pwm *mdp; int ret; - mdp = devm_kzalloc(&pdev->dev, sizeof(*mdp), GFP_KERNEL); - if (!mdp) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*mdp)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mdp = to_mtk_disp_pwm(chip); mdp->data = of_device_get_match_data(&pdev->dev); @@ -254,11 +255,9 @@ static int mtk_disp_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(mdp->clk_mm), "Failed to get mm clock\n"); - mdp->chip.dev = &pdev->dev; - mdp->chip.ops = &mtk_disp_pwm_ops; - mdp->chip.npwm = 1; + chip->ops = &mtk_disp_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &mdp->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); From 2ae7039fa7fbb41169703d529edcd92ff513d347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:03 +0100 Subject: [PATCH 0506/1038] pwm: mxs: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-mxs driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_mxs_pwm_chip() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/39203f3bfbf4e97654db78bdc7e0c9a2b72c1368.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-mxs.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/drivers/pwm/pwm-mxs.c b/drivers/pwm/pwm-mxs.c index 1b5e787d78f1..8cad214b1c29 100644 --- a/drivers/pwm/pwm-mxs.c +++ b/drivers/pwm/pwm-mxs.c @@ -37,12 +37,14 @@ static const u8 cdiv_shift[PERIOD_CDIV_MAX] = { }; struct mxs_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; }; -#define to_mxs_pwm_chip(_chip) container_of(_chip, struct mxs_pwm_chip, chip) +static inline struct mxs_pwm_chip *to_mxs_pwm_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int mxs_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) @@ -120,12 +122,21 @@ static const struct pwm_ops mxs_pwm_ops = { static int mxs_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; + struct pwm_chip *chip; struct mxs_pwm_chip *mxs; + u32 npwm; int ret; - mxs = devm_kzalloc(&pdev->dev, sizeof(*mxs), GFP_KERNEL); - if (!mxs) - return -ENOMEM; + ret = of_property_read_u32(np, "fsl,pwm-number", &npwm); + if (ret < 0) { + dev_err(&pdev->dev, "failed to get pwm number: %d\n", ret); + return ret; + } + + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*mxs)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mxs = to_mxs_pwm_chip(chip); mxs->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mxs->base)) @@ -135,21 +146,14 @@ static int mxs_pwm_probe(struct platform_device *pdev) if (IS_ERR(mxs->clk)) return PTR_ERR(mxs->clk); - mxs->chip.dev = &pdev->dev; - mxs->chip.ops = &mxs_pwm_ops; - - ret = of_property_read_u32(np, "fsl,pwm-number", &mxs->chip.npwm); - if (ret < 0) { - dev_err(&pdev->dev, "failed to get pwm number: %d\n", ret); - return ret; - } + chip->ops = &mxs_pwm_ops; /* FIXME: Only do this if the PWM isn't already running */ ret = stmp_reset_block(mxs->base); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to reset PWM\n"); - ret = devm_pwmchip_add(&pdev->dev, &mxs->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add pwm chip %d\n", ret); return ret; From 8c01031abe7c23b5ce9022b77731d8f6d2e64f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:04 +0100 Subject: [PATCH 0507/1038] pwm: ntxec: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-ntxec driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/c0c9d6cb3a6662268e660f4f6c89b32268ecf019.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-ntxec.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-ntxec.c b/drivers/pwm/pwm-ntxec.c index 78606039eda2..28d1c2e5a98f 100644 --- a/drivers/pwm/pwm-ntxec.c +++ b/drivers/pwm/pwm-ntxec.c @@ -25,12 +25,11 @@ struct ntxec_pwm { struct ntxec *ec; - struct pwm_chip chip; }; static struct ntxec_pwm *ntxec_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct ntxec_pwm, chip); + return pwmchip_get_drvdata(chip); } #define NTXEC_REG_AUTO_OFF_HI 0xa1 @@ -141,16 +140,13 @@ static int ntxec_pwm_probe(struct platform_device *pdev) device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = ntxec_pwm_from_chip(chip); priv->ec = ec; - - chip = &priv->chip; - chip->dev = &pdev->dev; chip->ops = &ntxec_pwm_ops; - chip->npwm = 1; return devm_pwmchip_add(&pdev->dev, chip); } From 7fc81e231fe9a5c88ce392c0c0fe962da390464b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:05 +0100 Subject: [PATCH 0508/1038] pwm: omap-dmtimer: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/6681ebd024c2b8b2315ca3da5cac612e5f433d63.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-omap-dmtimer.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c index 496bd73d29fe..527a7aa0ce61 100644 --- a/drivers/pwm/pwm-omap-dmtimer.c +++ b/drivers/pwm/pwm-omap-dmtimer.c @@ -155,7 +155,7 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, unsigned long clk_rate; struct clk *fclk; - dev_dbg(chip->dev, "requested duty cycle: %d ns, period: %d ns\n", + dev_dbg(pwmchip_parent(chip), "requested duty cycle: %d ns, period: %d ns\n", duty_ns, period_ns); if (duty_ns == pwm_get_duty_cycle(pwm) && @@ -164,17 +164,17 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, fclk = omap->pdata->get_fclk(omap->dm_timer); if (!fclk) { - dev_err(chip->dev, "invalid pmtimer fclk\n"); + dev_err(pwmchip_parent(chip), "invalid pmtimer fclk\n"); return -EINVAL; } clk_rate = clk_get_rate(fclk); if (!clk_rate) { - dev_err(chip->dev, "invalid pmtimer fclk rate\n"); + dev_err(pwmchip_parent(chip), "invalid pmtimer fclk rate\n"); return -EINVAL; } - dev_dbg(chip->dev, "clk rate: %luHz\n", clk_rate); + dev_dbg(pwmchip_parent(chip), "clk rate: %luHz\n", clk_rate); /* * Calculate the appropriate load and match values based on the @@ -196,27 +196,27 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, duty_cycles = pwm_omap_dmtimer_get_clock_cycles(clk_rate, duty_ns); if (period_cycles < 2) { - dev_info(chip->dev, + dev_info(pwmchip_parent(chip), "period %d ns too short for clock rate %lu Hz\n", period_ns, clk_rate); return -EINVAL; } if (duty_cycles < 1) { - dev_dbg(chip->dev, + dev_dbg(pwmchip_parent(chip), "duty cycle %d ns is too short for clock rate %lu Hz\n", duty_ns, clk_rate); - dev_dbg(chip->dev, "using minimum of 1 clock cycle\n"); + dev_dbg(pwmchip_parent(chip), "using minimum of 1 clock cycle\n"); duty_cycles = 1; } else if (duty_cycles >= period_cycles) { - dev_dbg(chip->dev, + dev_dbg(pwmchip_parent(chip), "duty cycle %d ns is too long for period %d ns at clock rate %lu Hz\n", duty_ns, period_ns, clk_rate); - dev_dbg(chip->dev, "using maximum of 1 clock cycle less than period\n"); + dev_dbg(pwmchip_parent(chip), "using maximum of 1 clock cycle less than period\n"); duty_cycles = period_cycles - 1; } - dev_dbg(chip->dev, "effective duty cycle: %lld ns, period: %lld ns\n", + dev_dbg(pwmchip_parent(chip), "effective duty cycle: %lld ns, period: %lld ns\n", DIV_ROUND_CLOSEST_ULL((u64)NSEC_PER_SEC * duty_cycles, clk_rate), DIV_ROUND_CLOSEST_ULL((u64)NSEC_PER_SEC * period_cycles, @@ -228,7 +228,7 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, omap->pdata->set_load(omap->dm_timer, load_value); omap->pdata->set_match(omap->dm_timer, true, match_value); - dev_dbg(chip->dev, "load value: %#08x (%d), match value: %#08x (%d)\n", + dev_dbg(pwmchip_parent(chip), "load value: %#08x (%d), match value: %#08x (%d)\n", load_value, load_value, match_value, match_value); return 0; From 623a9594b2d337818b5c1d44cd2f63280c1178a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:06 +0100 Subject: [PATCH 0509/1038] pwm: omap-dmtimer: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct pwm_omap_dmtimer_chip. Use the pwm_chip as driver data instead of the pwm_omap_dmtimer_chip to get access to the pwm_chip in pwm_omap_dmtimer_remove() without using omap->chip. Link: https://lore.kernel.org/r/809672026f3d583ba4fc4bd92b525951df172106.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-omap-dmtimer.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c index 527a7aa0ce61..729d3be2a013 100644 --- a/drivers/pwm/pwm-omap-dmtimer.c +++ b/drivers/pwm/pwm-omap-dmtimer.c @@ -311,6 +311,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) struct dmtimer_platform_data *timer_pdata; const struct omap_dm_timer_ops *pdata; struct platform_device *timer_pdev; + struct pwm_chip *chip; struct pwm_omap_dmtimer_chip *omap; struct omap_dm_timer *dm_timer; struct device_node *timer; @@ -373,6 +374,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) ret = -ENOMEM; goto err_alloc_omap; } + chip = &omap->chip; omap->pdata = pdata; omap->dm_timer = dm_timer; @@ -392,11 +394,11 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) if (!of_property_read_u32(pdev->dev.of_node, "ti,clock-source", &v)) omap->pdata->set_source(omap->dm_timer, v); - omap->chip.dev = &pdev->dev; - omap->chip.ops = &pwm_omap_dmtimer_ops; - omap->chip.npwm = 1; + chip->dev = &pdev->dev; + chip->ops = &pwm_omap_dmtimer_ops; + chip->npwm = 1; - ret = pwmchip_add(&omap->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "failed to register PWM\n"); goto err_pwmchip_add; @@ -404,7 +406,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) of_node_put(timer); - platform_set_drvdata(pdev, omap); + platform_set_drvdata(pdev, chip); return 0; @@ -432,9 +434,10 @@ err_find_timer_pdev: static void pwm_omap_dmtimer_remove(struct platform_device *pdev) { - struct pwm_omap_dmtimer_chip *omap = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct pwm_omap_dmtimer_chip *omap = to_pwm_omap_dmtimer_chip(chip); - pwmchip_remove(&omap->chip); + pwmchip_remove(chip); if (pm_runtime_active(&omap->dm_timer_pdev->dev)) omap->pdata->stop(omap->dm_timer); From f988b8196322b11c9b49c8df56aa4fada8a6b96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:07 +0100 Subject: [PATCH 0510/1038] pwm: omap-dmtimer: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-omap-dmtimer driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/78d1448b0fd1136b010f629feffb9d89c853e472.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-omap-dmtimer.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c index 729d3be2a013..cd51c4a938f5 100644 --- a/drivers/pwm/pwm-omap-dmtimer.c +++ b/drivers/pwm/pwm-omap-dmtimer.c @@ -53,13 +53,11 @@ /** * struct pwm_omap_dmtimer_chip - Structure representing a pwm chip * corresponding to omap dmtimer. - * @chip: PWM chip structure representing PWM controller * @dm_timer: Pointer to omap dm timer. * @pdata: Pointer to omap dm timer ops. * @dm_timer_pdev: Pointer to omap dm timer platform device */ struct pwm_omap_dmtimer_chip { - struct pwm_chip chip; /* Mutex to protect pwm apply state */ struct omap_dm_timer *dm_timer; const struct omap_dm_timer_ops *pdata; @@ -69,7 +67,7 @@ struct pwm_omap_dmtimer_chip { static inline struct pwm_omap_dmtimer_chip * to_pwm_omap_dmtimer_chip(struct pwm_chip *chip) { - return container_of(chip, struct pwm_omap_dmtimer_chip, chip); + return pwmchip_get_drvdata(chip); } /** @@ -369,12 +367,12 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) goto err_request_timer; } - omap = devm_kzalloc(&pdev->dev, sizeof(*omap), GFP_KERNEL); - if (!omap) { - ret = -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*omap)); + if (IS_ERR(chip)) { + ret = PTR_ERR(chip); goto err_alloc_omap; } - chip = &omap->chip; + omap = to_pwm_omap_dmtimer_chip(chip); omap->pdata = pdata; omap->dm_timer = dm_timer; @@ -394,9 +392,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) if (!of_property_read_u32(pdev->dev.of_node, "ti,clock-source", &v)) omap->pdata->set_source(omap->dm_timer, v); - chip->dev = &pdev->dev; chip->ops = &pwm_omap_dmtimer_ops; - chip->npwm = 1; ret = pwmchip_add(chip); if (ret < 0) { From da5a8c555c52e36040c7ccea6a9136349eb16f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:08 +0100 Subject: [PATCH 0511/1038] pwm: pca9685: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct pca9685. Use the pwm_chip as i2c client data and gpiochip data instead of the pca9685 to get access to the pwm_chip in pca9685_pwm_remove(), pca9685_pwm_runtime_suspend(), pca9685_pwm_runtime_resume(), pca9685_pwm_gpio_request() and the gpio callbacks without using pca->chip. Also the prototype for various helper functions is changed to take a pwm_chip instead of a pca9685 for the same reason. Link: https://lore.kernel.org/r/e87b378545cee15a52f0d410fcceebf671ab90e0.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-pca9685.c | 143 ++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 68 deletions(-) diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index e79b1de8c4d8..d7542517fa49 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -107,9 +107,10 @@ static bool pca9685_prescaler_can_change(struct pca9685 *pca, int channel) return test_bit(channel, pca->pwms_enabled); } -static int pca9685_read_reg(struct pca9685 *pca, unsigned int reg, unsigned int *val) +static int pca9685_read_reg(struct pwm_chip *chip, unsigned int reg, unsigned int *val) { - struct device *dev = pca->chip.dev; + struct pca9685 *pca = to_pca(chip); + struct device *dev = chip->dev; int err; err = regmap_read(pca->regmap, reg, val); @@ -119,9 +120,10 @@ static int pca9685_read_reg(struct pca9685 *pca, unsigned int reg, unsigned int return err; } -static int pca9685_write_reg(struct pca9685 *pca, unsigned int reg, unsigned int val) +static int pca9685_write_reg(struct pwm_chip *chip, unsigned int reg, unsigned int val) { - struct device *dev = pca->chip.dev; + struct pca9685 *pca = to_pca(chip); + struct device *dev = chip->dev; int err; err = regmap_write(pca->regmap, reg, val); @@ -132,19 +134,19 @@ static int pca9685_write_reg(struct pca9685 *pca, unsigned int reg, unsigned int } /* Helper function to set the duty cycle ratio to duty/4096 (e.g. duty=2048 -> 50%) */ -static void pca9685_pwm_set_duty(struct pca9685 *pca, int channel, unsigned int duty) +static void pca9685_pwm_set_duty(struct pwm_chip *chip, int channel, unsigned int duty) { - struct pwm_device *pwm = &pca->chip.pwms[channel]; + struct pwm_device *pwm = &chip->pwms[channel]; unsigned int on, off; if (duty == 0) { /* Set the full OFF bit, which has the highest precedence */ - pca9685_write_reg(pca, REG_OFF_H(channel), LED_FULL); + pca9685_write_reg(chip, REG_OFF_H(channel), LED_FULL); return; } else if (duty >= PCA9685_COUNTER_RANGE) { /* Set the full ON bit and clear the full OFF bit */ - pca9685_write_reg(pca, REG_ON_H(channel), LED_FULL); - pca9685_write_reg(pca, REG_OFF_H(channel), 0); + pca9685_write_reg(chip, REG_ON_H(channel), LED_FULL); + pca9685_write_reg(chip, REG_OFF_H(channel), 0); return; } @@ -164,16 +166,16 @@ static void pca9685_pwm_set_duty(struct pca9685 *pca, int channel, unsigned int off = (on + duty) % PCA9685_COUNTER_RANGE; /* Set ON time (clears full ON bit) */ - pca9685_write_reg(pca, REG_ON_L(channel), on & 0xff); - pca9685_write_reg(pca, REG_ON_H(channel), (on >> 8) & 0xf); + pca9685_write_reg(chip, REG_ON_L(channel), on & 0xff); + pca9685_write_reg(chip, REG_ON_H(channel), (on >> 8) & 0xf); /* Set OFF time (clears full OFF bit) */ - pca9685_write_reg(pca, REG_OFF_L(channel), off & 0xff); - pca9685_write_reg(pca, REG_OFF_H(channel), (off >> 8) & 0xf); + pca9685_write_reg(chip, REG_OFF_L(channel), off & 0xff); + pca9685_write_reg(chip, REG_OFF_H(channel), (off >> 8) & 0xf); } -static unsigned int pca9685_pwm_get_duty(struct pca9685 *pca, int channel) +static unsigned int pca9685_pwm_get_duty(struct pwm_chip *chip, int channel) { - struct pwm_device *pwm = &pca->chip.pwms[channel]; + struct pwm_device *pwm = &chip->pwms[channel]; unsigned int off = 0, on = 0, val = 0; if (WARN_ON(channel >= PCA9685_MAXCHAN)) { @@ -181,25 +183,25 @@ static unsigned int pca9685_pwm_get_duty(struct pca9685 *pca, int channel) return 0; } - pca9685_read_reg(pca, LED_N_OFF_H(channel), &off); + pca9685_read_reg(chip, LED_N_OFF_H(channel), &off); if (off & LED_FULL) { /* Full OFF bit is set */ return 0; } - pca9685_read_reg(pca, LED_N_ON_H(channel), &on); + pca9685_read_reg(chip, LED_N_ON_H(channel), &on); if (on & LED_FULL) { /* Full ON bit is set */ return PCA9685_COUNTER_RANGE; } - pca9685_read_reg(pca, LED_N_OFF_L(channel), &val); + pca9685_read_reg(chip, LED_N_OFF_L(channel), &val); off = ((off & 0xf) << 8) | (val & 0xff); if (!pwm->state.usage_power) return off; /* Read ON register to calculate duty cycle of staggered output */ - if (pca9685_read_reg(pca, LED_N_ON_L(channel), &val)) { + if (pca9685_read_reg(chip, LED_N_ON_L(channel), &val)) { /* Reset val to 0 in case reading LED_N_ON_L failed */ val = 0; } @@ -247,35 +249,37 @@ static void pca9685_pwm_clear_inuse(struct pca9685 *pca, int pwm_idx) static int pca9685_pwm_gpio_request(struct gpio_chip *gpio, unsigned int offset) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); + struct pca9685 *pca = to_pca(chip); if (pca9685_pwm_test_and_set_inuse(pca, offset)) return -EBUSY; - pm_runtime_get_sync(pca->chip.dev); + pm_runtime_get_sync(chip->dev); return 0; } static int pca9685_pwm_gpio_get(struct gpio_chip *gpio, unsigned int offset) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); - return pca9685_pwm_get_duty(pca, offset) != 0; + return pca9685_pwm_get_duty(chip, offset) != 0; } static void pca9685_pwm_gpio_set(struct gpio_chip *gpio, unsigned int offset, int value) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); - pca9685_pwm_set_duty(pca, offset, value ? PCA9685_COUNTER_RANGE : 0); + pca9685_pwm_set_duty(chip, offset, value ? PCA9685_COUNTER_RANGE : 0); } static void pca9685_pwm_gpio_free(struct gpio_chip *gpio, unsigned int offset) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); + struct pca9685 *pca = to_pca(chip); - pca9685_pwm_set_duty(pca, offset, 0); - pm_runtime_put(pca->chip.dev); + pca9685_pwm_set_duty(chip, offset, 0); + pm_runtime_put(chip->dev); pca9685_pwm_clear_inuse(pca, offset); } @@ -306,9 +310,10 @@ static int pca9685_pwm_gpio_direction_output(struct gpio_chip *gpio, * expose a GPIO chip here which can exclusively take over the underlying * PWM channel. */ -static int pca9685_pwm_gpio_probe(struct pca9685 *pca) +static int pca9685_pwm_gpio_probe(struct pwm_chip *chip) { - struct device *dev = pca->chip.dev; + struct pca9685 *pca = to_pca(chip); + struct device *dev = chip->dev; pca->gpio.label = dev_name(dev); pca->gpio.parent = dev; @@ -323,7 +328,7 @@ static int pca9685_pwm_gpio_probe(struct pca9685 *pca) pca->gpio.ngpio = PCA9685_MAXCHAN; pca->gpio.can_sleep = true; - return devm_gpiochip_add_data(dev, &pca->gpio, pca); + return devm_gpiochip_add_data(dev, &pca->gpio, chip); } #else static inline bool pca9685_pwm_test_and_set_inuse(struct pca9685 *pca, @@ -337,15 +342,16 @@ pca9685_pwm_clear_inuse(struct pca9685 *pca, int pwm_idx) { } -static inline int pca9685_pwm_gpio_probe(struct pca9685 *pca) +static inline int pca9685_pwm_gpio_probe(struct pwm_chip *chip) { return 0; } #endif -static void pca9685_set_sleep_mode(struct pca9685 *pca, bool enable) +static void pca9685_set_sleep_mode(struct pwm_chip *chip, bool enable) { - struct device *dev = pca->chip.dev; + struct device *dev = chip->dev; + struct pca9685 *pca = to_pca(chip); int err = regmap_update_bits(pca->regmap, PCA9685_MODE1, MODE1_SLEEP, enable ? MODE1_SLEEP : 0); if (err) { @@ -378,11 +384,11 @@ static int __pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } if (!state->enabled) { - pca9685_pwm_set_duty(pca, pwm->hwpwm, 0); + pca9685_pwm_set_duty(chip, pwm->hwpwm, 0); return 0; } - pca9685_read_reg(pca, PCA9685_PRESCALE, &val); + pca9685_read_reg(chip, PCA9685_PRESCALE, &val); if (prescale != val) { if (!pca9685_prescaler_can_change(pca, pwm->hwpwm)) { dev_err(chip->dev, @@ -397,18 +403,18 @@ static int __pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, * state is guaranteed active here. */ /* Put chip into sleep mode */ - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); /* Change the chip-wide output frequency */ - pca9685_write_reg(pca, PCA9685_PRESCALE, prescale); + pca9685_write_reg(chip, PCA9685_PRESCALE, prescale); /* Wake the chip up */ - pca9685_set_sleep_mode(pca, false); + pca9685_set_sleep_mode(chip, false); } duty = PCA9685_COUNTER_RANGE * state->duty_cycle; duty = DIV_ROUND_UP_ULL(duty, state->period); - pca9685_pwm_set_duty(pca, pwm->hwpwm, duty); + pca9685_pwm_set_duty(chip, pwm->hwpwm, duty); return 0; } @@ -434,12 +440,11 @@ static int pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, static int pca9685_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { - struct pca9685 *pca = to_pca(chip); unsigned long long duty; unsigned int val = 0; /* Calculate (chip-wide) period from prescale value */ - pca9685_read_reg(pca, PCA9685_PRESCALE, &val); + pca9685_read_reg(chip, PCA9685_PRESCALE, &val); /* * PCA9685_OSC_CLOCK_MHZ is 25, i.e. an integer divider of 1000. * The following calculation is therefore only a multiplication @@ -462,7 +467,7 @@ static int pca9685_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, } state->enabled = true; - duty = pca9685_pwm_get_duty(pca, pwm->hwpwm); + duty = pca9685_pwm_get_duty(chip, pwm->hwpwm); state->duty_cycle = DIV_ROUND_DOWN_ULL(duty * state->period, PCA9685_COUNTER_RANGE); return 0; @@ -492,7 +497,7 @@ static void pca9685_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) struct pca9685 *pca = to_pca(chip); mutex_lock(&pca->lock); - pca9685_pwm_set_duty(pca, pwm->hwpwm, 0); + pca9685_pwm_set_duty(chip, pwm->hwpwm, 0); clear_bit(pwm->hwpwm, pca->pwms_enabled); mutex_unlock(&pca->lock); @@ -516,6 +521,7 @@ static const struct regmap_config pca9685_regmap_i2c_config = { static int pca9685_pwm_probe(struct i2c_client *client) { + struct pwm_chip *chip; struct pca9685 *pca; unsigned int reg; int ret; @@ -531,12 +537,13 @@ static int pca9685_pwm_probe(struct i2c_client *client) ret); return ret; } + chip = &pca->chip; - i2c_set_clientdata(client, pca); + i2c_set_clientdata(client, chip); mutex_init(&pca->lock); - ret = pca9685_read_reg(pca, PCA9685_MODE2, ®); + ret = pca9685_read_reg(chip, PCA9685_MODE2, ®); if (ret) return ret; @@ -550,34 +557,34 @@ static int pca9685_pwm_probe(struct i2c_client *client) else reg |= MODE2_OUTDRV; - ret = pca9685_write_reg(pca, PCA9685_MODE2, reg); + ret = pca9685_write_reg(chip, PCA9685_MODE2, reg); if (ret) return ret; /* Disable all LED ALLCALL and SUBx addresses to avoid bus collisions */ - pca9685_read_reg(pca, PCA9685_MODE1, ®); + pca9685_read_reg(chip, PCA9685_MODE1, ®); reg &= ~(MODE1_ALLCALL | MODE1_SUB1 | MODE1_SUB2 | MODE1_SUB3); - pca9685_write_reg(pca, PCA9685_MODE1, reg); + pca9685_write_reg(chip, PCA9685_MODE1, reg); /* Reset OFF/ON registers to POR default */ - pca9685_write_reg(pca, PCA9685_ALL_LED_OFF_L, 0); - pca9685_write_reg(pca, PCA9685_ALL_LED_OFF_H, LED_FULL); - pca9685_write_reg(pca, PCA9685_ALL_LED_ON_L, 0); - pca9685_write_reg(pca, PCA9685_ALL_LED_ON_H, LED_FULL); + pca9685_write_reg(chip, PCA9685_ALL_LED_OFF_L, 0); + pca9685_write_reg(chip, PCA9685_ALL_LED_OFF_H, LED_FULL); + pca9685_write_reg(chip, PCA9685_ALL_LED_ON_L, 0); + pca9685_write_reg(chip, PCA9685_ALL_LED_ON_H, LED_FULL); - pca->chip.ops = &pca9685_pwm_ops; + chip->ops = &pca9685_pwm_ops; /* Add an extra channel for ALL_LED */ - pca->chip.npwm = PCA9685_MAXCHAN + 1; + chip->npwm = PCA9685_MAXCHAN + 1; - pca->chip.dev = &client->dev; + chip->dev = &client->dev; - ret = pwmchip_add(&pca->chip); + ret = pwmchip_add(chip); if (ret < 0) return ret; - ret = pca9685_pwm_gpio_probe(pca); + ret = pca9685_pwm_gpio_probe(chip); if (ret < 0) { - pwmchip_remove(&pca->chip); + pwmchip_remove(chip); return ret; } @@ -588,11 +595,11 @@ static int pca9685_pwm_probe(struct i2c_client *client) * Although the chip comes out of power-up in the sleep state, * we force it to sleep in case it was woken up before */ - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); pm_runtime_set_suspended(&client->dev); } else { /* Wake the chip up if runtime PM is disabled */ - pca9685_set_sleep_mode(pca, false); + pca9685_set_sleep_mode(chip, false); } return 0; @@ -600,13 +607,13 @@ static int pca9685_pwm_probe(struct i2c_client *client) static void pca9685_pwm_remove(struct i2c_client *client) { - struct pca9685 *pca = i2c_get_clientdata(client); + struct pwm_chip *chip = i2c_get_clientdata(client); - pwmchip_remove(&pca->chip); + pwmchip_remove(chip); if (!pm_runtime_enabled(&client->dev)) { /* Put chip in sleep state if runtime PM is disabled */ - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); } pm_runtime_disable(&client->dev); @@ -615,18 +622,18 @@ static void pca9685_pwm_remove(struct i2c_client *client) static int __maybe_unused pca9685_pwm_runtime_suspend(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); - struct pca9685 *pca = i2c_get_clientdata(client); + struct pwm_chip *chip = i2c_get_clientdata(client); - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); return 0; } static int __maybe_unused pca9685_pwm_runtime_resume(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); - struct pca9685 *pca = i2c_get_clientdata(client); + struct pwm_chip *chip = i2c_get_clientdata(client); - pca9685_set_sleep_mode(pca, false); + pca9685_set_sleep_mode(chip, false); return 0; } From 20666a774d17094207799b3d97044f60804d29aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:09 +0100 Subject: [PATCH 0512/1038] pwm: pca9685: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/14b6ec2bdaa7050b9f49a69f35edef731fd84c25.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-pca9685.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index d7542517fa49..e976a5ca1e7b 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -110,7 +110,7 @@ static bool pca9685_prescaler_can_change(struct pca9685 *pca, int channel) static int pca9685_read_reg(struct pwm_chip *chip, unsigned int reg, unsigned int *val) { struct pca9685 *pca = to_pca(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int err; err = regmap_read(pca->regmap, reg, val); @@ -123,7 +123,7 @@ static int pca9685_read_reg(struct pwm_chip *chip, unsigned int reg, unsigned in static int pca9685_write_reg(struct pwm_chip *chip, unsigned int reg, unsigned int val) { struct pca9685 *pca = to_pca(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int err; err = regmap_write(pca->regmap, reg, val); @@ -254,7 +254,7 @@ static int pca9685_pwm_gpio_request(struct gpio_chip *gpio, unsigned int offset) if (pca9685_pwm_test_and_set_inuse(pca, offset)) return -EBUSY; - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); return 0; } @@ -279,7 +279,7 @@ static void pca9685_pwm_gpio_free(struct gpio_chip *gpio, unsigned int offset) struct pca9685 *pca = to_pca(chip); pca9685_pwm_set_duty(chip, offset, 0); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); pca9685_pwm_clear_inuse(pca, offset); } @@ -313,7 +313,7 @@ static int pca9685_pwm_gpio_direction_output(struct gpio_chip *gpio, static int pca9685_pwm_gpio_probe(struct pwm_chip *chip) { struct pca9685 *pca = to_pca(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); pca->gpio.label = dev_name(dev); pca->gpio.parent = dev; @@ -350,7 +350,7 @@ static inline int pca9685_pwm_gpio_probe(struct pwm_chip *chip) static void pca9685_set_sleep_mode(struct pwm_chip *chip, bool enable) { - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); struct pca9685 *pca = to_pca(chip); int err = regmap_update_bits(pca->regmap, PCA9685_MODE1, MODE1_SLEEP, enable ? MODE1_SLEEP : 0); @@ -379,7 +379,7 @@ static int __pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, prescale = DIV_ROUND_CLOSEST_ULL(PCA9685_OSC_CLOCK_MHZ * state->period, PCA9685_COUNTER_RANGE * 1000) - 1; if (prescale < PCA9685_PRESCALE_MIN || prescale > PCA9685_PRESCALE_MAX) { - dev_err(chip->dev, "pwm not changed: period out of bounds!\n"); + dev_err(pwmchip_parent(chip), "pwm not changed: period out of bounds!\n"); return -EINVAL; } @@ -391,7 +391,7 @@ static int __pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, pca9685_read_reg(chip, PCA9685_PRESCALE, &val); if (prescale != val) { if (!pca9685_prescaler_can_change(pca, pwm->hwpwm)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "pwm not changed: periods of enabled pwms must match!\n"); return -EBUSY; } @@ -487,7 +487,7 @@ static int pca9685_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) mutex_unlock(&pca->lock); } - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); return 0; } @@ -501,7 +501,7 @@ static void pca9685_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) clear_bit(pwm->hwpwm, pca->pwms_enabled); mutex_unlock(&pca->lock); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); pca9685_pwm_clear_inuse(pca, pwm->hwpwm); } From 37e0f5800b096015e1757c130f0dde9e658d5daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:10 +0100 Subject: [PATCH 0513/1038] pwm: pca9685: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-pca9685 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/4cb65adc92414a901dd3c0b8412de075196a3c54.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-pca9685.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index e976a5ca1e7b..c5da2a6ed846 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -76,7 +76,6 @@ #define REG_OFF_L(C) ((C) >= PCA9685_MAXCHAN ? PCA9685_ALL_LED_OFF_L : LED_N_OFF_L((C))) struct pca9685 { - struct pwm_chip chip; struct regmap *regmap; struct mutex lock; DECLARE_BITMAP(pwms_enabled, PCA9685_MAXCHAN + 1); @@ -88,7 +87,7 @@ struct pca9685 { static inline struct pca9685 *to_pca(struct pwm_chip *chip) { - return container_of(chip, struct pca9685, chip); + return pwmchip_get_drvdata(chip); } /* This function is supposed to be called with the lock mutex held */ @@ -526,9 +525,11 @@ static int pca9685_pwm_probe(struct i2c_client *client) unsigned int reg; int ret; - pca = devm_kzalloc(&client->dev, sizeof(*pca), GFP_KERNEL); - if (!pca) - return -ENOMEM; + /* Add an extra channel for ALL_LED */ + chip = devm_pwmchip_alloc(&client->dev, PCA9685_MAXCHAN + 1, sizeof(*pca)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pca = to_pca(chip); pca->regmap = devm_regmap_init_i2c(client, &pca9685_regmap_i2c_config); if (IS_ERR(pca->regmap)) { @@ -537,7 +538,6 @@ static int pca9685_pwm_probe(struct i2c_client *client) ret); return ret; } - chip = &pca->chip; i2c_set_clientdata(client, chip); @@ -573,10 +573,6 @@ static int pca9685_pwm_probe(struct i2c_client *client) pca9685_write_reg(chip, PCA9685_ALL_LED_ON_H, LED_FULL); chip->ops = &pca9685_pwm_ops; - /* Add an extra channel for ALL_LED */ - chip->npwm = PCA9685_MAXCHAN + 1; - - chip->dev = &client->dev; ret = pwmchip_add(chip); if (ret < 0) From 8aebd7225d5ba6eb740d7e358651ab4a87c601ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:11 +0100 Subject: [PATCH 0514/1038] pwm: pxa: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-pxa driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/bee8ae436a9dfb8539637cbb7269a208f787a62a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-pxa.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c index 61b74fa1d348..bb7bb48b2e6d 100644 --- a/drivers/pwm/pwm-pxa.c +++ b/drivers/pwm/pwm-pxa.c @@ -49,7 +49,6 @@ MODULE_DEVICE_TABLE(platform, pwm_id_table); #define PWMDCR_FD (1 << 10) struct pxa_pwm_chip { - struct pwm_chip chip; struct device *dev; struct clk *clk; @@ -58,7 +57,7 @@ struct pxa_pwm_chip { static inline struct pxa_pwm_chip *to_pxa_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct pxa_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -159,6 +158,7 @@ MODULE_DEVICE_TABLE(of, pwm_of_match); static int pwm_probe(struct platform_device *pdev) { const struct platform_device_id *id = platform_get_device_id(pdev); + struct pwm_chip *chip; struct pxa_pwm_chip *pc; int ret = 0; @@ -168,26 +168,27 @@ static int pwm_probe(struct platform_device *pdev) if (id == NULL) return -EINVAL; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (pc == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, + (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1, + sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_pxa_pwm_chip(chip); pc->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(pc->clk)) return PTR_ERR(pc->clk); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &pxa_pwm_ops; - pc->chip.npwm = (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1; + chip->ops = &pxa_pwm_ops; if (IS_ENABLED(CONFIG_OF)) - pc->chip.of_xlate = of_pwm_single_xlate; + chip->of_xlate = of_pwm_single_xlate; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) return PTR_ERR(pc->mmio_base); - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); return ret; From fd20a3e1fc7d461189df2c7523f569a3ee0f8142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:12 +0100 Subject: [PATCH 0515/1038] pwm: raspberrypi-poe: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/6758bcddb150f2c4353469937f1a68ecb87465ed.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-raspberrypi-poe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-raspberrypi-poe.c b/drivers/pwm/pwm-raspberrypi-poe.c index 1ad814fdec6b..f10e8a624c51 100644 --- a/drivers/pwm/pwm-raspberrypi-poe.c +++ b/drivers/pwm/pwm-raspberrypi-poe.c @@ -122,7 +122,7 @@ static int raspberrypi_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = raspberrypi_pwm_set_property(rpipwm->firmware, RPI_PWM_CUR_DUTY_REG, duty_cycle); if (ret) { - dev_err(chip->dev, "Failed to set duty cycle: %pe\n", + dev_err(pwmchip_parent(chip), "Failed to set duty cycle: %pe\n", ERR_PTR(ret)); return ret; } From 05aa788c06cf61b126b49f7fdce5e96546be8453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:13 +0100 Subject: [PATCH 0516/1038] pwm: raspberrypi-poe: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-raspberrypi-poe driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/4654a6438ab618574afc3bb73bf402df7c254ee2.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-raspberrypi-poe.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-raspberrypi-poe.c b/drivers/pwm/pwm-raspberrypi-poe.c index f10e8a624c51..8921e7ea2cea 100644 --- a/drivers/pwm/pwm-raspberrypi-poe.c +++ b/drivers/pwm/pwm-raspberrypi-poe.c @@ -27,7 +27,6 @@ struct raspberrypi_pwm { struct rpi_firmware *firmware; - struct pwm_chip chip; unsigned int duty_cycle; }; @@ -40,7 +39,7 @@ struct raspberrypi_pwm_prop { static inline struct raspberrypi_pwm *raspberrypi_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct raspberrypi_pwm, chip); + return pwmchip_get_drvdata(chip); } static int raspberrypi_pwm_set_property(struct rpi_firmware *firmware, @@ -142,6 +141,7 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev) struct device_node *firmware_node; struct device *dev = &pdev->dev; struct rpi_firmware *firmware; + struct pwm_chip *chip; struct raspberrypi_pwm *rpipwm; int ret; @@ -157,14 +157,14 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev) return dev_err_probe(dev, -EPROBE_DEFER, "Failed to get firmware handle\n"); - rpipwm = devm_kzalloc(&pdev->dev, sizeof(*rpipwm), GFP_KERNEL); - if (!rpipwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, RASPBERRYPI_FIRMWARE_PWM_NUM, + sizeof(*rpipwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + rpipwm = raspberrypi_pwm_from_chip(chip); rpipwm->firmware = firmware; - rpipwm->chip.dev = dev; - rpipwm->chip.ops = &raspberrypi_pwm_ops; - rpipwm->chip.npwm = RASPBERRYPI_FIRMWARE_PWM_NUM; + chip->ops = &raspberrypi_pwm_ops; ret = raspberrypi_pwm_get_property(rpipwm->firmware, RPI_PWM_CUR_DUTY_REG, &rpipwm->duty_cycle); @@ -173,7 +173,7 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev) return ret; } - return devm_pwmchip_add(dev, &rpipwm->chip); + return devm_pwmchip_add(dev, chip); } static const struct of_device_id raspberrypi_pwm_of_match[] = { From ba0c182e5f276e0cb4297938824e55ffe9a88508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:14 +0100 Subject: [PATCH 0517/1038] pwm: rcar: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/8b33a106c84517430d90ee4a8dcb878d0fc3bad5.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rcar.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c index 13269f55fccf..4838762137d1 100644 --- a/drivers/pwm/pwm-rcar.c +++ b/drivers/pwm/pwm-rcar.c @@ -132,12 +132,12 @@ static int rcar_pwm_set_counter(struct rcar_pwm_chip *rp, int div, int duty_ns, static int rcar_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { - return pm_runtime_get_sync(chip->dev); + return pm_runtime_get_sync(pwmchip_parent(chip)); } static void rcar_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); } static int rcar_pwm_enable(struct rcar_pwm_chip *rp) From aa1b9f16772254a0d4b9154057992bf9fc9184bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:15 +0100 Subject: [PATCH 0518/1038] pwm: rcar: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct rcar_pwm_chip. Use the pwm_chip as driver data instead of the rcar_pwm_chip to get access to the pwm_chip in the .remove() callbacks without using rcar_pwm->chip. Link: https://lore.kernel.org/r/e588e3ccdd0ac1c9c3f3a79b3e52112f83f9d71c.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rcar.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c index 4838762137d1..1dfcc74efcfd 100644 --- a/drivers/pwm/pwm-rcar.c +++ b/drivers/pwm/pwm-rcar.c @@ -202,6 +202,7 @@ static const struct pwm_ops rcar_pwm_ops = { static int rcar_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct rcar_pwm_chip *rcar_pwm; int ret; @@ -219,15 +220,16 @@ static int rcar_pwm_probe(struct platform_device *pdev) return PTR_ERR(rcar_pwm->clk); } - platform_set_drvdata(pdev, rcar_pwm); + chip = &rcar_pwm->chip; + chip->dev = &pdev->dev; + chip->ops = &rcar_pwm_ops; + chip->npwm = 1; - rcar_pwm->chip.dev = &pdev->dev; - rcar_pwm->chip.ops = &rcar_pwm_ops; - rcar_pwm->chip.npwm = 1; + platform_set_drvdata(pdev, chip); pm_runtime_enable(&pdev->dev); - ret = pwmchip_add(&rcar_pwm->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "failed to register PWM chip: %d\n", ret); pm_runtime_disable(&pdev->dev); @@ -239,9 +241,9 @@ static int rcar_pwm_probe(struct platform_device *pdev) static void rcar_pwm_remove(struct platform_device *pdev) { - struct rcar_pwm_chip *rcar_pwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); - pwmchip_remove(&rcar_pwm->chip); + pwmchip_remove(chip); pm_runtime_disable(&pdev->dev); } From f0d907605d421a36af3e8a6274b2a560557e0db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:16 +0100 Subject: [PATCH 0519/1038] pwm: rcar: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-rcar driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/a37a167364366b6cbe2dd299dce02731706213b2.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rcar.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c index 1dfcc74efcfd..4cfecd88ede0 100644 --- a/drivers/pwm/pwm-rcar.c +++ b/drivers/pwm/pwm-rcar.c @@ -38,14 +38,13 @@ #define RCAR_PWMCNT_PH0_SHIFT 0 struct rcar_pwm_chip { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct rcar_pwm_chip *to_rcar_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct rcar_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void rcar_pwm_write(struct rcar_pwm_chip *rp, u32 data, @@ -206,9 +205,10 @@ static int rcar_pwm_probe(struct platform_device *pdev) struct rcar_pwm_chip *rcar_pwm; int ret; - rcar_pwm = devm_kzalloc(&pdev->dev, sizeof(*rcar_pwm), GFP_KERNEL); - if (rcar_pwm == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*rcar_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + rcar_pwm = to_rcar_pwm_chip(chip); rcar_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rcar_pwm->base)) @@ -220,10 +220,7 @@ static int rcar_pwm_probe(struct platform_device *pdev) return PTR_ERR(rcar_pwm->clk); } - chip = &rcar_pwm->chip; - chip->dev = &pdev->dev; chip->ops = &rcar_pwm_ops; - chip->npwm = 1; platform_set_drvdata(pdev, chip); From 3a284e0eae81790a13f855c9dea750197520d5ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:17 +0100 Subject: [PATCH 0520/1038] pwm: renesas-tpu: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-renesas-tpu driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_tpu_device() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/aac4a9546b0f474d991144aa925c83cfa7abe2c0.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-renesas-tpu.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-renesas-tpu.c b/drivers/pwm/pwm-renesas-tpu.c index 28265fdfc92a..2196080b4177 100644 --- a/drivers/pwm/pwm-renesas-tpu.c +++ b/drivers/pwm/pwm-renesas-tpu.c @@ -79,7 +79,6 @@ struct tpu_pwm_device { struct tpu_device { struct platform_device *pdev; - struct pwm_chip chip; spinlock_t lock; void __iomem *base; @@ -87,7 +86,10 @@ struct tpu_device { struct tpu_pwm_device tpd[TPU_CHANNEL_MAX]; }; -#define to_tpu_device(c) container_of(c, struct tpu_device, chip) +static inline struct tpu_device *to_tpu_device(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static void tpu_pwm_write(struct tpu_pwm_device *tpd, int reg_nr, u16 value) { @@ -438,12 +440,14 @@ static const struct pwm_ops tpu_pwm_ops = { static int tpu_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct tpu_device *tpu; int ret; - tpu = devm_kzalloc(&pdev->dev, sizeof(*tpu), GFP_KERNEL); - if (tpu == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, TPU_CHANNEL_MAX, sizeof(*tpu)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + tpu = to_tpu_device(chip); spin_lock_init(&tpu->lock); tpu->pdev = pdev; @@ -460,15 +464,13 @@ static int tpu_probe(struct platform_device *pdev) /* Initialize and register the device. */ platform_set_drvdata(pdev, tpu); - tpu->chip.dev = &pdev->dev; - tpu->chip.ops = &tpu_pwm_ops; - tpu->chip.npwm = TPU_CHANNEL_MAX; + chip->ops = &tpu_pwm_ops; ret = devm_pm_runtime_enable(&pdev->dev); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Failed to enable runtime PM\n"); - ret = devm_pwmchip_add(&pdev->dev, &tpu->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Failed to register PWM chip\n"); From 77e94c37db10fd46c87c3cf05019eec3478cc36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:18 +0100 Subject: [PATCH 0521/1038] pwm: rochchip: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct rockchip_pwm_chip. Use the pwm_chip as driver data instead of the rockchip_pwm_chip to get access to the pwm_chip in rockchip_pwm_remove() without using pc->chip. Link: https://lore.kernel.org/r/301657442c578cc8d330ad2b07463d59fed4bff6.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rockchip.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index a7c647e37837..c001eb6b17ae 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -296,6 +296,7 @@ MODULE_DEVICE_TABLE(of, rockchip_pwm_dt_ids); static int rockchip_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct rockchip_pwm_chip *pc; u32 enable_conf, ctrl; bool enabled; @@ -304,6 +305,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev) pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); if (!pc) return -ENOMEM; + chip = &pc->chip; pc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->base)) @@ -337,18 +339,18 @@ static int rockchip_pwm_probe(struct platform_device *pdev) goto err_clk; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pc->data = device_get_match_data(&pdev->dev); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &rockchip_pwm_ops; - pc->chip.npwm = 1; + chip->dev = &pdev->dev; + chip->ops = &rockchip_pwm_ops; + chip->npwm = 1; enable_conf = pc->data->enable_conf; ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl); enabled = (ctrl & enable_conf) == enable_conf; - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); goto err_pclk; @@ -372,9 +374,10 @@ err_clk: static void rockchip_pwm_remove(struct platform_device *pdev) { - struct rockchip_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); clk_unprepare(pc->pclk); clk_unprepare(pc->clk); From 2528428ec8491b58bca1035383026a653d1391a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:19 +0100 Subject: [PATCH 0522/1038] pwm: rockchip: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-rockchip driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/61a3f48710e3e795beb2e496c8c673bbc0f6c932.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rockchip.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index c001eb6b17ae..0fa7575dbb54 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -30,7 +30,6 @@ #define PWM_LP_DISABLE (0 << 8) struct rockchip_pwm_chip { - struct pwm_chip chip; struct clk *clk; struct clk *pclk; const struct rockchip_pwm_data *data; @@ -54,7 +53,7 @@ struct rockchip_pwm_data { static inline struct rockchip_pwm_chip *to_rockchip_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct rockchip_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int rockchip_pwm_get_state(struct pwm_chip *chip, @@ -302,10 +301,10 @@ static int rockchip_pwm_probe(struct platform_device *pdev) bool enabled; int ret, count; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; - chip = &pc->chip; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_rockchip_pwm_chip(chip); pc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->base)) @@ -342,9 +341,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev) platform_set_drvdata(pdev, chip); pc->data = device_get_match_data(&pdev->dev); - chip->dev = &pdev->dev; chip->ops = &rockchip_pwm_ops; - chip->npwm = 1; enable_conf = pc->data->enable_conf; ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl); From 27262029e9fd88e313a719258713af441cd5db1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:20 +0100 Subject: [PATCH 0523/1038] pwm: rz-mtu3: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/90e6b5d5215425f3a5ff4468ba8aedf3d387f812.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rz-mtu3.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-rz-mtu3.c b/drivers/pwm/pwm-rz-mtu3.c index bdda315b3bd3..acdc45ff90cf 100644 --- a/drivers/pwm/pwm-rz-mtu3.c +++ b/drivers/pwm/pwm-rz-mtu3.c @@ -219,7 +219,7 @@ static int rz_mtu3_pwm_enable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, u8 val; int rc; - rc = pm_runtime_resume_and_get(rz_mtu3_pwm->chip.dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(&rz_mtu3_pwm->chip)); if (rc) return rc; @@ -265,7 +265,7 @@ static void rz_mtu3_pwm_disable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, mutex_unlock(&rz_mtu3_pwm->lock); - pm_runtime_put_sync(rz_mtu3_pwm->chip.dev); + pm_runtime_put_sync(pwmchip_parent(&rz_mtu3_pwm->chip)); } static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, @@ -274,7 +274,7 @@ static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); int rc; - rc = pm_runtime_resume_and_get(chip->dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; @@ -307,7 +307,7 @@ static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, } state->polarity = PWM_POLARITY_NORMAL; - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -362,7 +362,7 @@ static int rz_mtu3_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (!pwm->state.enabled) { int rc; - rc = pm_runtime_resume_and_get(chip->dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; } @@ -399,7 +399,7 @@ static int rz_mtu3_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* If the PWM is not enabled, turn the clock off again to save power. */ if (!pwm->state.enabled) - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -465,8 +465,8 @@ static void rz_mtu3_pwm_pm_disable(void *data) struct rz_mtu3_pwm_chip *rz_mtu3_pwm = data; clk_rate_exclusive_put(rz_mtu3_pwm->clk); - pm_runtime_disable(rz_mtu3_pwm->chip.dev); - pm_runtime_set_suspended(rz_mtu3_pwm->chip.dev); + pm_runtime_disable(pwmchip_parent(&rz_mtu3_pwm->chip)); + pm_runtime_set_suspended(pwmchip_parent(&rz_mtu3_pwm->chip)); } static int rz_mtu3_pwm_probe(struct platform_device *pdev) From dfec83b4fe455224ccd9117dbe3bb1c0c79cbf66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:21 +0100 Subject: [PATCH 0524/1038] pwm: rz-mtu3: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct rz_mtu3_pwm_chip. Use the pwm_chip as driver data and devm callback parameter instead of the rz_mtu3_pwm_chip to get access to the pwm_chip in various functions without using rz_mtu3_pwm->chip. Link: https://lore.kernel.org/r/707f2aced94a235f244022f10202ce7730b1168b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rz-mtu3.c | 43 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/drivers/pwm/pwm-rz-mtu3.c b/drivers/pwm/pwm-rz-mtu3.c index acdc45ff90cf..a129ae4ef338 100644 --- a/drivers/pwm/pwm-rz-mtu3.c +++ b/drivers/pwm/pwm-rz-mtu3.c @@ -211,15 +211,15 @@ static void rz_mtu3_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) mutex_unlock(&rz_mtu3_pwm->lock); } -static int rz_mtu3_pwm_enable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, - struct pwm_device *pwm) +static int rz_mtu3_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); struct rz_mtu3_pwm_channel *priv; u32 ch; u8 val; int rc; - rc = pm_runtime_resume_and_get(pwmchip_parent(&rz_mtu3_pwm->chip)); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; @@ -243,9 +243,9 @@ static int rz_mtu3_pwm_enable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, return 0; } -static void rz_mtu3_pwm_disable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, - struct pwm_device *pwm) +static void rz_mtu3_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); struct rz_mtu3_pwm_channel *priv; u32 ch; @@ -265,7 +265,7 @@ static void rz_mtu3_pwm_disable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, mutex_unlock(&rz_mtu3_pwm->lock); - pm_runtime_put_sync(pwmchip_parent(&rz_mtu3_pwm->chip)); + pm_runtime_put_sync(pwmchip_parent(chip)); } static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, @@ -416,7 +416,7 @@ static int rz_mtu3_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!state->enabled) { if (enabled) - rz_mtu3_pwm_disable(rz_mtu3_pwm, pwm); + rz_mtu3_pwm_disable(chip, pwm); return 0; } @@ -428,7 +428,7 @@ static int rz_mtu3_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, return ret; if (!enabled) - ret = rz_mtu3_pwm_enable(rz_mtu3_pwm, pwm); + ret = rz_mtu3_pwm_enable(chip, pwm); return ret; } @@ -442,7 +442,8 @@ static const struct pwm_ops rz_mtu3_pwm_ops = { static int rz_mtu3_pwm_pm_runtime_suspend(struct device *dev) { - struct rz_mtu3_pwm_chip *rz_mtu3_pwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); clk_disable_unprepare(rz_mtu3_pwm->clk); @@ -451,7 +452,8 @@ static int rz_mtu3_pwm_pm_runtime_suspend(struct device *dev) static int rz_mtu3_pwm_pm_runtime_resume(struct device *dev) { - struct rz_mtu3_pwm_chip *rz_mtu3_pwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); return clk_prepare_enable(rz_mtu3_pwm->clk); } @@ -462,17 +464,19 @@ static DEFINE_RUNTIME_DEV_PM_OPS(rz_mtu3_pwm_pm_ops, static void rz_mtu3_pwm_pm_disable(void *data) { - struct rz_mtu3_pwm_chip *rz_mtu3_pwm = data; + struct pwm_chip *chip = data; + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); clk_rate_exclusive_put(rz_mtu3_pwm->clk); - pm_runtime_disable(pwmchip_parent(&rz_mtu3_pwm->chip)); - pm_runtime_set_suspended(pwmchip_parent(&rz_mtu3_pwm->chip)); + pm_runtime_disable(pwmchip_parent(chip)); + pm_runtime_set_suspended(pwmchip_parent(chip)); } static int rz_mtu3_pwm_probe(struct platform_device *pdev) { struct rz_mtu3 *parent_ddata = dev_get_drvdata(pdev->dev.parent); struct rz_mtu3_pwm_chip *rz_mtu3_pwm; + struct pwm_chip *chip; struct device *dev = &pdev->dev; unsigned int i, j = 0; int ret; @@ -480,6 +484,7 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) rz_mtu3_pwm = devm_kzalloc(&pdev->dev, sizeof(*rz_mtu3_pwm), GFP_KERNEL); if (!rz_mtu3_pwm) return -ENOMEM; + chip = &rz_mtu3_pwm->chip; rz_mtu3_pwm->clk = parent_ddata->clk; @@ -494,7 +499,7 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) } mutex_init(&rz_mtu3_pwm->lock); - platform_set_drvdata(pdev, rz_mtu3_pwm); + platform_set_drvdata(pdev, chip); ret = clk_prepare_enable(rz_mtu3_pwm->clk); if (ret) return dev_err_probe(dev, ret, "Clock enable failed\n"); @@ -514,15 +519,15 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - rz_mtu3_pwm->chip.dev = &pdev->dev; + chip->dev = &pdev->dev; ret = devm_add_action_or_reset(&pdev->dev, rz_mtu3_pwm_pm_disable, - rz_mtu3_pwm); + chip); if (ret < 0) return ret; - rz_mtu3_pwm->chip.ops = &rz_mtu3_pwm_ops; - rz_mtu3_pwm->chip.npwm = RZ_MTU3_MAX_PWM_CHANNELS; - ret = devm_pwmchip_add(&pdev->dev, &rz_mtu3_pwm->chip); + chip->ops = &rz_mtu3_pwm_ops; + chip->npwm = RZ_MTU3_MAX_PWM_CHANNELS; + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); From a629a77e9dfb94f98b3b450cc0309cec1b594157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:22 +0100 Subject: [PATCH 0525/1038] pwm: rz-mtu3: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-rz-mtu3 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/b05ffb9bcaf4ddb6305f8505715a5542805e3227.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-rz-mtu3.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-rz-mtu3.c b/drivers/pwm/pwm-rz-mtu3.c index a129ae4ef338..ab39bd37edaf 100644 --- a/drivers/pwm/pwm-rz-mtu3.c +++ b/drivers/pwm/pwm-rz-mtu3.c @@ -61,7 +61,6 @@ struct rz_mtu3_pwm_channel { /** * struct rz_mtu3_pwm_chip - MTU3 pwm private data * - * @chip: MTU3 pwm chip data * @clk: MTU3 module clock * @lock: Lock to prevent concurrent access for usage count * @rate: MTU3 clock rate @@ -72,7 +71,6 @@ struct rz_mtu3_pwm_channel { */ struct rz_mtu3_pwm_chip { - struct pwm_chip chip; struct clk *clk; struct mutex lock; unsigned long rate; @@ -92,7 +90,7 @@ static const struct rz_mtu3_channel_io_map channel_map[] = { static inline struct rz_mtu3_pwm_chip *to_rz_mtu3_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct rz_mtu3_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void rz_mtu3_pwm_read_tgr_registers(struct rz_mtu3_pwm_channel *priv, @@ -481,10 +479,11 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) unsigned int i, j = 0; int ret; - rz_mtu3_pwm = devm_kzalloc(&pdev->dev, sizeof(*rz_mtu3_pwm), GFP_KERNEL); - if (!rz_mtu3_pwm) - return -ENOMEM; - chip = &rz_mtu3_pwm->chip; + chip = devm_pwmchip_alloc(&pdev->dev, RZ_MTU3_MAX_PWM_CHANNELS, + sizeof(*rz_mtu3_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); rz_mtu3_pwm->clk = parent_ddata->clk; @@ -519,14 +518,12 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - chip->dev = &pdev->dev; ret = devm_add_action_or_reset(&pdev->dev, rz_mtu3_pwm_pm_disable, chip); if (ret < 0) return ret; chip->ops = &rz_mtu3_pwm_ops; - chip->npwm = RZ_MTU3_MAX_PWM_CHANNELS; ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); From 56696c1c04f43f1dcf8fba5842de6122c112b6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:23 +0100 Subject: [PATCH 0526/1038] pwm: samsung: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct samsung_pwm_chip *our_chip just obtained from it. This also has the advantage of not using struct samsung_pwm_chip::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/4e8400a42979b6be0b80dff5ae964d8e9d6232eb.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-samsung.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index 6e77302f7368..0f9a2a5e3ebc 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -326,12 +326,12 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, period = NSEC_PER_SEC / period_ns; - dev_dbg(our_chip->chip.dev, "duty_ns=%d, period_ns=%d (%u)\n", + dev_dbg(chip->dev, "duty_ns=%d, period_ns=%d (%u)\n", duty_ns, period_ns, period); tin_rate = pwm_samsung_calc_tin(our_chip, pwm->hwpwm, period); - dev_dbg(our_chip->chip.dev, "tin_rate=%lu\n", tin_rate); + dev_dbg(chip->dev, "tin_rate=%lu\n", tin_rate); tin_ns = NSEC_PER_SEC / tin_rate; tcnt = period_ns / tin_ns; @@ -355,8 +355,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, /* -1UL will give 100% duty. */ --tcmp; - dev_dbg(our_chip->chip.dev, - "tin_ns=%u, tcmp=%u/%u\n", tin_ns, tcmp, tcnt); + dev_dbg(chip->dev, "tin_ns=%u, tcmp=%u/%u\n", tin_ns, tcmp, tcnt); /* Update PWM registers. */ writel(tcnt, our_chip->base + REG_TCNTB(pwm->hwpwm)); @@ -368,7 +367,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, * shortly afer this update (before it autoreloaded the new values). */ if (oldtcmp == (u32) -1) { - dev_dbg(our_chip->chip.dev, "Forcing manual update"); + dev_dbg(chip->dev, "Forcing manual update"); pwm_samsung_manual_update(our_chip, pwm); } From 70efbf88912cf33bf04f66955a931bb8d7398c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:24 +0100 Subject: [PATCH 0527/1038] pwm: samsung: Change prototype of helpers to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given samsung_pwm_chip. To just not have to do that, rework pwm_samsung_calc_tin() and pwm_samsung_parse_dt take a pwm_chip. Also use the pwm_chip as driver data instead of the samsung_pwm_chip. Link: https://lore.kernel.org/r/33ea7d7fbf3be4a542ae8aafa213470c5831487e.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-samsung.c | 41 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index 0f9a2a5e3ebc..ecd612adf19d 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -181,9 +181,10 @@ static unsigned long pwm_samsung_get_tin_rate(struct samsung_pwm_chip *our_chip, return rate / (reg + 1); } -static unsigned long pwm_samsung_calc_tin(struct samsung_pwm_chip *our_chip, +static unsigned long pwm_samsung_calc_tin(struct pwm_chip *chip, unsigned int chan, unsigned long freq) { + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); struct samsung_pwm_variant *variant = &our_chip->variant; unsigned long rate; struct clk *clk; @@ -197,12 +198,12 @@ static unsigned long pwm_samsung_calc_tin(struct samsung_pwm_chip *our_chip, return rate; } - dev_warn(our_chip->chip.dev, + dev_warn(chip->dev, "tclk of PWM %d is inoperational, using tdiv\n", chan); } rate = pwm_samsung_get_tin_rate(our_chip, chan); - dev_dbg(our_chip->chip.dev, "tin parent at %lu\n", rate); + dev_dbg(chip->dev, "tin parent at %lu\n", rate); /* * Compare minimum PWM frequency that can be achieved with possible @@ -329,7 +330,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, dev_dbg(chip->dev, "duty_ns=%d, period_ns=%d (%u)\n", duty_ns, period_ns, period); - tin_rate = pwm_samsung_calc_tin(our_chip, pwm->hwpwm, period); + tin_rate = pwm_samsung_calc_tin(chip, pwm->hwpwm, period); dev_dbg(chip->dev, "tin_rate=%lu\n", tin_rate); @@ -506,9 +507,10 @@ static const struct of_device_id samsung_pwm_matches[] = { }; MODULE_DEVICE_TABLE(of, samsung_pwm_matches); -static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) +static int pwm_samsung_parse_dt(struct pwm_chip *chip) { - struct device_node *np = our_chip->chip.dev->of_node; + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); + struct device_node *np = chip->dev->of_node; const struct of_device_id *match; struct property *prop; const __be32 *cur; @@ -522,7 +524,7 @@ static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) of_property_for_each_u32(np, "samsung,pwm-outputs", prop, cur, val) { if (val >= SAMSUNG_PWM_NUM) { - dev_err(our_chip->chip.dev, + dev_err(chip->dev, "%s: invalid channel index in samsung,pwm-outputs property\n", __func__); continue; @@ -533,7 +535,7 @@ static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) return 0; } #else -static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) +static int pwm_samsung_parse_dt(struct pwm_chip *chip) { return -ENODEV; } @@ -543,6 +545,7 @@ static int pwm_samsung_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct samsung_pwm_chip *our_chip; + struct pwm_chip *chip; unsigned int chan; int ret; @@ -550,13 +553,14 @@ static int pwm_samsung_probe(struct platform_device *pdev) if (our_chip == NULL) return -ENOMEM; - our_chip->chip.dev = &pdev->dev; - our_chip->chip.ops = &pwm_samsung_ops; - our_chip->chip.npwm = SAMSUNG_PWM_NUM; + chip = &our_chip->chip; + chip->dev = &pdev->dev; + chip->ops = &pwm_samsung_ops; + chip->npwm = SAMSUNG_PWM_NUM; our_chip->inverter_mask = BIT(SAMSUNG_PWM_NUM) - 1; if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) { - ret = pwm_samsung_parse_dt(our_chip); + ret = pwm_samsung_parse_dt(chip); if (ret) return ret; } else { @@ -593,9 +597,9 @@ static int pwm_samsung_probe(struct platform_device *pdev) our_chip->tclk0 = devm_clk_get(&pdev->dev, "pwm-tclk0"); our_chip->tclk1 = devm_clk_get(&pdev->dev, "pwm-tclk1"); - platform_set_drvdata(pdev, our_chip); + platform_set_drvdata(pdev, chip); - ret = pwmchip_add(&our_chip->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(dev, "failed to register PWM chip\n"); clk_disable_unprepare(our_chip->base_clk); @@ -612,17 +616,18 @@ static int pwm_samsung_probe(struct platform_device *pdev) static void pwm_samsung_remove(struct platform_device *pdev) { - struct samsung_pwm_chip *our_chip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); - pwmchip_remove(&our_chip->chip); + pwmchip_remove(chip); clk_disable_unprepare(our_chip->base_clk); } static int pwm_samsung_resume(struct device *dev) { - struct samsung_pwm_chip *our_chip = dev_get_drvdata(dev); - struct pwm_chip *chip = &our_chip->chip; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); unsigned int i; for (i = 0; i < SAMSUNG_PWM_NUM; i++) { From 87b63f73e248343d527d2ea6c0b9241b89a60680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:25 +0100 Subject: [PATCH 0528/1038] pwm: samsung: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/421ee291a065aeae7a1552f22da4173313e789ad.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-samsung.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index ecd612adf19d..cabdcdf37ec9 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -198,12 +198,12 @@ static unsigned long pwm_samsung_calc_tin(struct pwm_chip *chip, return rate; } - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "tclk of PWM %d is inoperational, using tdiv\n", chan); } rate = pwm_samsung_get_tin_rate(our_chip, chan); - dev_dbg(chip->dev, "tin parent at %lu\n", rate); + dev_dbg(pwmchip_parent(chip), "tin parent at %lu\n", rate); /* * Compare minimum PWM frequency that can be achieved with possible @@ -233,7 +233,7 @@ static int pwm_samsung_request(struct pwm_chip *chip, struct pwm_device *pwm) struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); if (!(our_chip->variant.output_mask & BIT(pwm->hwpwm))) { - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "tried to request PWM channel %d without output\n", pwm->hwpwm); return -EINVAL; @@ -327,12 +327,12 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, period = NSEC_PER_SEC / period_ns; - dev_dbg(chip->dev, "duty_ns=%d, period_ns=%d (%u)\n", + dev_dbg(pwmchip_parent(chip), "duty_ns=%d, period_ns=%d (%u)\n", duty_ns, period_ns, period); tin_rate = pwm_samsung_calc_tin(chip, pwm->hwpwm, period); - dev_dbg(chip->dev, "tin_rate=%lu\n", tin_rate); + dev_dbg(pwmchip_parent(chip), "tin_rate=%lu\n", tin_rate); tin_ns = NSEC_PER_SEC / tin_rate; tcnt = period_ns / tin_ns; @@ -356,7 +356,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, /* -1UL will give 100% duty. */ --tcmp; - dev_dbg(chip->dev, "tin_ns=%u, tcmp=%u/%u\n", tin_ns, tcmp, tcnt); + dev_dbg(pwmchip_parent(chip), "tin_ns=%u, tcmp=%u/%u\n", tin_ns, tcmp, tcnt); /* Update PWM registers. */ writel(tcnt, our_chip->base + REG_TCNTB(pwm->hwpwm)); @@ -368,7 +368,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, * shortly afer this update (before it autoreloaded the new values). */ if (oldtcmp == (u32) -1) { - dev_dbg(chip->dev, "Forcing manual update"); + dev_dbg(pwmchip_parent(chip), "Forcing manual update"); pwm_samsung_manual_update(our_chip, pwm); } @@ -510,7 +510,7 @@ MODULE_DEVICE_TABLE(of, samsung_pwm_matches); static int pwm_samsung_parse_dt(struct pwm_chip *chip) { struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); - struct device_node *np = chip->dev->of_node; + struct device_node *np = pwmchip_parent(chip)->of_node; const struct of_device_id *match; struct property *prop; const __be32 *cur; @@ -524,7 +524,7 @@ static int pwm_samsung_parse_dt(struct pwm_chip *chip) of_property_for_each_u32(np, "samsung,pwm-outputs", prop, cur, val) { if (val >= SAMSUNG_PWM_NUM) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "%s: invalid channel index in samsung,pwm-outputs property\n", __func__); continue; From ef2092b5af05355e24fef83d0237cb450cd5f190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:26 +0100 Subject: [PATCH 0529/1038] pwm: samsung: Simplify by using devm functions in probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With using devm_clk_get_enabled() and devm_pwmchip_add() the .remove_new() callback can be dropped and an error path in .probe() simplified. Link: https://lore.kernel.org/r/d46ec625c14c460039082f74ec8ca3084b7c8e3a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-samsung.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index cabdcdf37ec9..81a2e66e927e 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -577,18 +577,12 @@ static int pwm_samsung_probe(struct platform_device *pdev) if (IS_ERR(our_chip->base)) return PTR_ERR(our_chip->base); - our_chip->base_clk = devm_clk_get(&pdev->dev, "timers"); + our_chip->base_clk = devm_clk_get_enabled(&pdev->dev, "timers"); if (IS_ERR(our_chip->base_clk)) { dev_err(dev, "failed to get timer base clk\n"); return PTR_ERR(our_chip->base_clk); } - ret = clk_prepare_enable(our_chip->base_clk); - if (ret < 0) { - dev_err(dev, "failed to enable base clock\n"); - return ret; - } - for (chan = 0; chan < SAMSUNG_PWM_NUM; ++chan) if (our_chip->variant.output_mask & BIT(chan)) pwm_samsung_set_invert(our_chip, chan, true); @@ -599,10 +593,9 @@ static int pwm_samsung_probe(struct platform_device *pdev) platform_set_drvdata(pdev, chip); - ret = pwmchip_add(chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(dev, "failed to register PWM chip\n"); - clk_disable_unprepare(our_chip->base_clk); return ret; } @@ -614,16 +607,6 @@ static int pwm_samsung_probe(struct platform_device *pdev) return 0; } -static void pwm_samsung_remove(struct platform_device *pdev) -{ - struct pwm_chip *chip = platform_get_drvdata(pdev); - struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); - - pwmchip_remove(chip); - - clk_disable_unprepare(our_chip->base_clk); -} - static int pwm_samsung_resume(struct device *dev) { struct pwm_chip *chip = dev_get_drvdata(dev); @@ -666,7 +649,6 @@ static struct platform_driver pwm_samsung_driver = { .of_match_table = of_match_ptr(samsung_pwm_matches), }, .probe = pwm_samsung_probe, - .remove_new = pwm_samsung_remove, }; module_platform_driver(pwm_samsung_driver); From f83c29f43d593c0d4b5132b2004ef28599fad801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:27 +0100 Subject: [PATCH 0530/1038] pwm: samsung: Simplify using dev_err_probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dev_err_probe() simplifies and improves the idiom "emit error message + return error code". Apply it accordingly. Link: https://lore.kernel.org/r/27a55fc05eb13802c9eddc600d0460e279388999.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-samsung.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index 81a2e66e927e..d3dc9b5b80d6 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -564,10 +564,9 @@ static int pwm_samsung_probe(struct platform_device *pdev) if (ret) return ret; } else { - if (!pdev->dev.platform_data) { - dev_err(&pdev->dev, "no platform data specified\n"); - return -EINVAL; - } + if (!pdev->dev.platform_data) + return dev_err_probe(&pdev->dev, -EINVAL, + "no platform data specified\n"); memcpy(&our_chip->variant, pdev->dev.platform_data, sizeof(our_chip->variant)); @@ -578,10 +577,9 @@ static int pwm_samsung_probe(struct platform_device *pdev) return PTR_ERR(our_chip->base); our_chip->base_clk = devm_clk_get_enabled(&pdev->dev, "timers"); - if (IS_ERR(our_chip->base_clk)) { - dev_err(dev, "failed to get timer base clk\n"); - return PTR_ERR(our_chip->base_clk); - } + if (IS_ERR(our_chip->base_clk)) + return dev_err_probe(dev, PTR_ERR(our_chip->base_clk), + "failed to get timer base clk\n"); for (chan = 0; chan < SAMSUNG_PWM_NUM; ++chan) if (our_chip->variant.output_mask & BIT(chan)) @@ -594,10 +592,8 @@ static int pwm_samsung_probe(struct platform_device *pdev) platform_set_drvdata(pdev, chip); ret = devm_pwmchip_add(&pdev->dev, chip); - if (ret < 0) { - dev_err(dev, "failed to register PWM chip\n"); - return ret; - } + if (ret < 0) + return dev_err_probe(dev, ret, "failed to register PWM chip\n"); dev_dbg(dev, "base_clk at %lu, tclk0 at %lu, tclk1 at %lu\n", clk_get_rate(our_chip->base_clk), From 6727fdf748971ac42cd9357af9e308ddcca2818b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:28 +0100 Subject: [PATCH 0531/1038] pwm: samsung: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-samsung driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/f188e68bdea8d5a24277a10c8c9a6350a9c246ac.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-samsung.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index d3dc9b5b80d6..efb60c9f0cb3 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -69,7 +69,6 @@ struct samsung_pwm_channel { /** * struct samsung_pwm_chip - private data of PWM chip - * @chip: generic PWM chip * @variant: local copy of hardware variant data * @inverter_mask: inverter status for all channels - one bit per channel * @disabled_mask: disabled status for all channels - one bit per channel @@ -80,7 +79,6 @@ struct samsung_pwm_channel { * @channel: per channel driver data */ struct samsung_pwm_chip { - struct pwm_chip chip; struct samsung_pwm_variant variant; u8 inverter_mask; u8 disabled_mask; @@ -110,7 +108,7 @@ static DEFINE_SPINLOCK(samsung_pwm_lock); static inline struct samsung_pwm_chip *to_samsung_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct samsung_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline unsigned int to_tcon_channel(unsigned int channel) @@ -549,14 +547,12 @@ static int pwm_samsung_probe(struct platform_device *pdev) unsigned int chan; int ret; - our_chip = devm_kzalloc(&pdev->dev, sizeof(*our_chip), GFP_KERNEL); - if (our_chip == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, SAMSUNG_PWM_NUM, sizeof(*our_chip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + our_chip = to_samsung_pwm_chip(chip); - chip = &our_chip->chip; - chip->dev = &pdev->dev; chip->ops = &pwm_samsung_ops; - chip->npwm = SAMSUNG_PWM_NUM; our_chip->inverter_mask = BIT(SAMSUNG_PWM_NUM) - 1; if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) { From c63f0bbbc99dcf1391f1d8a6e9c2aa66ba143d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:29 +0100 Subject: [PATCH 0532/1038] pwm: sifive: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct pwm_sifive_ddata *ddata just obtained from it. This also has the advantage of not using struct pwm_sifive_ddata::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/78da5070d3ff5767da6f1c053d069c1e25229375.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sifive.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c index 089e50bdbbf0..cf00f902fc30 100644 --- a/drivers/pwm/pwm-sifive.c +++ b/drivers/pwm/pwm-sifive.c @@ -185,7 +185,7 @@ static int pwm_sifive_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!enabled) { ret = clk_enable(ddata->clk); if (ret) { - dev_err(ddata->chip.dev, "Enable clk failed\n"); + dev_err(chip->dev, "Enable clk failed\n"); return ret; } } From bb472da2148f39b4c235ff7db75ce96a7d33fb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:30 +0100 Subject: [PATCH 0533/1038] pwm: sifive: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct pwm_sifive_ddata. Use the pwm_chip as driver data instead of the pwm_sifive_ddata to get access to the pwm_chip in pwm_sifive_remove() without using ddata->chip. In the clock rate notifier it's not possible to get the pwm_chip without adding a pointer to this to struct pwm_sifive_ddata. Instead of that add a parent device pointer which is all that is needed there. Link: https://lore.kernel.org/r/b7b7985f4dc746f6a36c5048d428c4ed0a2d42dc.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sifive.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c index cf00f902fc30..ed40a687f909 100644 --- a/drivers/pwm/pwm-sifive.c +++ b/drivers/pwm/pwm-sifive.c @@ -42,6 +42,7 @@ struct pwm_sifive_ddata { struct pwm_chip chip; + struct device *parent; struct mutex lock; /* lock to protect user_count and approx_period */ struct notifier_block notifier; struct clk *clk; @@ -102,7 +103,7 @@ static void pwm_sifive_update_clock(struct pwm_sifive_ddata *ddata, /* As scale <= 15 the shift operation cannot overflow. */ num = (unsigned long long)NSEC_PER_SEC << (PWM_SIFIVE_CMPWIDTH + scale); ddata->real_period = div64_ul(num, rate); - dev_dbg(ddata->chip.dev, + dev_dbg(ddata->parent, "New real_period = %u ns\n", ddata->real_period); } @@ -236,7 +237,7 @@ static int pwm_sifive_probe(struct platform_device *pdev) mutex_init(&ddata->lock); chip = &ddata->chip; - chip->dev = dev; + chip->dev = ddata->parent = dev; chip->ops = &pwm_sifive_ops; chip->npwm = 4; @@ -296,7 +297,7 @@ static int pwm_sifive_probe(struct platform_device *pdev) goto unregister_clk; } - platform_set_drvdata(pdev, ddata); + platform_set_drvdata(pdev, chip); dev_dbg(dev, "SiFive PWM chip registered %d PWMs\n", chip->npwm); return 0; @@ -314,15 +315,16 @@ disable_clk: static void pwm_sifive_remove(struct platform_device *dev) { - struct pwm_sifive_ddata *ddata = platform_get_drvdata(dev); + struct pwm_chip *chip = platform_get_drvdata(dev); + struct pwm_sifive_ddata *ddata = pwm_sifive_chip_to_ddata(chip); struct pwm_device *pwm; int ch; - pwmchip_remove(&ddata->chip); + pwmchip_remove(chip); clk_notifier_unregister(ddata->clk, &ddata->notifier); - for (ch = 0; ch < ddata->chip.npwm; ch++) { - pwm = &ddata->chip.pwms[ch]; + for (ch = 0; ch < chip->npwm; ch++) { + pwm = &chip->pwms[ch]; if (pwm->state.enabled) clk_disable(ddata->clk); } From c4f4af7d792c74d3cbae3a4bc16845bb3e80e2f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:31 +0100 Subject: [PATCH 0534/1038] pwm: sifive: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/170c3c61707992cbc57dce31f70c168fcd3dbe51.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sifive.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c index ed40a687f909..6abe91d2f21c 100644 --- a/drivers/pwm/pwm-sifive.c +++ b/drivers/pwm/pwm-sifive.c @@ -186,7 +186,7 @@ static int pwm_sifive_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!enabled) { ret = clk_enable(ddata->clk); if (ret) { - dev_err(chip->dev, "Enable clk failed\n"); + dev_err(pwmchip_parent(chip), "Enable clk failed\n"); return ret; } } From 554d9acae42b3ee49f4861ef036733cb7afeead9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:32 +0100 Subject: [PATCH 0535/1038] pwm: sifive: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-sifive driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/30a4cacafe2c323f2531dd1c1126f0bf0fe5e03c.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sifive.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c index 6abe91d2f21c..ed7957cc51fd 100644 --- a/drivers/pwm/pwm-sifive.c +++ b/drivers/pwm/pwm-sifive.c @@ -41,7 +41,6 @@ #define PWM_SIFIVE_DEFAULT_PERIOD 10000000 struct pwm_sifive_ddata { - struct pwm_chip chip; struct device *parent; struct mutex lock; /* lock to protect user_count and approx_period */ struct notifier_block notifier; @@ -55,7 +54,7 @@ struct pwm_sifive_ddata { static inline struct pwm_sifive_ddata *pwm_sifive_chip_to_ddata(struct pwm_chip *chip) { - return container_of(chip, struct pwm_sifive_ddata, chip); + return pwmchip_get_drvdata(chip); } static int pwm_sifive_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -231,15 +230,14 @@ static int pwm_sifive_probe(struct platform_device *pdev) u32 val; unsigned int enabled_pwms = 0, enabled_clks = 1; - ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL); - if (!ddata) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 4, sizeof(*ddata)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + ddata = pwm_sifive_chip_to_ddata(chip); + ddata->parent = dev; mutex_init(&ddata->lock); - chip = &ddata->chip; - chip->dev = ddata->parent = dev; chip->ops = &pwm_sifive_ops; - chip->npwm = 4; ddata->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ddata->regs)) From 57014f0726e36936113251a20cbbf4fa56d9aba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:33 +0100 Subject: [PATCH 0536/1038] pwm: sl28cpld: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-sl28cpld driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Reviewed-by: Michael Walle Link: https://lore.kernel.org/r/ee687086f7fc78264fac723f65ddc96cb7518c03.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sl28cpld.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-sl28cpld.c b/drivers/pwm/pwm-sl28cpld.c index 88b01ff9e460..934378d6a002 100644 --- a/drivers/pwm/pwm-sl28cpld.c +++ b/drivers/pwm/pwm-sl28cpld.c @@ -81,14 +81,13 @@ regmap_write((priv)->regmap, (priv)->offset + (reg), (val)) struct sl28cpld_pwm { - struct pwm_chip chip; struct regmap *regmap; u32 offset; }; static inline struct sl28cpld_pwm *sl28cpld_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct sl28cpld_pwm, chip); + return pwmchip_get_drvdata(chip); } static int sl28cpld_pwm_get_state(struct pwm_chip *chip, @@ -213,9 +212,10 @@ static int sl28cpld_pwm_probe(struct platform_device *pdev) return -ENODEV; } - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = sl28cpld_pwm_from_chip(chip); priv->regmap = dev_get_regmap(pdev->dev.parent, NULL); if (!priv->regmap) { @@ -231,10 +231,7 @@ static int sl28cpld_pwm_probe(struct platform_device *pdev) } /* Initialize the pwm_chip structure */ - chip = &priv->chip; - chip->dev = &pdev->dev; chip->ops = &sl28cpld_pwm_ops; - chip->npwm = 1; ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) { From 792bc12573a3784f008271f650ca0fc381f961db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:34 +0100 Subject: [PATCH 0537/1038] pwm: spear: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-spear driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/c67e3f26435a8fa047ad6887803dbcb9c1e032ee.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-spear.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-spear.c b/drivers/pwm/pwm-spear.c index ff991319feef..6c6f3b38c835 100644 --- a/drivers/pwm/pwm-spear.c +++ b/drivers/pwm/pwm-spear.c @@ -48,17 +48,15 @@ * * @mmio_base: base address of pwm chip * @clk: pointer to clk structure of pwm chip - * @chip: linux pwm chip representation */ struct spear_pwm_chip { void __iomem *mmio_base; struct clk *clk; - struct pwm_chip chip; }; static inline struct spear_pwm_chip *to_spear_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct spear_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 spear_pwm_readl(struct spear_pwm_chip *chip, unsigned int num, @@ -194,13 +192,15 @@ static const struct pwm_ops spear_pwm_ops = { static int spear_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; + struct pwm_chip *chip; struct spear_pwm_chip *pc; int ret; u32 val; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, NUM_PWM, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_spear_pwm_chip(chip); pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) @@ -211,9 +211,7 @@ static int spear_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk), "Failed to get clock\n"); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &spear_pwm_ops; - pc->chip.npwm = NUM_PWM; + chip->ops = &spear_pwm_ops; if (of_device_is_compatible(np, "st,spear1340-pwm")) { ret = clk_enable(pc->clk); @@ -232,7 +230,7 @@ static int spear_pwm_probe(struct platform_device *pdev) clk_disable(pc->clk); } - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); From 0f14aca0dca2492f8a55221f6d9a117924f0ba0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:35 +0100 Subject: [PATCH 0538/1038] pwm: sprd: Rework how the available channels are counted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To be able to convert the sprd driver to pwmchip_alloc() the number of PWM channels must be known when the driver private data is allocated. So make sprd_pwm_clk_init() not take a struct sprd_pwm_chip. On the plus side this allows to drop a member of said driver private data. Reviewed-by: Chunyan Zhang Link: https://lore.kernel.org/r/682cbbafbfc4982383d66c3871bb9e52d4e03195.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sprd.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/drivers/pwm/pwm-sprd.c b/drivers/pwm/pwm-sprd.c index 77939e161006..2e87666ad2b9 100644 --- a/drivers/pwm/pwm-sprd.c +++ b/drivers/pwm/pwm-sprd.c @@ -36,7 +36,6 @@ struct sprd_pwm_chip { void __iomem *base; struct device *dev; struct pwm_chip chip; - int num_pwms; struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; }; @@ -215,45 +214,48 @@ static const struct pwm_ops sprd_pwm_ops = { .get_state = sprd_pwm_get_state, }; -static int sprd_pwm_clk_init(struct sprd_pwm_chip *spc) +static int sprd_pwm_clk_init(struct device *dev, + struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]) { struct clk *clk_pwm; int ret, i; for (i = 0; i < SPRD_PWM_CHN_NUM; i++) { - struct sprd_pwm_chn *chn = &spc->chn[i]; int j; for (j = 0; j < SPRD_PWM_CHN_CLKS_NUM; ++j) - chn->clks[j].id = + chn[i].clks[j].id = sprd_pwm_clks[i * SPRD_PWM_CHN_CLKS_NUM + j]; - ret = devm_clk_bulk_get(spc->dev, SPRD_PWM_CHN_CLKS_NUM, - chn->clks); + ret = devm_clk_bulk_get(dev, SPRD_PWM_CHN_CLKS_NUM, + chn[i].clks); if (ret) { if (ret == -ENOENT) break; - return dev_err_probe(spc->dev, ret, + return dev_err_probe(dev, ret, "failed to get channel clocks\n"); } - clk_pwm = chn->clks[SPRD_PWM_CHN_OUTPUT_CLK].clk; - chn->clk_rate = clk_get_rate(clk_pwm); + clk_pwm = chn[i].clks[SPRD_PWM_CHN_OUTPUT_CLK].clk; + chn[i].clk_rate = clk_get_rate(clk_pwm); } if (!i) - return dev_err_probe(spc->dev, -ENODEV, "no available PWM channels\n"); + return dev_err_probe(dev, -ENODEV, "no available PWM channels\n"); - spc->num_pwms = i; - - return 0; + return i; } static int sprd_pwm_probe(struct platform_device *pdev) { struct sprd_pwm_chip *spc; - int ret; + struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; + int ret, npwm; + + npwm = sprd_pwm_clk_init(&pdev->dev, chn); + if (npwm < 0) + return npwm; spc = devm_kzalloc(&pdev->dev, sizeof(*spc), GFP_KERNEL); if (!spc) @@ -264,14 +266,11 @@ static int sprd_pwm_probe(struct platform_device *pdev) return PTR_ERR(spc->base); spc->dev = &pdev->dev; - - ret = sprd_pwm_clk_init(spc); - if (ret) - return ret; + memcpy(spc->chn, chn, sizeof(chn)); spc->chip.dev = &pdev->dev; spc->chip.ops = &sprd_pwm_ops; - spc->chip.npwm = spc->num_pwms; + spc->chip.npwm = npwm; ret = devm_pwmchip_add(&pdev->dev, &spc->chip); if (ret) From a783b2af74ca5ef78cd6559b35b987f37af3be27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 13 Feb 2024 08:47:47 +0100 Subject: [PATCH 0539/1038] dt-bindings: ata: convert MediaTek controller to the json-schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This helps validating DTS files. Signed-off-by: Rafał Miłecki Reviewed-by: Krzysztof Kozlowski Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Niklas Cassel --- .../devicetree/bindings/ata/ahci-mtk.txt | 51 ---------- .../bindings/ata/mediatek,mtk-ahci.yaml | 98 +++++++++++++++++++ 2 files changed, 98 insertions(+), 51 deletions(-) delete mode 100644 Documentation/devicetree/bindings/ata/ahci-mtk.txt create mode 100644 Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml diff --git a/Documentation/devicetree/bindings/ata/ahci-mtk.txt b/Documentation/devicetree/bindings/ata/ahci-mtk.txt deleted file mode 100644 index d2aa696b161b..000000000000 --- a/Documentation/devicetree/bindings/ata/ahci-mtk.txt +++ /dev/null @@ -1,51 +0,0 @@ -MediaTek Serial ATA controller - -Required properties: - - compatible : Must be "mediatek,-ahci", "mediatek,mtk-ahci". - When using "mediatek,mtk-ahci" compatible strings, you - need SoC specific ones in addition, one of: - - "mediatek,mt7622-ahci" - - reg : Physical base addresses and length of register sets. - - interrupts : Interrupt associated with the SATA device. - - interrupt-names : Associated name must be: "hostc". - - clocks : A list of phandle and clock specifier pairs, one for each - entry in clock-names. - - clock-names : Associated names must be: "ahb", "axi", "asic", "rbc", "pm". - - phys : A phandle and PHY specifier pair for the PHY port. - - phy-names : Associated name must be: "sata-phy". - - ports-implemented : See ./ahci-platform.txt for details. - -Optional properties: - - power-domains : A phandle and power domain specifier pair to the power - domain which is responsible for collapsing and restoring - power to the peripheral. - - resets : Must contain an entry for each entry in reset-names. - See ../reset/reset.txt for details. - - reset-names : Associated names must be: "axi", "sw", "reg". - - mediatek,phy-mode : A phandle to the system controller, used to enable - SATA function. - -Example: - - sata: sata@1a200000 { - compatible = "mediatek,mt7622-ahci", - "mediatek,mtk-ahci"; - reg = <0 0x1a200000 0 0x1100>; - interrupts = ; - interrupt-names = "hostc"; - clocks = <&pciesys CLK_SATA_AHB_EN>, - <&pciesys CLK_SATA_AXI_EN>, - <&pciesys CLK_SATA_ASIC_EN>, - <&pciesys CLK_SATA_RBC_EN>, - <&pciesys CLK_SATA_PM_EN>; - clock-names = "ahb", "axi", "asic", "rbc", "pm"; - phys = <&u3port1 PHY_TYPE_SATA>; - phy-names = "sata-phy"; - ports-implemented = <0x1>; - power-domains = <&scpsys MT7622_POWER_DOMAIN_HIF0>; - resets = <&pciesys MT7622_SATA_AXI_BUS_RST>, - <&pciesys MT7622_SATA_PHY_SW_RST>, - <&pciesys MT7622_SATA_PHY_REG_RST>; - reset-names = "axi", "sw", "reg"; - mediatek,phy-mode = <&pciesys>; - }; diff --git a/Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml b/Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml new file mode 100644 index 000000000000..a34bd2e9c352 --- /dev/null +++ b/Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ata/mediatek,mtk-ahci.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Serial ATA controller + +maintainers: + - Ryder Lee + +allOf: + - $ref: ahci-common.yaml# + +properties: + compatible: + items: + - enum: + - mediatek,mt7622-ahci + - const: mediatek,mtk-ahci + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + interrupt-names: + const: hostc + + clocks: + maxItems: 5 + + clock-names: + items: + - const: ahb + - const: axi + - const: asic + - const: rbc + - const: pm + + power-domains: + maxItems: 1 + + resets: + maxItems: 3 + + reset-names: + items: + - const: axi + - const: sw + - const: reg + + mediatek,phy-mode: + description: System controller phandle, used to enable SATA function + $ref: /schemas/types.yaml#/definitions/phandle + +required: + - reg + - interrupts + - interrupt-names + - clocks + - clock-names + - phys + - phy-names + - ports-implemented + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + #include + #include + + sata@1a200000 { + compatible = "mediatek,mt7622-ahci", "mediatek,mtk-ahci"; + reg = <0x1a200000 0x1100>; + interrupts = ; + interrupt-names = "hostc"; + clocks = <&pciesys CLK_SATA_AHB_EN>, + <&pciesys CLK_SATA_AXI_EN>, + <&pciesys CLK_SATA_ASIC_EN>, + <&pciesys CLK_SATA_RBC_EN>, + <&pciesys CLK_SATA_PM_EN>; + clock-names = "ahb", "axi", "asic", "rbc", "pm"; + phys = <&u3port1 PHY_TYPE_SATA>; + phy-names = "sata-phy"; + ports-implemented = <0x1>; + power-domains = <&scpsys MT7622_POWER_DOMAIN_HIF0>; + resets = <&pciesys MT7622_SATA_AXI_BUS_RST>, + <&pciesys MT7622_SATA_PHY_SW_RST>, + <&pciesys MT7622_SATA_PHY_REG_RST>; + reset-names = "axi", "sw", "reg"; + mediatek,phy-mode = <&pciesys>; + }; From afd2a4ae296d5e8b13aefb056c1060ddf302a199 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 15 Feb 2024 21:16:37 -0800 Subject: [PATCH 0540/1038] spi: spi-summary.rst: fix underline length The change to use "target" requires an underline to be extended by one more character to fix a documentation build warning: Documentation/spi/spi-summary.rst:274: WARNING: Title underline too short. Declare target Devices ^^^^^^^^^^^^^^^^^^^^^ Fixes: hash ("spi: Update the "master/slave" terminology in documentation") Signed-off-by: Randy Dunlap Cc: Dhruva Gole Cc: Mark Brown Cc: linux-spi@vger.kernel.org Reviewed-by: Dhruva Gole Link: https://msgid.link/r/20240216051637.10920-1-rdunlap@infradead.org Signed-off-by: Mark Brown --- Documentation/spi/spi-summary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/spi/spi-summary.rst b/Documentation/spi/spi-summary.rst index c8395cf7e4fc..546de37d6caf 100644 --- a/Documentation/spi/spi-summary.rst +++ b/Documentation/spi/spi-summary.rst @@ -271,7 +271,7 @@ an external clock, where another derives the SPI clock from current settings of some master clock. Declare target Devices -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^ The second kind of information is a list of what SPI target devices exist on the target board, often with some board-specific data needed for the From ec29a4d9b7c7329afc61e7932cb91e9b292b2b74 Mon Sep 17 00:00:00 2001 From: Danila Tikhonov Date: Sat, 17 Feb 2024 19:31:59 +0300 Subject: [PATCH 0541/1038] dt-bindings: regulator: qcom,usb-vbus-regulator: Add PM6150 compatible The VBUS register block on the PM6150 PMIC shares the design with the PM8150B one. Define corresponding compatible string, having the qcom,pm8150b-vbus-reg as a fallback. Signed-off-by: Danila Tikhonov Reviewed-by: Krzysztof Kozlowski Link: https://msgid.link/r/20240217163201.32989-2-danila@jiaxyga.com Signed-off-by: Mark Brown --- .../bindings/regulator/qcom,usb-vbus-regulator.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml index 534f87e98716..bf6336850be6 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml @@ -19,8 +19,13 @@ allOf: properties: compatible: - enum: - - qcom,pm8150b-vbus-reg + oneOf: + - enum: + - qcom,pm8150b-vbus-reg + - items: + - enum: + - qcom,pm6150-vbus-reg + - const: qcom,pm8150b-vbus-reg reg: maxItems: 1 From 09235bf3170fd4a0052af06825016d9557564f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 16 Feb 2024 08:18:30 +0100 Subject: [PATCH 0542/1038] regulator: pwm-regulator: Use dev_err_probe() for error paths in .probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One error path already used the dev_err_probe() helper. Make use of it in the other error paths, too, for consistent output. This results in a more compact source code and symbolic output of the error code. Signed-off-by: Uwe Kleine-König Link: https://msgid.link/r/20240216071829.1513748-2-u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/regulator/pwm-regulator.c | 40 +++++++++++++------------------ 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c index 60cfcd741c2a..7434b6b22d32 100644 --- a/drivers/regulator/pwm-regulator.c +++ b/drivers/regulator/pwm-regulator.c @@ -271,11 +271,10 @@ static int pwm_regulator_init_table(struct platform_device *pdev, of_find_property(np, "voltage-table", &length); if ((length < sizeof(*duty_cycle_table)) || - (length % sizeof(*duty_cycle_table))) { - dev_err(&pdev->dev, "voltage-table length(%d) is invalid\n", - length); - return -EINVAL; - } + (length % sizeof(*duty_cycle_table))) + return dev_err_probe(&pdev->dev, -EINVAL, + "voltage-table length(%d) is invalid\n", + length); duty_cycle_table = devm_kzalloc(&pdev->dev, length, GFP_KERNEL); if (!duty_cycle_table) @@ -284,10 +283,9 @@ static int pwm_regulator_init_table(struct platform_device *pdev, ret = of_property_read_u32_array(np, "voltage-table", (u32 *)duty_cycle_table, length / sizeof(u32)); - if (ret) { - dev_err(&pdev->dev, "Failed to read voltage-table: %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to read voltage-table\n"); drvdata->state = -ENOTRECOVERABLE; drvdata->duty_cycle_table = duty_cycle_table; @@ -359,10 +357,9 @@ static int pwm_regulator_probe(struct platform_device *pdev) enum gpiod_flags gpio_flags; int ret; - if (!np) { - dev_err(&pdev->dev, "Device Tree node missing\n"); - return -EINVAL; - } + if (!np) + return dev_err_probe(&pdev->dev, -EINVAL, + "Device Tree node missing\n"); drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) @@ -400,8 +397,7 @@ static int pwm_regulator_probe(struct platform_device *pdev) gpio_flags); if (IS_ERR(drvdata->enb_gpio)) { ret = PTR_ERR(drvdata->enb_gpio); - dev_err(&pdev->dev, "Failed to get enable GPIO: %d\n", ret); - return ret; + return dev_err_probe(&pdev->dev, ret, "Failed to get enable GPIO\n"); } ret = pwm_adjust_config(drvdata->pwm); @@ -409,19 +405,17 @@ static int pwm_regulator_probe(struct platform_device *pdev) return ret; ret = pwm_regulator_init_boot_on(pdev, drvdata, init_data); - if (ret) { - dev_err(&pdev->dev, "Failed to apply boot_on settings: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to apply boot_on settings\n"); regulator = devm_regulator_register(&pdev->dev, &drvdata->desc, &config); if (IS_ERR(regulator)) { ret = PTR_ERR(regulator); - dev_err(&pdev->dev, "Failed to register regulator %s: %d\n", - drvdata->desc.name, ret); - return ret; + return dev_err_probe(&pdev->dev, ret, + "Failed to register regulator %s\n", + drvdata->desc.name); } return 0; From 6037733963b8d4cd9ff9c1cabd3017ac5c1af1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 16 Feb 2024 08:18:30 +0100 Subject: [PATCH 0543/1038] regulator: pwm-regulator: Use dev_err_probe() for error paths in .probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One error path already used the dev_err_probe() helper. Make use of it in the other error paths, too, for consistent output. This results in a more compact source code and symbolic output of the error code. Signed-off-by: Uwe Kleine-König Link: https://msgid.link/r/20240216071829.1513748-2-u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/regulator/pwm-regulator.c | 40 +++++++++++++------------------ 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c index 60cfcd741c2a..7434b6b22d32 100644 --- a/drivers/regulator/pwm-regulator.c +++ b/drivers/regulator/pwm-regulator.c @@ -271,11 +271,10 @@ static int pwm_regulator_init_table(struct platform_device *pdev, of_find_property(np, "voltage-table", &length); if ((length < sizeof(*duty_cycle_table)) || - (length % sizeof(*duty_cycle_table))) { - dev_err(&pdev->dev, "voltage-table length(%d) is invalid\n", - length); - return -EINVAL; - } + (length % sizeof(*duty_cycle_table))) + return dev_err_probe(&pdev->dev, -EINVAL, + "voltage-table length(%d) is invalid\n", + length); duty_cycle_table = devm_kzalloc(&pdev->dev, length, GFP_KERNEL); if (!duty_cycle_table) @@ -284,10 +283,9 @@ static int pwm_regulator_init_table(struct platform_device *pdev, ret = of_property_read_u32_array(np, "voltage-table", (u32 *)duty_cycle_table, length / sizeof(u32)); - if (ret) { - dev_err(&pdev->dev, "Failed to read voltage-table: %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to read voltage-table\n"); drvdata->state = -ENOTRECOVERABLE; drvdata->duty_cycle_table = duty_cycle_table; @@ -359,10 +357,9 @@ static int pwm_regulator_probe(struct platform_device *pdev) enum gpiod_flags gpio_flags; int ret; - if (!np) { - dev_err(&pdev->dev, "Device Tree node missing\n"); - return -EINVAL; - } + if (!np) + return dev_err_probe(&pdev->dev, -EINVAL, + "Device Tree node missing\n"); drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) @@ -400,8 +397,7 @@ static int pwm_regulator_probe(struct platform_device *pdev) gpio_flags); if (IS_ERR(drvdata->enb_gpio)) { ret = PTR_ERR(drvdata->enb_gpio); - dev_err(&pdev->dev, "Failed to get enable GPIO: %d\n", ret); - return ret; + return dev_err_probe(&pdev->dev, ret, "Failed to get enable GPIO\n"); } ret = pwm_adjust_config(drvdata->pwm); @@ -409,19 +405,17 @@ static int pwm_regulator_probe(struct platform_device *pdev) return ret; ret = pwm_regulator_init_boot_on(pdev, drvdata, init_data); - if (ret) { - dev_err(&pdev->dev, "Failed to apply boot_on settings: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to apply boot_on settings\n"); regulator = devm_regulator_register(&pdev->dev, &drvdata->desc, &config); if (IS_ERR(regulator)) { ret = PTR_ERR(regulator); - dev_err(&pdev->dev, "Failed to register regulator %s: %d\n", - drvdata->desc.name, ret); - return ret; + return dev_err_probe(&pdev->dev, ret, + "Failed to register regulator %s\n", + drvdata->desc.name); } return 0; From 7f719a953975dc3bb970ed8f7ef8a97bad6000ab Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 16 Feb 2024 13:49:18 +0000 Subject: [PATCH 0544/1038] regulator: core: Remove redundant assignment to variable possible_uV The variable possible_uV being assigned a value that is never read, the control flow via the following goto statement takes a path where the variable is not accessed. The assignment is redundant and can be removed. Cleans up clang scan build warning: drivers/regulator/core.c:3935:3: warning: Value stored to 'possible_uV' is never read [deadcode.DeadStores] Signed-off-by: Colin Ian King Link: https://msgid.link/r/20240216134918.2108262-1-colin.i.king@gmail.com Signed-off-by: Mark Brown --- drivers/regulator/core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index a968dabb48f5..17c98c5fa45f 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -3932,7 +3932,6 @@ static int regulator_get_optimal_voltage(struct regulator_dev *rdev, if (ret < 0) return ret; - possible_uV = desired_min_uV; done = true; goto finish; From 055100d1a3b27ce154b3e3041d3cef24778821b3 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 16 Feb 2024 13:49:18 +0000 Subject: [PATCH 0545/1038] regulator: core: Remove redundant assignment to variable possible_uV The variable possible_uV being assigned a value that is never read, the control flow via the following goto statement takes a path where the variable is not accessed. The assignment is redundant and can be removed. Cleans up clang scan build warning: drivers/regulator/core.c:3935:3: warning: Value stored to 'possible_uV' is never read [deadcode.DeadStores] Signed-off-by: Colin Ian King Link: https://msgid.link/r/20240216134918.2108262-1-colin.i.king@gmail.com Signed-off-by: Mark Brown --- drivers/regulator/core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index a968dabb48f5..17c98c5fa45f 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -3932,7 +3932,6 @@ static int regulator_get_optimal_voltage(struct regulator_dev *rdev, if (ret < 0) return ret; - possible_uV = desired_min_uV; done = true; goto finish; From ee9d5895672fe723a1626a91e4957fd3cd5cfdda Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 20 Feb 2024 08:26:02 +0100 Subject: [PATCH 0546/1038] gpio: constify of_phandle_args in of_find_gpio_device_by_xlate() Pointer to the struct of_phandle_args can be made const after gpio_device_find() arguments got constified. This should be part of commit 4a92857d6e83 ("gpio: constify opaque pointer "data" in gpio_device_find()"). Signed-off-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-of.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 523b047a2803..e35a9c7da4ee 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -129,7 +129,7 @@ static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, } static struct gpio_device * -of_find_gpio_device_by_xlate(struct of_phandle_args *gpiospec) +of_find_gpio_device_by_xlate(const struct of_phandle_args *gpiospec) { return gpio_device_find(gpiospec, of_gpiochip_match_node_and_xlate); } From 47c00dcdd53bb118b03a465c75f70abf09024812 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 15 Nov 2023 13:31:21 -0500 Subject: [PATCH 0547/1038] dm thin: add braces around conditional code that spans lines Signed-off-by: Mike Snitzer --- drivers/md/dm-thin.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 07c7f9795b10..f88646f9f81f 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -453,12 +453,13 @@ static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bi cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); - if (r) + if (r) { /* * We reused an old cell; we can get rid of * the new one. */ dm_bio_prison_free_cell(pool->prison, cell_prealloc); + } return r; } @@ -707,9 +708,10 @@ static void get_bio_block_range(struct thin_c *tc, struct bio *bio, (void) sector_div(e, pool->sectors_per_block); } - if (e < b) + if (e < b) { /* Can happen if the bio is within a single block. */ e = b; + } *begin = b; *end = e; @@ -721,13 +723,14 @@ static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) sector_t bi_sector = bio->bi_iter.bi_sector; bio_set_dev(bio, tc->pool_dev->bdev); - if (block_size_is_power_of_two(pool)) + if (block_size_is_power_of_two(pool)) { bio->bi_iter.bi_sector = (block << pool->sectors_per_block_shift) | (bi_sector & (pool->sectors_per_block - 1)); - else + } else { bio->bi_iter.bi_sector = (block * pool->sectors_per_block) + sector_div(bi_sector, pool->sectors_per_block); + } } static void remap_to_origin(struct thin_c *tc, struct bio *bio) @@ -1401,9 +1404,10 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, if (pool->pf.zero_new_blocks) { if (io_overwrites_block(pool, bio)) remap_and_issue_overwrite(tc, bio, data_block, m); - else + else { ll_zero(tc, m, data_block * pool->sectors_per_block, (data_block + 1) * pool->sectors_per_block); + } } else process_prepared_mapping(m); } @@ -1416,17 +1420,17 @@ static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block, sector_t virt_block_begin = virt_block * pool->sectors_per_block; sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block; - if (virt_block_end <= tc->origin_size) + if (virt_block_end <= tc->origin_size) { schedule_copy(tc, virt_block, tc->origin_dev, virt_block, data_dest, cell, bio, pool->sectors_per_block); - else if (virt_block_begin < tc->origin_size) + } else if (virt_block_begin < tc->origin_size) { schedule_copy(tc, virt_block, tc->origin_dev, virt_block, data_dest, cell, bio, tc->origin_size - virt_block_begin); - else + } else schedule_zero(tc, virt_block, data_dest, cell, bio); } From b9d3ac48a855ed295ce660f73d3b6ba6f91fae42 Mon Sep 17 00:00:00 2001 From: Lizhe Date: Thu, 11 Jan 2024 21:00:05 -0800 Subject: [PATCH 0548/1038] dm crypt: remove redundant state settings after waking up The task status has been set to TASK_RUNNING in schedule(). No need to set again here. Signed-off-by: Lizhe Signed-off-by: Mike Snitzer --- drivers/md/dm-crypt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 59445763e55a..7388f45ada07 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1964,7 +1964,6 @@ continue_locked: schedule(); - set_current_state(TASK_RUNNING); spin_lock_irq(&cc->write_thread_lock); goto continue_locked; From 04bf7ac646ab5eb0987bbd6e87b156b856b1b03f Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:34:28 -0500 Subject: [PATCH 0549/1038] dm: add documentation for dm-vdo target This adds the admin-guide documentation for dm-vdo. vdo.rst is the guide to using dm-vdo. vdo-design is an overview of the design of dm-vdo. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- .../admin-guide/device-mapper/vdo-design.rst | 415 ++++++++++++++++++ .../admin-guide/device-mapper/vdo.rst | 388 ++++++++++++++++ 2 files changed, 803 insertions(+) create mode 100644 Documentation/admin-guide/device-mapper/vdo-design.rst create mode 100644 Documentation/admin-guide/device-mapper/vdo.rst diff --git a/Documentation/admin-guide/device-mapper/vdo-design.rst b/Documentation/admin-guide/device-mapper/vdo-design.rst new file mode 100644 index 000000000000..c82d51071c7d --- /dev/null +++ b/Documentation/admin-guide/device-mapper/vdo-design.rst @@ -0,0 +1,415 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +================ +Design of dm-vdo +================ + +The dm-vdo (virtual data optimizer) target provides inline deduplication, +compression, zero-block elimination, and thin provisioning. A dm-vdo target +can be backed by up to 256TB of storage, and can present a logical size of +up to 4PB. This target was originally developed at Permabit Technology +Corp. starting in 2009. It was first released in 2013 and has been used in +production environments ever since. It was made open-source in 2017 after +Permabit was acquired by Red Hat. This document describes the design of +dm-vdo. For usage, see vdo.rst in the same directory as this file. + +Because deduplication rates fall drastically as the block size increases, a +vdo target has a maximum block size of 4K. However, it can achieve +deduplication rates of 254:1, i.e. up to 254 copies of a given 4K block can +reference a single 4K of actual storage. It can achieve compression rates +of 14:1. All zero blocks consume no storage at all. + +Theory of Operation +=================== + +The design of dm-vdo is based on the idea that deduplication is a two-part +problem. The first is to recognize duplicate data. The second is to avoid +storing multiple copies of those duplicates. Therefore, dm-vdo has two main +parts: a deduplication index (called UDS) that is used to discover +duplicate data, and a data store with a reference counted block map that +maps from logical block addresses to the actual storage location of the +data. + +Zones and Threading +------------------- + +Due to the complexity of data optimization, the number of metadata +structures involved in a single write operation to a vdo target is larger +than most other targets. Furthermore, because vdo must operate on small +block sizes in order to achieve good deduplication rates, acceptable +performance can only be achieved through parallelism. Therefore, vdo's +design attempts to be lock-free. Most of a vdo's main data structures are +designed to be easily divided into "zones" such that any given bio must +only access a single zone of any zoned structure. Safety with minimal +locking is achieved by ensuring that during normal operation, each zone is +assigned to a specific thread, and only that thread will access the portion +of that data structure in that zone. Associated with each thread is a work +queue. Each bio is associated with a request object which can be added to a +work queue when the next phase of its operation requires access to the +structures in the zone associated with that queue. Although each structure +may be divided into zones, this division is not reflected in the on-disk +representation of each data structure. Therefore, the number of zones for +each structure, and hence the number of threads, is configured each time a +vdo target is started. + +The Deduplication Index +----------------------- + +In order to identify duplicate data efficiently, vdo was designed to +leverage some common characteristics of duplicate data. From empirical +observations, we gathered two key insights. The first is that in most data +sets with significant amounts of duplicate data, the duplicates tend to +have temporal locality. When a duplicate appears, it is more likely that +other duplicates will be detected, and that those duplicates will have been +written at about the same time. This is why the index keeps records in +temporal order. The second insight is that new data is more likely to +duplicate recent data than it is to duplicate older data and in general, +there are diminishing returns to looking further back in time. Therefore, +when the index is full, it should cull its oldest records to make space for +new ones. Another important idea behind the design of the index is that the +ultimate goal of deduplication is to reduce storage costs. Since there is a +trade-off between the storage saved and the resources expended to achieve +those savings, vdo does not attempt to find every last duplicate block. It +is sufficient to find and eliminate most of the redundancy. + +Each block of data is hashed to produce a 16-byte block name. An index +record consists of this block name paired with the presumed location of +that data on the underlying storage. However, it is not possible to +guarantee that the index is accurate. Most often, this occurs because it is +too costly to update the index when a block is over-written or discarded. +Doing so would require either storing the block name along with the blocks, +which is difficult to do efficiently in block-based storage, or reading and +rehashing each block before overwriting it. Inaccuracy can also result from +a hash collision where two different blocks have the same name. In +practice, this is extremely unlikely, but because vdo does not use a +cryptographic hash, a malicious workload can be constructed. Because of +these inaccuracies, vdo treats the locations in the index as hints, and +reads each indicated block to verify that it is indeed a duplicate before +sharing the existing block with a new one. + +Records are collected into groups called chapters. New records are added to +the newest chapter, called the open chapter. This chapter is stored in a +format optimized for adding and modifying records, and the content of the +open chapter is not finalized until it runs out of space for new records. +When the open chapter fills up, it is closed and a new open chapter is +created to collect new records. + +Closing a chapter converts it to a different format which is optimized for +writing. The records are written to a series of record pages based on the +order in which they were received. This means that records with temporal +locality should be on a small number of pages, reducing the I/O required to +retrieve them. The chapter also compiles an index that indicates which +record page contains any given name. This index means that a request for a +name can determine exactly which record page may contain that record, +without having to load the entire chapter from storage. This index uses +only a subset of the block name as its key, so it cannot guarantee that an +index entry refers to the desired block name. It can only guarantee that if +there is a record for this name, it will be on the indicated page. The +contents of a closed chapter are never altered in any way; these chapters +are read-only structures. + +Once enough records have been written to fill up all the available index +space, the oldest chapter gets removed to make space for new chapters. Any +time a request finds a matching record in the index, that record is copied +to the open chapter. This ensures that useful block names remain available +in the index, while unreferenced block names are forgotten. + +In order to find records in older chapters, the index also maintains a +higher level structure called the volume index, which contains entries +mapping a block name to the chapter containing its newest record. This +mapping is updated as records for the block name are copied or updated, +ensuring that only the newer record for a given block name is findable. +Older records for a block name can no longer be found even though they have +not been deleted. Like the chapter index, the volume index uses only a +subset of the block name as its key and can not definitively say that a +record exists for a name. It can only say which chapter would contain the +record if a record exists. The volume index is stored entirely in memory +and is saved to storage only when the vdo target is shut down. + +From the viewpoint of a request for a particular block name, first it will +look up the name in the volume index which will indicate either that the +record is new, or which chapter to search. If the latter, the request looks +up its name in the chapter index to determine if the record is new, or +which record page to search. Finally, if not new, the request will look for +its record on the indicated record page. This process may require up to two +page reads per request (one for the chapter index page and one for the +request page). However, recently accessed pages are cached so that these +page reads can be amortized across many block name requests. + +The volume index and the chapter indexes are implemented using a +memory-efficient structure called a delta index. Instead of storing the +entire key (the block name) for each entry, the entries are sorted by name +and only the difference between adjacent keys (the delta) is stored. +Because we expect the hashes to be evenly distributed, the size of the +deltas follows an exponential distribution. Because of this distribution, +the deltas are expressed in a Huffman code to take up even less space. The +entire sorted list of keys is called a delta list. This structure allows +the index to use many fewer bytes per entry than a traditional hash table, +but it is slightly more expensive to look up entries, because a request +must read every entry in a delta list to add up the deltas in order to find +the record it needs. The delta index reduces this lookup cost by splitting +its key space into many sub-lists, each starting at a fixed key value, so +that each individual list is short. + +The default index size can hold 64 million records, corresponding to about +256GB. This means that the index can identify duplicate data if the +original data was written within the last 256GB of writes. This range is +called the deduplication window. If new writes duplicate data that is older +than that, the index will not be able to find it because the records of the +older data have been removed. So when writing a 200 GB file to a vdo +target, and then immediately writing it again, the two copies will +deduplicate perfectly. Doing the same with a 500 GB file will result in no +deduplication, because the beginning of the file will no longer be in the +index by the time the second write begins (assuming there is no duplication +within the file itself). + +If you anticipate a data workload that will see useful deduplication beyond +the 256GB threshold, vdo can be configured to use a larger index with a +correspondingly larger deduplication window. (This configuration can only +be set when the target is created, not altered later. It is important to +consider the expected workload for a vdo target before configuring it.) +There are two ways to do this. + +One way is to increase the memory size of the index, which also increases +the amount of backing storage required. Doubling the size of the index will +double the length of the deduplication window at the expense of doubling +the storage size and the memory requirements. + +The other way is to enable sparse indexing. Sparse indexing increases the +deduplication window by a factor of 10, at the expense of also increasing +the storage size by a factor of 10. However with sparse indexing, the +memory requirements do not increase; the trade-off is slightly more +computation per request, and a slight decrease in the amount of +deduplication detected. (For workloads with significant amounts of +duplicate data, sparse indexing will detect 97-99% of the deduplication +that a standard, or "dense", index will detect.) + +The Data Store +-------------- + +The data store is implemented by three main data structures, all of which +work in concert to reduce or amortize metadata updates across as many data +writes as possible. + +*The Slab Depot* + +Most of the vdo volume belongs to the slab depot. The depot contains a +collection of slabs. The slabs can be up to 32GB, and are divided into +three sections. Most of a slab consists of a linear sequence of 4K blocks. +These blocks are used either to store data, or to hold portions of the +block map (see below). In addition to the data blocks, each slab has a set +of reference counters, using 1 byte for each data block. Finally each slab +has a journal. Reference updates are written to the slab journal, which is +written out one block at a time as each block fills. A copy of the +reference counters are kept in memory, and are written out a block at a +time, in oldest-dirtied-order whenever there is a need to reclaim slab +journal space. The journal is used both to ensure that the main recovery +journal (see below) can regularly free up space, and also to amortize the +cost of updating individual reference blocks. + +Each slab is independent of every other. They are assigned to "physical +zones" in round-robin fashion. If there are P physical zones, then slab n +is assigned to zone n mod P. + +The slab depot maintains an additional small data structure, the "slab +summary," which is used to reduce the amount of work needed to come back +online after a crash. The slab summary maintains an entry for each slab +indicating whether or not the slab has ever been used, whether it is clean +(i.e. all of its reference count updates have been persisted to storage), +and approximately how full it is. During recovery, each physical zone will +attempt to recover at least one slab, stopping whenever it has recovered a +slab which has some free blocks. Once each zone has some space (or has +determined that none is available), the target can resume normal operation +in a degraded mode. Read and write requests can be serviced, perhaps with +degraded performance, while the remainder of the dirty slabs are recovered. + +*The Block Map* + +The block map contains the logical to physical mapping. It can be thought +of as an array with one entry per logical address. Each entry is 5 bytes, +36 bits of which contain the physical block number which holds the data for +the given logical address. The other 4 bits are used to indicate the nature +of the mapping. Of the 16 possible states, one represents a logical address +which is unmapped (i.e. it has never been written, or has been discarded), +one represents an uncompressed block, and the other 14 states are used to +indicate that the mapped data is compressed, and which of the compression +slots in the compressed block this logical address maps to (see below). + +In practice, the array of mapping entries is divided into "block map +pages," each of which fits in a single 4K block. Each block map page +consists of a header, and 812 mapping entries (812 being the number that +fit). Each mapping page is actually a leaf of a radix tree which consists +of block map pages at each level. There are 60 radix trees which are +assigned to "logical zones" in round robin fashion (if there are L logical +zones, tree n will belong to zone n mod L). At each level, the trees are +interleaved, so logical addresses 0-811 belong to tree 0, logical addresses +812-1623 belong to tree 1, and so on. The interleaving is maintained all +the way up the forest. 60 was chosen as the number of trees because it is +highly composite and hence results in an evenly distributed number of trees +per zone for a large number of possible logical zone counts. The storage +for the 60 tree roots is allocated at format time. All other block map +pages are allocated out of the slabs as needed. This flexible allocation +avoids the need to pre-allocate space for the entire set of logical +mappings and also makes growing the logical size of a vdo easy to +implement. + +In operation, the block map maintains two caches. It is prohibitive to keep +the entire leaf level of the trees in memory, so each logical zone +maintains its own cache of leaf pages. The size of this cache is +configurable at target start time. The second cache is allocated at start +time, and is large enough to hold all the non-leaf pages of the entire +block map. This cache is populated as needed. + +*The Recovery Journal* + +The recovery journal is used to amortize updates across the block map and +slab depot. Each write request causes an entry to be made in the journal. +Entries are either "data remappings" or "block map remappings." For a data +remapping, the journal records the logical address affected and its old and +new physical mappings. For a block map remapping, the journal records the +block map page number and the physical block allocated for it (block map +pages are never reclaimed, so the old mapping is always 0). Each journal +entry and the data write it represents must be stable on disk before the +other metadata structures may be updated to reflect the operation. + +*Write Path* + +A write bio is first assigned a "data_vio," the request object which will +operate on behalf of the bio. (A "vio," from Vdo I/O, is vdo's wrapper for +bios; metadata operations use a vio, whereas submitted bios require the +much larger data_vio.) There is a fixed pool of 2048 data_vios. This number +was chosen both to bound the amount of work that is required to recover +from a crash, and because measurements indicate that increasing it consumes +more resources, but does not improve performance. These measurements have +been, and should continue to be, revisited over time. + +Once a data_vio is assigned, the following steps are performed: + +1. The bio's data is checked to see if it is all zeros, and copied if not. + +2. A lock is obtained on the logical address of the bio. Because + deduplication involves sharing blocks, it is vital to prevent + simultaneous modifications of the same block. + +3. The block map tree is traversed, loading any non-leaf pages which cover + the logical address and are not already in memory. If any of these + pages, or the leaf page which covers the logical address have not been + allocated, and the block is not all zeros, they are allocated at this + time. + +4. If the block is a zero block, skip to step 9. Otherwise, an attempt is + made to allocate a free data block. + +5. If an allocation was obtained, the bio is acknowledged. + +6. The bio's data is hashed. + +7. The data_vio obtains or joins a "hash lock," which represents all of + the bios currently writing the same data. + +8. If the hash lock does not already have a data_vio acting as its agent, + the current one assumes that role. As the agent: + + a) The index is queried. + + b) If an entry is found, the indicated block is read and compared + to the data being written. + + c) If the data matches, we have identified duplicate data. As many + of the data_vios as there are references available for that + block (including the agent) are shared. If there are more + data_vios in the hash lock than there are references available, + one of them becomes the new agent and continues as if there was + no duplicate found. + + d) If no duplicate was found, and the agent in the hash lock does + not have an allocation (fron step 3), another data_vio in the + hash lock will become the agent and write the data. If no + data_vio in the hash lock has an allocation, the data_vios will + be marked out of space and go to step 13 for cleanup. + + If there is an allocation, the data being written will be + compressed. If the compressed size is sufficiently small, the + data_vio will go to the packer where it may be placed in a bin + along with other data_vios. + + e) Once a bin is full, either because it is out of space, or + because all 14 of its slots are in use, it is written out. + + f) Each data_vio from the bin just written is the agent of some + hash lock, it will now proceed to treat the just written + compressed block as if it were a duplicate and share it with as + many other data_vios in its hash lock as possible. + + g) If the agent's data is not compressed, it will attempt to write + its data to the block it has allocated. + + h) If the data was written, this new block is treated as a + duplicate and shared as much as possible with any other + data_vios in the hash lock. + + i) If the agent wrote new data (whether compressed or not), the + index is updated to reflect the new entry. + +9. The block map is queried to determine the previous mapping of the + logical address. + +10. An entry is made in the recovery journal. The data_vio will block in + the journal until a flush has completed to ensure the data it may have + written is stable. It must also wait until its journal entry is stable + on disk. (Journal writes are all issued with the FUA bit set.) + +11. Once the recovery journal entry is stable, the data_vio makes two slab + journal entries: an increment entry for the new mapping, and a + decrement entry for the old mapping, if that mapping was non-zero. For + correctness during recovery, the slab journal entries in any given slab + journal must be in the same order as the corresponding recovery journal + entries. Therefore, if the two entries are in different zones, they are + made concurrently, and if they are in the same zone, the increment is + always made before the decrement in order to avoid underflow. After + each slab journal entry is made in memory, the associated reference + count is also updated in memory. Each of these updates will get written + out as needed. (Slab journal blocks are written out either when they + are full, or when the recovery journal requests they do so in order to + allow the recovery journal to free up space; reference count blocks are + written out whenever the associated slab journal requests they do so in + order to free up slab journal space.) + +12. Once all the reference count updates are done, the block map is updated + and the write is complete. + +13. If the data_vio did not use its allocation, it releases the allocated + block, the hash lock (if it has one), and its logical lock. The + data_vio then returns to the pool. + +*Read Path* + +Reads are much simpler than writes. After a data_vio is assigned to the +bio, and the logical lock is obtained, the block map is queried. If the +block is mapped, the appropriate physical block is read, and if necessary, +decompressed. + +*Recovery* + +When a vdo is restarted after a crash, it will attempt to recover from the +recovery journal. During the pre-resume phase of the next start, the +recovery journal is read. The increment portion of valid entries are played +into the block map. Next, valid entries are played, in order as required, +into the slab journals. Finally, each physical zone attempts to replay at +least one slab journal to reconstruct the reference counts of one slab. +Once each zone has some free space (or has determined that it has none), +the vdo comes back online, while the remainder of the slab journals are +used to reconstruct the rest of the reference counts. + +*Read-only Rebuild* + +If a vdo encounters an unrecoverable error, it will enter read-only mode. +This mode indicates that some previously acknowledged data may have been +lost. The vdo may be instructed to rebuild as best it can in order to +return to a writable state. However, this is never done automatically due +to the likelihood that data has been lost. During a read-only rebuild, the +block map is recovered from the recovery journal as before. However, the +reference counts are not rebuilt from the slab journals. Rather, the +reference counts are zeroed, and then the entire block map is traversed, +and the reference counts are updated from it. While this may lose some +data, it ensures that the block map and reference counts are consistent. diff --git a/Documentation/admin-guide/device-mapper/vdo.rst b/Documentation/admin-guide/device-mapper/vdo.rst new file mode 100644 index 000000000000..a199009979d9 --- /dev/null +++ b/Documentation/admin-guide/device-mapper/vdo.rst @@ -0,0 +1,388 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +dm-vdo +====== + +The dm-vdo (virtual data optimizer) device mapper target provides +block-level deduplication, compression, and thin provisioning. As a device +mapper target, it can add these features to the storage stack, compatible +with any file system. The vdo target does not protect against data +corruption, relying instead on integrity protection of the storage below +it. It is strongly recommended that lvm be used to manage vdo volumes. See +lvmvdo(7). + +Userspace component +=================== + +Formatting a vdo volume requires the use of the 'vdoformat' tool, available +at: + +https://github.com/dm-vdo/vdo/ + +In most cases, a vdo target will recover from a crash automatically the +next time it is started. In cases where it encountered an unrecoverable +error (either during normal operation or crash recovery) the target will +enter or come up in read-only mode. Because read-only mode is indicative of +data-loss, a positive action must be taken to bring vdo out of read-only +mode. The 'vdoforcerebuild' tool, available from the same repo, is used to +prepare a read-only vdo to exit read-only mode. After running this tool, +the vdo target will rebuild its metadata the next time it is +started. Although some data may be lost, the rebuilt vdo's metadata will be +internally consistent and the target will be writable again. + +The repo also contains additional userspace tools which can be used to +inspect a vdo target's on-disk metadata. Fortunately, these tools are +rarely needed except by dm-vdo developers. + +Target interface +================ + +Table line +---------- + +:: + + vdo V4 + + [optional arguments] + + +Required parameters: + + offset: + The offset, in sectors, at which the vdo volume's logical + space begins. + + logical device size: + The size of the device which the vdo volume will service, + in sectors. Must match the current logical size of the vdo + volume. + + storage device: + The device holding the vdo volume's data and metadata. + + storage device size: + The size of the device holding the vdo volume, as a number + of 4096-byte blocks. Must match the current size of the vdo + volume. + + minimum I/O size: + The minimum I/O size for this vdo volume to accept, in + bytes. Valid values are 512 or 4096. The recommended value + is 4096. + + block map cache size: + The size of the block map cache, as a number of 4096-byte + blocks. The minimum and recommended value is 32768 blocks. + If the logical thread count is non-zero, the cache size + must be at least 4096 blocks per logical thread. + + block map era length: + The speed with which the block map cache writes out + modified block map pages. A smaller era length is likely to + reduce the amount of time spent rebuilding, at the cost of + increased block map writes during normal operation. The + maximum and recommended value is 16380; the minimum value + is 1. + +Optional parameters: +-------------------- +Some or all of these parameters may be specified as pairs. + +Thread related parameters: + +Different categories of work are assigned to separate thread groups, and +the number of threads in each group can be configured separately. + +If , , and are all set to 0, the work handled by +all three thread types will be handled by a single thread. If any of these +values are non-zero, all of them must be non-zero. + + ack: + The number of threads used to complete bios. Since + completing a bio calls an arbitrary completion function + outside the vdo volume, threads of this type allow the vdo + volume to continue processing requests even when bio + completion is slow. The default is 1. + + bio: + The number of threads used to issue bios to the underlying + storage. Threads of this type allow the vdo volume to + continue processing requests even when bio submission is + slow. The default is 4. + + bioRotationInterval: + The number of bios to enqueue on each bio thread before + switching to the next thread. The value must be greater + than 0 and not more than 1024; the default is 64. + + cpu: + The number of threads used to do CPU-intensive work, such + as hashing and compression. The default is 1. + + hash: + The number of threads used to manage data comparisons for + deduplication based on the hash value of data blocks. The + default is 0. + + logical: + The number of threads used to manage caching and locking + based on the logical address of incoming bios. The default + is 0; the maximum is 60. + + physical: + The number of threads used to manage administration of the + underlying storage device. At format time, a slab size for + the vdo is chosen; the vdo storage device must be large + enough to have at least 1 slab per physical thread. The + default is 0; the maximum is 16. + +Miscellaneous parameters: + + maxDiscard: + The maximum size of discard bio accepted, in 4096-byte + blocks. I/O requests to a vdo volume are normally split + into 4096-byte blocks, and processed up to 2048 at a time. + However, discard requests to a vdo volume can be + automatically split to a larger size, up to + 4096-byte blocks in a single bio, and are limited to 1500 + at a time. Increasing this value may provide better overall + performance, at the cost of increased latency for the + individual discard requests. The default and minimum is 1; + the maximum is UINT_MAX / 4096. + + deduplication: + Whether deduplication is enabled. The default is 'on'; the + acceptable values are 'on' and 'off'. + + compression: + Whether compression is enabled. The default is 'off'; the + acceptable values are 'on' and 'off'. + +Device modification +------------------- + +A modified table may be loaded into a running, non-suspended vdo volume. +The modifications will take effect when the device is next resumed. The +modifiable parameters are , , +, , and . + +If the logical device size or physical device size are changed, upon +successful resume vdo will store the new values and require them on future +startups. These two parameters may not be decreased. The logical device +size may not exceed 4 PB. The physical device size must increase by at +least 32832 4096-byte blocks if at all, and must not exceed the size of the +underlying storage device. Additionally, when formatting the vdo device, a +slab size is chosen: the physical device size may never increase above the +size which provides 8192 slabs, and each increase must be large enough to +add at least one new slab. + +Examples: + +Start a previously-formatted vdo volume with 1 GB logical space and 1 GB +physical space, storing to /dev/dm-1 which has more than 1 GB of space. + +:: + + dmsetup create vdo0 --table \ + "0 2097152 vdo V4 /dev/dm-1 262144 4096 32768 16380" + +Grow the logical size to 4 GB. + +:: + + dmsetup reload vdo0 --table \ + "0 8388608 vdo V4 /dev/dm-1 262144 4096 32768 16380" + dmsetup resume vdo0 + +Grow the physical size to 2 GB. + +:: + + dmsetup reload vdo0 --table \ + "0 8388608 vdo V4 /dev/dm-1 524288 4096 32768 16380" + dmsetup resume vdo0 + +Grow the physical size by 1 GB more and increase max discard sectors. + +:: + + dmsetup reload vdo0 --table \ + "0 10485760 vdo V4 /dev/dm-1 786432 4096 32768 16380 maxDiscard 8" + dmsetup resume vdo0 + +Stop the vdo volume. + +:: + + dmsetup remove vdo0 + +Start the vdo volume again. Note that the logical and physical device sizes +must still match, but other parameters can change. + +:: + + dmsetup create vdo1 --table \ + "0 10485760 vdo V4 /dev/dm-1 786432 512 65550 5000 hash 1 logical 3 physical 2" + +Messages +-------- +All vdo devices accept messages in the form: + +:: + dmsetup message 0 + +The messages are: + + stats: + Outputs the current view of the vdo statistics. Mostly used + by the vdostats userspace program to interpret the output + buffer. + + dump: + Dumps many internal structures to the system log. This is + not always safe to run, so it should only be used to debug + a hung vdo. Optional parameters to specify structures to + dump are: + + viopool: The pool of I/O requests incoming bios + pools: A synonym of 'viopool' + vdo: Most of the structures managing on-disk data + queues: Basic information about each vdo thread + threads: A synonym of 'queues' + default: Equivalent to 'queues vdo' + all: All of the above. + + dump-on-shutdown: + Perform a default dump next time vdo shuts down. + + +Status +------ + +:: + + + + + device: + The name of the vdo volume. + + operating mode: + The current operating mode of the vdo volume; values may be + 'normal', 'recovering' (the volume has detected an issue + with its metadata and is attempting to repair itself), and + 'read-only' (an error has occurred that forces the vdo + volume to only support read operations and not writes). + + in recovery: + Whether the vdo volume is currently in recovery mode; + values may be 'recovering' or '-' which indicates not + recovering. + + index state: + The current state of the deduplication index in the vdo + volume; values may be 'closed', 'closing', 'error', + 'offline', 'online', 'opening', and 'unknown'. + + compression state: + The current state of compression in the vdo volume; values + may be 'offline' and 'online'. + + used physical blocks: + The number of physical blocks in use by the vdo volume. + + total physical blocks: + The total number of physical blocks the vdo volume may use; + the difference between this value and the + is the number of blocks the vdo + volume has left before being full. + +Memory Requirements +=================== + +A vdo target requires a fixed 38 MB of RAM along with the following amounts +that scale with the target: + +- 1.15 MB of RAM for each 1 MB of configured block map cache size. The + block map cache requires a minimum of 150 MB. +- 1.6 MB of RAM for each 1 TB of logical space. +- 268 MB of RAM for each 1 TB of physical storage managed by the volume. + +The deduplication index requires additional memory which scales with the +size of the deduplication window. For dense indexes, the index requires 1 +GB of RAM per 1 TB of window. For sparse indexes, the index requires 1 GB +of RAM per 10 TB of window. The index configuration is set when the target +is formatted and may not be modified. + +Run-time Usage +============== + +When using dm-vdo, it is important to be aware of the ways in which its +behavior differs from other storage targets. + +- There is no guarantee that over-writes of existing blocks will succeed. + Because the underlying storage may be multiply referenced, over-writing + an existing block generally requires a vdo to have a free block + available. + +- When blocks are no longer in use, sending a discard request for those + blocks lets the vdo release references for those blocks. If the vdo is + thinly provisioned, discarding unused blocks is essential to prevent the + target from running out of space. However, due to the sharing of + duplicate blocks, no discard request for any given logical block is + guaranteed to reclaim space. + +- Assuming the underlying storage properly implements flush requests, vdo + is resilient against crashes, however, unflushed writes may or may not + persist after a crash. + +- Each write to a vdo target entails a significant amount of processing. + However, much of the work is paralellizable. Therefore, vdo targets + achieve better throughput at higher I/O depths, and can support up 2048 + requests in parallel. + +Tuning +====== + +The vdo device has many options, and it can be difficult to make optimal +choices without perfect knowledge of the workload. Additionally, most +configuration options must be set when a vdo target is started, and cannot +be changed without shutting it down completely; the configuration cannot be +changed while the target is active. Ideally, tuning with simulated +workloads should be performed before deploying vdo in production +environments. + +The most important value to adjust is the block map cache size. In order to +service a request for any logical address, a vdo must load the portion of +the block map which holds the relevant mapping. These mappings are cached. +Performance will suffer when the working set does not fit in the cache. By +default, a vdo allocates 128 MB of metadata cache in RAM to support +efficient access to 100 GB of logical space at a time. It should be scaled +up proportionally for larger working sets. + +The logical and physical thread counts should also be adjusted. A logical +thread controls a disjoint section of the block map, so additional logical +threads increase parallelism and can increase throughput. Physical threads +control a disjoint section of the data blocks, so additional physical +threads can also increase throughput. However, excess threads can waste +resources and increase contention. + +Bio submission threads control the parallelism involved in sending I/O to +the underlying storage; fewer threads mean there is more opportunity to +reorder I/O requests for performance benefit, but also that each I/O +request has to wait longer before being submitted. + +Bio acknowledgment threads are used for finishing I/O requests. This is +done on dedicated threads since the amount of work required to execute a +bio's callback can not be controlled by the vdo itself. Usually one thread +is sufficient but additional threads may be beneficial, particularly when +bios have CPU-heavy callbacks. + +CPU threads are used for hashing and for compression; in workloads with +compression enabled, more threads may result in higher throughput. + +Hash threads are used to sort active requests by hash and determine whether +they should deduplicate; the most CPU intensive actions done by these +threads are comparison of 4096-byte data blocks. In most cases, a single +hash thread is sufficient. From a3957b1f3ec9ec78dd33422f3a8b4ee6781d8e3b Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:37:30 -0500 Subject: [PATCH 0550/1038] dm vdo: add the MurmurHash3 fast hashing algorithm MurmurHash3 is a fast, non-cryptographic, 128-bit hash. It was originally written by Austin Appleby and placed in the public domain. This version has been modified to produce the same result on both big endian and little endian processors, making it suitable for use in portable persistent data. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Co-developed-by: John Wiele Signed-off-by: John Wiele Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/murmurhash3.c | 175 ++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/murmurhash3.h | 15 +++ 2 files changed, 190 insertions(+) create mode 100644 drivers/md/dm-vdo/murmurhash3.c create mode 100644 drivers/md/dm-vdo/murmurhash3.h diff --git a/drivers/md/dm-vdo/murmurhash3.c b/drivers/md/dm-vdo/murmurhash3.c new file mode 100644 index 000000000000..00c9b9c05001 --- /dev/null +++ b/drivers/md/dm-vdo/murmurhash3.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: LGPL-2.1+ +/* + * MurmurHash3 was written by Austin Appleby, and is placed in the public + * domain. The author hereby disclaims copyright to this source code. + * + * Adapted by John Wiele (jwiele@redhat.com). + */ + +#include "murmurhash3.h" + +static inline u64 rotl64(u64 x, s8 r) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL64(x, y) rotl64(x, y) +static __always_inline u64 getblock64(const u64 *p, int i) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return p[i]; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return __builtin_bswap64(p[i]); +#else +#error "can't figure out byte order" +#endif +} + +static __always_inline void putblock64(u64 *p, int i, u64 value) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + p[i] = value; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + p[i] = __builtin_bswap64(value); +#else +#error "can't figure out byte order" +#endif +} + +/* Finalization mix - force all bits of a hash block to avalanche */ + +static __always_inline u64 fmix64(u64 k) +{ + k ^= k >> 33; + k *= 0xff51afd7ed558ccdLLU; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53LLU; + k ^= k >> 33; + + return k; +} + +void murmurhash3_128(const void *key, const int len, const u32 seed, void *out) +{ + const u8 *data = key; + const int nblocks = len / 16; + + u64 h1 = seed; + u64 h2 = seed; + + const u64 c1 = 0x87c37b91114253d5LLU; + const u64 c2 = 0x4cf5ad432745937fLLU; + + /* body */ + + const u64 *blocks = (const u64 *)(data); + + int i; + + for (i = 0; i < nblocks; i++) { + u64 k1 = getblock64(blocks, i * 2 + 0); + u64 k2 = getblock64(blocks, i * 2 + 1); + + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + + h1 = ROTL64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + + h2 = ROTL64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + + /* tail */ + + { + const u8 *tail = (const u8 *)(data + nblocks * 16); + + u64 k1 = 0; + u64 k2 = 0; + + switch (len & 15) { + case 15: + k2 ^= ((u64)tail[14]) << 48; + fallthrough; + case 14: + k2 ^= ((u64)tail[13]) << 40; + fallthrough; + case 13: + k2 ^= ((u64)tail[12]) << 32; + fallthrough; + case 12: + k2 ^= ((u64)tail[11]) << 24; + fallthrough; + case 11: + k2 ^= ((u64)tail[10]) << 16; + fallthrough; + case 10: + k2 ^= ((u64)tail[9]) << 8; + fallthrough; + case 9: + k2 ^= ((u64)tail[8]) << 0; + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + fallthrough; + + case 8: + k1 ^= ((u64)tail[7]) << 56; + fallthrough; + case 7: + k1 ^= ((u64)tail[6]) << 48; + fallthrough; + case 6: + k1 ^= ((u64)tail[5]) << 40; + fallthrough; + case 5: + k1 ^= ((u64)tail[4]) << 32; + fallthrough; + case 4: + k1 ^= ((u64)tail[3]) << 24; + fallthrough; + case 3: + k1 ^= ((u64)tail[2]) << 16; + fallthrough; + case 2: + k1 ^= ((u64)tail[1]) << 8; + fallthrough; + case 1: + k1 ^= ((u64)tail[0]) << 0; + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + break; + default: + break; + }; + } + /* finalization */ + + h1 ^= len; + h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + putblock64((u64 *)out, 0, h1); + putblock64((u64 *)out, 1, h2); +} diff --git a/drivers/md/dm-vdo/murmurhash3.h b/drivers/md/dm-vdo/murmurhash3.h new file mode 100644 index 000000000000..d84711ddb659 --- /dev/null +++ b/drivers/md/dm-vdo/murmurhash3.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/* + * MurmurHash3 was written by Austin Appleby, and is placed in the public + * domain. The author hereby disclaims copyright to this source code. + */ + +#ifndef _MURMURHASH3_H_ +#define _MURMURHASH3_H_ + +#include +#include + +void murmurhash3_128(const void *key, int len, u32 seed, void *out); + +#endif /* _MURMURHASH3_H_ */ From 46766d4888ffdfd2bce91a6879bd6285a92a4881 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:41:15 -0500 Subject: [PATCH 0551/1038] dm vdo: add memory allocation utilities This patch adds standardized allocation macros and memory tracking tools to track and report any allocated memory that is not freed. This makes it easier to ensure that the vdo target does not leak memory. This patch also adds utilities for controlling whether certain threads are allowed to allocate memory, since memory allocation during certain critical code sections can cause the vdo target to deadlock. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/memory-alloc.c | 442 +++++++++++++++++++++++++++++++ drivers/md/dm-vdo/memory-alloc.h | 162 +++++++++++ 2 files changed, 604 insertions(+) create mode 100644 drivers/md/dm-vdo/memory-alloc.c create mode 100644 drivers/md/dm-vdo/memory-alloc.h diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c new file mode 100644 index 000000000000..46dd5bda6825 --- /dev/null +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +/* + * UDS and VDO keep track of which threads are allowed to allocate memory freely, and which threads + * must be careful to not do a memory allocation that does an I/O request. The allocating_threads + * threads_registry and its associated methods implement this tracking. + */ +static struct thread_registry allocating_threads; + +static bool allocations_allowed(void) +{ + const bool *pointer = uds_lookup_thread(&allocating_threads); + + return (pointer != NULL) ? *pointer : false; +} + +/* + * Register the current thread as an allocating thread. + * + * An optional flag location can be supplied indicating whether, at any given point in time, the + * threads associated with that flag should be allocating storage. If the flag is false, a message + * will be logged. + * + * If no flag is supplied, the thread is always allowed to allocate storage without complaint. + * + * @new_thread: registered_thread structure to use for the current thread + * @flag_ptr: Location of the allocation-allowed flag + */ +void uds_register_allocating_thread(struct registered_thread *new_thread, + const bool *flag_ptr) +{ + if (flag_ptr == NULL) { + static const bool allocation_always_allowed = true; + + flag_ptr = &allocation_always_allowed; + } + + uds_register_thread(&allocating_threads, new_thread, flag_ptr); +} + +/* Unregister the current thread as an allocating thread. */ +void uds_unregister_allocating_thread(void) +{ + uds_unregister_thread(&allocating_threads); +} + +/* + * We track how much memory has been allocated and freed. When we unload the module, we log an + * error if we have not freed all the memory that we allocated. Nearly all memory allocation and + * freeing is done using this module. + * + * We do not use kernel functions like the kvasprintf() method, which allocate memory indirectly + * using kmalloc. + * + * These data structures and methods are used to track the amount of memory used. + */ + +/* + * We allocate very few large objects, and allocation/deallocation isn't done in a + * performance-critical stage for us, so a linked list should be fine. + */ +struct vmalloc_block_info { + void *ptr; + size_t size; + struct vmalloc_block_info *next; +}; + +static struct { + spinlock_t lock; + size_t kmalloc_blocks; + size_t kmalloc_bytes; + size_t vmalloc_blocks; + size_t vmalloc_bytes; + size_t peak_bytes; + struct vmalloc_block_info *vmalloc_list; +} memory_stats __cacheline_aligned; + +static void update_peak_usage(void) +{ + size_t total_bytes = memory_stats.kmalloc_bytes + memory_stats.vmalloc_bytes; + + if (total_bytes > memory_stats.peak_bytes) + memory_stats.peak_bytes = total_bytes; +} + +static void add_kmalloc_block(size_t size) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + memory_stats.kmalloc_blocks++; + memory_stats.kmalloc_bytes += size; + update_peak_usage(); + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +static void remove_kmalloc_block(size_t size) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + memory_stats.kmalloc_blocks--; + memory_stats.kmalloc_bytes -= size; + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +static void add_vmalloc_block(struct vmalloc_block_info *block) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + block->next = memory_stats.vmalloc_list; + memory_stats.vmalloc_list = block; + memory_stats.vmalloc_blocks++; + memory_stats.vmalloc_bytes += block->size; + update_peak_usage(); + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +static void remove_vmalloc_block(void *ptr) +{ + struct vmalloc_block_info *block; + struct vmalloc_block_info **block_ptr; + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + for (block_ptr = &memory_stats.vmalloc_list; + (block = *block_ptr) != NULL; + block_ptr = &block->next) { + if (block->ptr == ptr) { + *block_ptr = block->next; + memory_stats.vmalloc_blocks--; + memory_stats.vmalloc_bytes -= block->size; + break; + } + } + + spin_unlock_irqrestore(&memory_stats.lock, flags); + if (block != NULL) + uds_free(block); + else + uds_log_info("attempting to remove ptr %px not found in vmalloc list", ptr); +} + +/* + * Determine whether allocating a memory block should use kmalloc or __vmalloc. + * + * vmalloc can allocate any integral number of pages. + * + * kmalloc can allocate any number of bytes up to a configured limit, which defaults to 8 megabytes + * on some systems. kmalloc is especially good when memory is being both allocated and freed, and + * it does this efficiently in a multi CPU environment. + * + * kmalloc usually rounds the size of the block up to the next power of two, so when the requested + * block is bigger than PAGE_SIZE / 2 bytes, kmalloc will never give you less space than the + * corresponding vmalloc allocation. Sometimes vmalloc will use less overhead than kmalloc. + * + * The advantages of kmalloc do not help out UDS or VDO, because we allocate all our memory up + * front and do not free and reallocate it. Sometimes we have problems using kmalloc, because the + * Linux memory page map can become so fragmented that kmalloc will not give us a 32KB chunk. We + * have used vmalloc as a backup to kmalloc in the past, and a follow-up vmalloc of 32KB will work. + * But there is no strong case to be made for using kmalloc over vmalloc for these size chunks. + * + * The kmalloc/vmalloc boundary is set at 4KB, and kmalloc gets the 4KB requests. There is no + * strong reason for favoring either kmalloc or vmalloc for 4KB requests, except that tracking + * vmalloc statistics uses a linked list implementation. Using a simple test, this choice of + * boundary results in 132 vmalloc calls. Using vmalloc for requests of exactly 4KB results in an + * additional 6374 vmalloc calls, which is much less efficient for tracking. + * + * @size: How many bytes to allocate + */ +static inline bool use_kmalloc(size_t size) +{ + return size <= PAGE_SIZE; +} + +/* + * Allocate storage based on memory size and alignment, logging an error if the allocation fails. + * The memory will be zeroed. + * + * @size: The size of an object + * @align: The required alignment + * @what: What is being allocated (for error logging) + * @ptr: A pointer to hold the allocated memory + * + * Return: UDS_SUCCESS or an error code + */ +int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) +{ + /* + * The __GFP_RETRY_MAYFAIL flag means the VM implementation will retry memory reclaim + * procedures that have previously failed if there is some indication that progress has + * been made elsewhere. It can wait for other tasks to attempt high level approaches to + * freeing memory such as compaction (which removes fragmentation) and page-out. There is + * still a definite limit to the number of retries, but it is a larger limit than with + * __GFP_NORETRY. Allocations with this flag may fail, but only when there is genuinely + * little unused memory. While these allocations do not directly trigger the OOM killer, + * their failure indicates that the system is likely to need to use the OOM killer soon. + * The caller must handle failure, but can reasonably do so by failing a higher-level + * request, or completing it only in a much less efficient manner. + */ + const gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL; + unsigned int noio_flags; + bool allocations_restricted = !allocations_allowed(); + unsigned long start_time; + void *p = NULL; + + if (ptr == NULL) + return UDS_INVALID_ARGUMENT; + + if (size == 0) { + *((void **) ptr) = NULL; + return UDS_SUCCESS; + } + + if (allocations_restricted) + noio_flags = memalloc_noio_save(); + + start_time = jiffies; + if (use_kmalloc(size) && (align < PAGE_SIZE)) { + p = kmalloc(size, gfp_flags | __GFP_NOWARN); + if (p == NULL) { + /* + * It is possible for kmalloc to fail to allocate memory because there is + * no page available (see VDO-3688). A short sleep may allow the page + * reclaimer to free a page. + */ + fsleep(1000); + p = kmalloc(size, gfp_flags); + } + + if (p != NULL) + add_kmalloc_block(ksize(p)); + } else { + struct vmalloc_block_info *block; + + if (uds_allocate(1, struct vmalloc_block_info, __func__, &block) == + UDS_SUCCESS) { + /* + * It is possible for __vmalloc to fail to allocate memory because there + * are no pages available (see VDO-3661). A short sleep may allow the page + * reclaimer to free enough pages for a small allocation. + * + * For larger allocations, the page_alloc code is racing against the page + * reclaimer. If the page reclaimer can stay ahead of page_alloc, the + * __vmalloc will succeed. But if page_alloc overtakes the page reclaimer, + * the allocation fails. It is possible that more retries will succeed. + */ + for (;;) { + p = __vmalloc(size, gfp_flags | __GFP_NOWARN); + + if (p != NULL) + break; + + if (jiffies_to_msecs(jiffies - start_time) > 1000) { + /* Try one more time, logging a failure for this call. */ + p = __vmalloc(size, gfp_flags); + break; + } + + fsleep(1000); + } + + if (p == NULL) { + uds_free(block); + } else { + block->ptr = p; + block->size = PAGE_ALIGN(size); + add_vmalloc_block(block); + } + } + } + + if (allocations_restricted) + memalloc_noio_restore(noio_flags); + + if (unlikely(p == NULL)) { + uds_log_error("Could not allocate %zu bytes for %s in %u msecs", + size, what, jiffies_to_msecs(jiffies - start_time)); + return -ENOMEM; + } + + *((void **) ptr) = p; + return UDS_SUCCESS; +} + +/* + * Allocate storage based on memory size, failing immediately if the required memory is not + * available. The memory will be zeroed. + * + * @size: The size of an object. + * @what: What is being allocated (for error logging) + * + * Return: pointer to the allocated memory, or NULL if the required space is not available. + */ +void *uds_allocate_memory_nowait(size_t size, const char *what __maybe_unused) +{ + void *p = kmalloc(size, GFP_NOWAIT | __GFP_ZERO); + + if (p != NULL) + add_kmalloc_block(ksize(p)); + + return p; +} + +void uds_free(void *ptr) +{ + if (ptr != NULL) { + if (is_vmalloc_addr(ptr)) { + remove_vmalloc_block(ptr); + vfree(ptr); + } else { + remove_kmalloc_block(ksize(ptr)); + kfree(ptr); + } + } +} + +/* + * Reallocate dynamically allocated memory. There are no alignment guarantees for the reallocated + * memory. If the new memory is larger than the old memory, the new space will be zeroed. + * + * @ptr: The memory to reallocate. + * @old_size: The old size of the memory + * @size: The new size to allocate + * @what: What is being allocated (for error logging) + * @new_ptr: A pointer to hold the reallocated pointer + * + * Return: UDS_SUCCESS or an error code + */ +int uds_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what, + void *new_ptr) +{ + int result; + + if (size == 0) { + uds_free(ptr); + *(void **) new_ptr = NULL; + return UDS_SUCCESS; + } + + result = uds_allocate(size, char, what, new_ptr); + if (result != UDS_SUCCESS) + return result; + + if (ptr != NULL) { + if (old_size < size) + size = old_size; + + memcpy(*((void **) new_ptr), ptr, size); + uds_free(ptr); + } + + return UDS_SUCCESS; +} + +int uds_duplicate_string(const char *string, const char *what, char **new_string) +{ + int result; + u8 *dup; + + result = uds_allocate(strlen(string) + 1, u8, what, &dup); + if (result != UDS_SUCCESS) + return result; + + memcpy(dup, string, strlen(string) + 1); + *new_string = dup; + return UDS_SUCCESS; +} + +void uds_memory_init(void) +{ + spin_lock_init(&memory_stats.lock); + uds_initialize_thread_registry(&allocating_threads); +} + +void uds_memory_exit(void) +{ + ASSERT_LOG_ONLY(memory_stats.kmalloc_bytes == 0, + "kmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", + memory_stats.kmalloc_bytes, memory_stats.kmalloc_blocks); + ASSERT_LOG_ONLY(memory_stats.vmalloc_bytes == 0, + "vmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", + memory_stats.vmalloc_bytes, memory_stats.vmalloc_blocks); + uds_log_debug("peak usage %zd bytes", memory_stats.peak_bytes); +} + +void uds_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + *bytes_used = memory_stats.kmalloc_bytes + memory_stats.vmalloc_bytes; + *peak_bytes_used = memory_stats.peak_bytes; + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +/* + * Report stats on any allocated memory that we're tracking. Not all allocation types are + * guaranteed to be tracked in bytes (e.g., bios). + */ +void uds_report_memory_usage(void) +{ + unsigned long flags; + u64 kmalloc_blocks; + u64 kmalloc_bytes; + u64 vmalloc_blocks; + u64 vmalloc_bytes; + u64 peak_usage; + u64 total_bytes; + + spin_lock_irqsave(&memory_stats.lock, flags); + kmalloc_blocks = memory_stats.kmalloc_blocks; + kmalloc_bytes = memory_stats.kmalloc_bytes; + vmalloc_blocks = memory_stats.vmalloc_blocks; + vmalloc_bytes = memory_stats.vmalloc_bytes; + peak_usage = memory_stats.peak_bytes; + spin_unlock_irqrestore(&memory_stats.lock, flags); + total_bytes = kmalloc_bytes + vmalloc_bytes; + uds_log_info("current module memory tracking (actual allocation sizes, not requested):"); + uds_log_info(" %llu bytes in %llu kmalloc blocks", + (unsigned long long) kmalloc_bytes, + (unsigned long long) kmalloc_blocks); + uds_log_info(" %llu bytes in %llu vmalloc blocks", + (unsigned long long) vmalloc_bytes, + (unsigned long long) vmalloc_blocks); + uds_log_info(" total %llu bytes, peak usage %llu bytes", + (unsigned long long) total_bytes, (unsigned long long) peak_usage); +} diff --git a/drivers/md/dm-vdo/memory-alloc.h b/drivers/md/dm-vdo/memory-alloc.h new file mode 100644 index 000000000000..d72d597f98cf --- /dev/null +++ b/drivers/md/dm-vdo/memory-alloc.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_MEMORY_ALLOC_H +#define UDS_MEMORY_ALLOC_H + +#include +#include /* for PAGE_SIZE */ + +#include "permassert.h" +#include "thread-registry.h" + +/* Custom memory allocation function for UDS that tracks memory usage */ +int __must_check uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr); + +/* + * Allocate storage based on element counts, sizes, and alignment. + * + * This is a generalized form of our allocation use case: It allocates an array of objects, + * optionally preceded by one object of another type (i.e., a struct with trailing variable-length + * array), with the alignment indicated. + * + * Why is this inline? The sizes and alignment will always be constant, when invoked through the + * macros below, and often the count will be a compile-time constant 1 or the number of extra bytes + * will be a compile-time constant 0. So at least some of the arithmetic can usually be optimized + * away, and the run-time selection between allocation functions always can. In many cases, it'll + * boil down to just a function call with a constant size. + * + * @count: The number of objects to allocate + * @size: The size of an object + * @extra: The number of additional bytes to allocate + * @align: The required alignment + * @what: What is being allocated (for error logging) + * @ptr: A pointer to hold the allocated memory + * + * Return: UDS_SUCCESS or an error code + */ +static inline int uds_do_allocation(size_t count, size_t size, size_t extra, + size_t align, const char *what, void *ptr) +{ + size_t total_size = count * size + extra; + + /* Overflow check: */ + if ((size > 0) && (count > ((SIZE_MAX - extra) / size))) { + /* + * This is kind of a hack: We rely on the fact that SIZE_MAX would cover the entire + * address space (minus one byte) and thus the system can never allocate that much + * and the call will always fail. So we can report an overflow as "out of memory" + * by asking for "merely" SIZE_MAX bytes. + */ + total_size = SIZE_MAX; + } + + return uds_allocate_memory(total_size, align, what, ptr); +} + +/* + * Allocate one or more elements of the indicated type, logging an error if the allocation fails. + * The memory will be zeroed. + * + * @COUNT: The number of objects to allocate + * @TYPE: The type of objects to allocate. This type determines the alignment of the allocation. + * @WHAT: What is being allocated (for error logging) + * @PTR: A pointer to hold the allocated memory + * + * Return: UDS_SUCCESS or an error code + */ +#define uds_allocate(COUNT, TYPE, WHAT, PTR) \ + uds_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) + +/* + * Allocate one object of an indicated type, followed by one or more elements of a second type, + * logging an error if the allocation fails. The memory will be zeroed. + * + * @TYPE1: The type of the primary object to allocate. This type determines the alignment of the + * allocated memory. + * @COUNT: The number of objects to allocate + * @TYPE2: The type of array objects to allocate + * @WHAT: What is being allocated (for error logging) + * @PTR: A pointer to hold the allocated memory + * + * Return: UDS_SUCCESS or an error code + */ +#define uds_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ + __extension__({ \ + int _result; \ + TYPE1 **_ptr = (PTR); \ + BUILD_BUG_ON(__alignof__(TYPE1) < __alignof__(TYPE2)); \ + _result = uds_do_allocation(COUNT, \ + sizeof(TYPE2), \ + sizeof(TYPE1), \ + __alignof__(TYPE1), \ + WHAT, \ + _ptr); \ + _result; \ + }) + +/* + * Allocate memory starting on a cache line boundary, logging an error if the allocation fails. The + * memory will be zeroed. + * + * @size: The number of bytes to allocate + * @what: What is being allocated (for error logging) + * @ptr: A pointer to hold the allocated memory + * + * Return: UDS_SUCCESS or an error code + */ +static inline int __must_check uds_allocate_cache_aligned(size_t size, const char *what, void *ptr) +{ + return uds_allocate_memory(size, L1_CACHE_BYTES, what, ptr); +} + +/* + * Allocate one element of the indicated type immediately, failing if the required memory is not + * immediately available. + * + * @size: The number of bytes to allocate + * @what: What is being allocated (for error logging) + * + * Return: pointer to the memory, or NULL if the memory is not available. + */ +void *__must_check uds_allocate_memory_nowait(size_t size, const char *what); + +int __must_check uds_reallocate_memory(void *ptr, size_t old_size, size_t size, + const char *what, void *new_ptr); + +int __must_check uds_duplicate_string(const char *string, const char *what, + char **new_string); + +/* Free memory allocated with uds_allocate(). */ +void uds_free(void *ptr); + +static inline void *__uds_forget(void **ptr_ptr) +{ + void *ptr = *ptr_ptr; + + *ptr_ptr = NULL; + return ptr; +} + +/* + * Null out a pointer and return a copy to it. This macro should be used when passing a pointer to + * a function for which it is not safe to access the pointer once the function returns. + */ +#define uds_forget(ptr) __uds_forget((void **) &(ptr)) + +void uds_memory_init(void); + +void uds_memory_exit(void); + +void uds_register_allocating_thread(struct registered_thread *new_thread, + const bool *flag_ptr); + +void uds_unregister_allocating_thread(void); + +void uds_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used); + +void uds_report_memory_usage(void); + +#endif /* UDS_MEMORY_ALLOC_H */ From 03d1089e1dc7f05afb2bfc76bffc71de17e4297e Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:44:01 -0500 Subject: [PATCH 0552/1038] dm vdo: add basic logging and support utilities Add various support utilities for the vdo target and deduplication index, including logging utilities, string and time management, and index-specific error codes. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/errors.c | 312 +++++++++++++++++++++++++++++++ drivers/md/dm-vdo/errors.h | 81 ++++++++ drivers/md/dm-vdo/logger.c | 291 ++++++++++++++++++++++++++++ drivers/md/dm-vdo/logger.h | 105 +++++++++++ drivers/md/dm-vdo/permassert.c | 26 +++ drivers/md/dm-vdo/permassert.h | 46 +++++ drivers/md/dm-vdo/string-utils.c | 28 +++ drivers/md/dm-vdo/string-utils.h | 23 +++ drivers/md/dm-vdo/time-utils.h | 28 +++ 9 files changed, 940 insertions(+) create mode 100644 drivers/md/dm-vdo/errors.c create mode 100644 drivers/md/dm-vdo/errors.h create mode 100644 drivers/md/dm-vdo/logger.c create mode 100644 drivers/md/dm-vdo/logger.h create mode 100644 drivers/md/dm-vdo/permassert.c create mode 100644 drivers/md/dm-vdo/permassert.h create mode 100644 drivers/md/dm-vdo/string-utils.c create mode 100644 drivers/md/dm-vdo/string-utils.h create mode 100644 drivers/md/dm-vdo/time-utils.h diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c new file mode 100644 index 000000000000..d19504a1445e --- /dev/null +++ b/drivers/md/dm-vdo/errors.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "errors.h" + +#include +#include + +#include "logger.h" +#include "permassert.h" +#include "string-utils.h" + +static const struct error_info successful = { "UDS_SUCCESS", "Success" }; + +static const char *const message_table[] = { + [EPERM] = "Operation not permitted", + [ENOENT] = "No such file or directory", + [ESRCH] = "No such process", + [EINTR] = "Interrupted system call", + [EIO] = "Input/output error", + [ENXIO] = "No such device or address", + [E2BIG] = "Argument list too long", + [ENOEXEC] = "Exec format error", + [EBADF] = "Bad file descriptor", + [ECHILD] = "No child processes", + [EAGAIN] = "Resource temporarily unavailable", + [ENOMEM] = "Cannot allocate memory", + [EACCES] = "Permission denied", + [EFAULT] = "Bad address", + [ENOTBLK] = "Block device required", + [EBUSY] = "Device or resource busy", + [EEXIST] = "File exists", + [EXDEV] = "Invalid cross-device link", + [ENODEV] = "No such device", + [ENOTDIR] = "Not a directory", + [EISDIR] = "Is a directory", + [EINVAL] = "Invalid argument", + [ENFILE] = "Too many open files in system", + [EMFILE] = "Too many open files", + [ENOTTY] = "Inappropriate ioctl for device", + [ETXTBSY] = "Text file busy", + [EFBIG] = "File too large", + [ENOSPC] = "No space left on device", + [ESPIPE] = "Illegal seek", + [EROFS] = "Read-only file system", + [EMLINK] = "Too many links", + [EPIPE] = "Broken pipe", + [EDOM] = "Numerical argument out of domain", + [ERANGE] = "Numerical result out of range" +}; + +static const struct error_info error_list[] = { + { "UDS_OVERFLOW", "Index overflow" }, + { "UDS_INVALID_ARGUMENT", "Invalid argument passed to internal routine" }, + { "UDS_BAD_STATE", "UDS data structures are in an invalid state" }, + { "UDS_DUPLICATE_NAME", "Attempt to enter the same name into a delta index twice" }, + { "UDS_ASSERTION_FAILED", "Assertion failed" }, + { "UDS_QUEUED", "Request queued" }, + { "UDS_BUFFER_ERROR", "Buffer error" }, + { "UDS_NO_DIRECTORY", "Expected directory is missing" }, + { "UDS_ALREADY_REGISTERED", "Error range already registered" }, + { "UDS_OUT_OF_RANGE", "Cannot access data outside specified limits" }, + { "UDS_EMODULE_LOAD", "Could not load modules" }, + { "UDS_DISABLED", "UDS library context is disabled" }, + { "UDS_UNKNOWN_ERROR", "Unknown error" }, + { "UDS_UNSUPPORTED_VERSION", "Unsupported version" }, + { "UDS_CORRUPT_DATA", "Some index structure is corrupt" }, + { "UDS_NO_INDEX", "No index found" }, + { "UDS_INDEX_NOT_SAVED_CLEANLY", "Index not saved cleanly" }, +}; + +struct error_block { + const char *name; + int base; + int last; + int max; + const struct error_info *infos; +}; + +enum { + MAX_ERROR_BLOCKS = 6, +}; + +static struct { + int allocated; + int count; + struct error_block blocks[MAX_ERROR_BLOCKS]; +} registered_errors = { + .allocated = MAX_ERROR_BLOCKS, + .count = 1, + .blocks = { { + .name = "UDS Error", + .base = UDS_ERROR_CODE_BASE, + .last = UDS_ERROR_CODE_LAST, + .max = UDS_ERROR_CODE_BLOCK_END, + .infos = error_list, + } }, +}; + +/* Get the error info for an error number. Also returns the name of the error block, if known. */ +static const char *get_error_info(int errnum, const struct error_info **info_ptr) +{ + struct error_block *block; + + if (errnum == UDS_SUCCESS) { + *info_ptr = &successful; + return NULL; + } + + for (block = registered_errors.blocks; + block < registered_errors.blocks + registered_errors.count; + block++) { + if ((errnum >= block->base) && (errnum < block->last)) { + *info_ptr = block->infos + (errnum - block->base); + return block->name; + } else if ((errnum >= block->last) && (errnum < block->max)) { + *info_ptr = NULL; + return block->name; + } + } + + return NULL; +} + +/* Return a string describing a system error message. */ +static const char *system_string_error(int errnum, char *buf, size_t buflen) +{ + size_t len; + const char *error_string = NULL; + + if ((errnum > 0) && (errnum < ARRAY_SIZE(message_table))) + error_string = message_table[errnum]; + + len = ((error_string == NULL) ? + snprintf(buf, buflen, "Unknown error %d", errnum) : + snprintf(buf, buflen, "%s", error_string)); + if (len < buflen) + return buf; + + buf[0] = '\0'; + return "System error"; +} + +/* Convert an error code to a descriptive string. */ +const char *uds_string_error(int errnum, char *buf, size_t buflen) +{ + char *buffer = buf; + char *buf_end = buf + buflen; + const struct error_info *info = NULL; + const char *block_name; + + if (buf == NULL) + return NULL; + + if (errnum < 0) + errnum = -errnum; + + block_name = get_error_info(errnum, &info); + if (block_name != NULL) { + if (info != NULL) { + buffer = uds_append_to_buffer(buffer, buf_end, "%s: %s", + block_name, info->message); + } else { + buffer = uds_append_to_buffer(buffer, buf_end, "Unknown %s %d", + block_name, errnum); + } + } else if (info != NULL) { + buffer = uds_append_to_buffer(buffer, buf_end, "%s", info->message); + } else { + const char *tmp = system_string_error(errnum, buffer, buf_end - buffer); + + if (tmp != buffer) + buffer = uds_append_to_buffer(buffer, buf_end, "%s", tmp); + else + buffer += strlen(tmp); + } + + return buf; +} + +/* Convert an error code to its name. */ +const char *uds_string_error_name(int errnum, char *buf, size_t buflen) +{ + char *buffer = buf; + char *buf_end = buf + buflen; + const struct error_info *info = NULL; + const char *block_name; + + if (errnum < 0) + errnum = -errnum; + + block_name = get_error_info(errnum, &info); + if (block_name != NULL) { + if (info != NULL) { + buffer = uds_append_to_buffer(buffer, buf_end, "%s", info->name); + } else { + buffer = uds_append_to_buffer(buffer, buf_end, "%s %d", + block_name, errnum); + } + } else if (info != NULL) { + buffer = uds_append_to_buffer(buffer, buf_end, "%s", info->name); + } else { + const char *tmp; + + tmp = system_string_error(errnum, buffer, buf_end - buffer); + if (tmp != buffer) + buffer = uds_append_to_buffer(buffer, buf_end, "%s", tmp); + else + buffer += strlen(tmp); + } + + return buf; +} + +/* + * Translate an error code into a value acceptable to the kernel. The input error code may be a + * system-generated value (such as -EIO), or an internal UDS status code. The result will be a + * negative errno value. + */ +int uds_map_to_system_error(int error) +{ + char error_name[UDS_MAX_ERROR_NAME_SIZE]; + char error_message[UDS_MAX_ERROR_MESSAGE_SIZE]; + + /* 0 is success, and negative values are already system error codes. */ + if (likely(error <= 0)) + return error; + + if (error < 1024) { + /* This is probably an errno from userspace. */ + return -error; + } + + /* Internal UDS errors */ + switch (error) { + case UDS_NO_INDEX: + case UDS_CORRUPT_DATA: + /* The index doesn't exist or can't be recovered. */ + return -ENOENT; + + case UDS_INDEX_NOT_SAVED_CLEANLY: + case UDS_UNSUPPORTED_VERSION: + /* + * The index exists, but can't be loaded. Tell the client it exists so they don't + * destroy it inadvertently. + */ + return -EEXIST; + + case UDS_DISABLED: + /* The session is unusable; only returned by requests. */ + return -EIO; + + default: + /* Translate an unexpected error into something generic. */ + uds_log_info("%s: mapping status code %d (%s: %s) to -EIO", + __func__, error, + uds_string_error_name(error, error_name, + sizeof(error_name)), + uds_string_error(error, error_message, + sizeof(error_message))); + return -EIO; + } +} + +/* + * Register a block of error codes. + * + * @block_name: the name of the block of error codes + * @first_error: the first error code in the block + * @next_free_error: one past the highest possible error in the block + * @infos: a pointer to the error info array for the block + * @info_size: the size of the error info array + */ +int uds_register_error_block(const char *block_name, int first_error, + int next_free_error, const struct error_info *infos, + size_t info_size) +{ + int result; + struct error_block *block; + struct error_block new_block = { + .name = block_name, + .base = first_error, + .last = first_error + (info_size / sizeof(struct error_info)), + .max = next_free_error, + .infos = infos, + }; + + result = ASSERT(first_error < next_free_error, "well-defined error block range"); + if (result != UDS_SUCCESS) + return result; + + if (registered_errors.count == registered_errors.allocated) { + /* This should never happen. */ + return UDS_OVERFLOW; + } + + for (block = registered_errors.blocks; + block < registered_errors.blocks + registered_errors.count; + block++) { + if (strcmp(block_name, block->name) == 0) + return UDS_DUPLICATE_NAME; + + /* Ensure error ranges do not overlap. */ + if ((first_error < block->max) && (next_free_error > block->base)) + return UDS_ALREADY_REGISTERED; + } + + registered_errors.blocks[registered_errors.count++] = new_block; + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/errors.h b/drivers/md/dm-vdo/errors.h new file mode 100644 index 000000000000..0129fa37d8d8 --- /dev/null +++ b/drivers/md/dm-vdo/errors.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_ERRORS_H +#define UDS_ERRORS_H + +#include +#include + +/* Custom error codes and error-related utilities for UDS */ + +/* Valid status codes for internal UDS functions. */ +enum uds_status_codes { + /* Successful return */ + UDS_SUCCESS = 0, + + /* Used as a base value for reporting internal errors */ + UDS_ERROR_CODE_BASE = 1024, + /* Index overflow */ + UDS_OVERFLOW = UDS_ERROR_CODE_BASE + 0, + /* Invalid argument passed to internal routine */ + UDS_INVALID_ARGUMENT = UDS_ERROR_CODE_BASE + 1, + /* UDS data structures are in an invalid state */ + UDS_BAD_STATE = UDS_ERROR_CODE_BASE + 2, + /* Attempt to enter the same name into an internal structure twice */ + UDS_DUPLICATE_NAME = UDS_ERROR_CODE_BASE + 3, + /* An assertion failed */ + UDS_ASSERTION_FAILED = UDS_ERROR_CODE_BASE + 4, + /* A request has been queued for later processing (not an error) */ + UDS_QUEUED = UDS_ERROR_CODE_BASE + 5, + /* A problem has occurred with a buffer */ + UDS_BUFFER_ERROR = UDS_ERROR_CODE_BASE + 6, + /* No directory was found where one was expected */ + UDS_NO_DIRECTORY = UDS_ERROR_CODE_BASE + 7, + /* This error range has already been registered */ + UDS_ALREADY_REGISTERED = UDS_ERROR_CODE_BASE + 8, + /* Attempt to read or write data outside the valid range */ + UDS_OUT_OF_RANGE = UDS_ERROR_CODE_BASE + 9, + /* Could not load modules */ + UDS_EMODULE_LOAD = UDS_ERROR_CODE_BASE + 10, + /* The index session is disabled */ + UDS_DISABLED = UDS_ERROR_CODE_BASE + 11, + /* Unknown error */ + UDS_UNKNOWN_ERROR = UDS_ERROR_CODE_BASE + 12, + /* The index configuration or volume format is no longer supported */ + UDS_UNSUPPORTED_VERSION = UDS_ERROR_CODE_BASE + 13, + /* Some index structure is corrupt */ + UDS_CORRUPT_DATA = UDS_ERROR_CODE_BASE + 14, + /* No index state found */ + UDS_NO_INDEX = UDS_ERROR_CODE_BASE + 15, + /* Attempt to access incomplete index save data */ + UDS_INDEX_NOT_SAVED_CLEANLY = UDS_ERROR_CODE_BASE + 16, + /* One more than the last UDS_INTERNAL error code */ + UDS_ERROR_CODE_LAST, + /* One more than the last error this block will ever use */ + UDS_ERROR_CODE_BLOCK_END = UDS_ERROR_CODE_BASE + 440, +}; + +enum { + UDS_MAX_ERROR_NAME_SIZE = 80, + UDS_MAX_ERROR_MESSAGE_SIZE = 128, +}; + +struct error_info { + const char *name; + const char *message; +}; + +const char * __must_check uds_string_error(int errnum, char *buf, size_t buflen); + +const char *uds_string_error_name(int errnum, char *buf, size_t buflen); + +int uds_map_to_system_error(int error); + +int uds_register_error_block(const char *block_name, int first_error, + int last_reserved_error, const struct error_info *infos, + size_t info_size); + +#endif /* UDS_ERRORS_H */ diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c new file mode 100644 index 000000000000..fe632f10b5d9 --- /dev/null +++ b/drivers/md/dm-vdo/logger.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "logger.h" + +#include +#include +#include +#include +#include + +#include "thread-device.h" +#include "uds-threads.h" + +struct priority_name { + const char *name; + const int priority; +}; + +static const struct priority_name PRIORITIES[] = { + { "ALERT", UDS_LOG_ALERT }, + { "CRITICAL", UDS_LOG_CRIT }, + { "CRIT", UDS_LOG_CRIT }, + { "DEBUG", UDS_LOG_DEBUG }, + { "EMERGENCY", UDS_LOG_EMERG }, + { "EMERG", UDS_LOG_EMERG }, + { "ERROR", UDS_LOG_ERR }, + { "ERR", UDS_LOG_ERR }, + { "INFO", UDS_LOG_INFO }, + { "NOTICE", UDS_LOG_NOTICE }, + { "PANIC", UDS_LOG_EMERG }, + { "WARN", UDS_LOG_WARNING }, + { "WARNING", UDS_LOG_WARNING }, + { NULL, -1 }, +}; + +static const char *const PRIORITY_STRINGS[] = { + "EMERGENCY", + "ALERT", + "CRITICAL", + "ERROR", + "WARN", + "NOTICE", + "INFO", + "DEBUG", +}; + +static int log_level = UDS_LOG_INFO; + +int uds_get_log_level(void) +{ + return log_level; +} + +void uds_set_log_level(int new_log_level) +{ + log_level = new_log_level; +} + +int uds_log_string_to_priority(const char *string) +{ + int i; + + for (i = 0; PRIORITIES[i].name != NULL; i++) { + if (strcasecmp(string, PRIORITIES[i].name) == 0) + return PRIORITIES[i].priority; + } + + return UDS_LOG_INFO; +} + +const char *uds_log_priority_to_string(int priority) +{ + if ((priority < 0) || (priority >= (int) ARRAY_SIZE(PRIORITY_STRINGS))) + return "unknown"; + + return PRIORITY_STRINGS[priority]; +} + +static const char *get_current_interrupt_type(void) +{ + if (in_nmi()) + return "NMI"; + + if (in_irq()) + return "HI"; + + if (in_softirq()) + return "SI"; + + return "INTR"; +} + +/** + * emit_log_message_to_kernel() - Emit a log message to the kernel at the specified priority. + * + * @priority: The priority at which to log the message + * @fmt: The format string of the message + */ +static void emit_log_message_to_kernel(int priority, const char *fmt, ...) +{ + va_list args; + struct va_format vaf; + + if (priority > uds_get_log_level()) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + switch (priority) { + case UDS_LOG_EMERG: + case UDS_LOG_ALERT: + case UDS_LOG_CRIT: + printk(KERN_CRIT "%pV", &vaf); + break; + case UDS_LOG_ERR: + printk(KERN_ERR "%pV", &vaf); + break; + case UDS_LOG_WARNING: + printk(KERN_WARNING "%pV", &vaf); + break; + case UDS_LOG_NOTICE: + printk(KERN_NOTICE "%pV", &vaf); + break; + case UDS_LOG_INFO: + printk(KERN_INFO "%pV", &vaf); + break; + case UDS_LOG_DEBUG: + printk(KERN_DEBUG "%pV", &vaf); + break; + default: + printk(KERN_DEFAULT "%pV", &vaf); + break; + } + + va_end(args); +} + +/** + * emit_log_message() - Emit a log message to the kernel log in a format suited to the current + * thread context. + * + * Context info formats: + * + * interrupt: uds[NMI]: blah + * kvdo thread: kvdo12:foobarQ: blah + * thread w/device id: kvdo12:myprog: blah + * other thread: uds: myprog: blah + * + * Fields: module name, interrupt level, process name, device ID. + * + * @priority: the priority at which to log the message + * @module: The name of the module doing the logging + * @prefix: The prefix of the log message + * @vaf1: The first message format descriptor + * @vaf2: The second message format descriptor + */ +static void emit_log_message(int priority, const char *module, const char *prefix, + const struct va_format *vaf1, const struct va_format *vaf2) +{ + int device_instance; + + /* + * In interrupt context, identify the interrupt type and module. Ignore the process/thread + * since it could be anything. + */ + if (in_interrupt()) { + const char *type = get_current_interrupt_type(); + + emit_log_message_to_kernel(priority, "%s[%s]: %s%pV%pV\n", module, type, + prefix, vaf1, vaf2); + return; + } + + /* Not at interrupt level; we have a process we can look at, and might have a device ID. */ + device_instance = uds_get_thread_device_id(); + if (device_instance >= 0) { + emit_log_message_to_kernel(priority, "%s%u:%s: %s%pV%pV\n", module, + device_instance, current->comm, prefix, vaf1, + vaf2); + return; + } + + /* + * If it's a kernel thread and the module name is a prefix of its name, assume it is ours + * and only identify the thread. + */ + if (((current->flags & PF_KTHREAD) != 0) && + (strncmp(module, current->comm, strlen(module)) == 0)) { + emit_log_message_to_kernel(priority, "%s: %s%pV%pV\n", current->comm, + prefix, vaf1, vaf2); + return; + } + + /* Identify the module and the process. */ + emit_log_message_to_kernel(priority, "%s: %s: %s%pV%pV\n", module, current->comm, + prefix, vaf1, vaf2); +} + +/* + * uds_log_embedded_message() - Log a message embedded within another message. + * @priority: the priority at which to log the message + * @module: the name of the module doing the logging + * @prefix: optional string prefix to message, may be NULL + * @fmt1: format of message first part (required) + * @args1: arguments for message first part (required) + * @fmt2: format of message second part + */ +void uds_log_embedded_message(int priority, const char *module, const char *prefix, + const char *fmt1, va_list args1, const char *fmt2, ...) +{ + va_list args1_copy; + va_list args2; + struct va_format vaf1, vaf2; + + va_start(args2, fmt2); + + if (module == NULL) + module = UDS_LOGGING_MODULE_NAME; + + if (prefix == NULL) + prefix = ""; + + /* + * It is implementation dependent whether va_list is defined as an array type that decays + * to a pointer when passed as an argument. Copy args1 and args2 with va_copy so that vaf1 + * and vaf2 get proper va_list pointers irrespective of how va_list is defined. + */ + va_copy(args1_copy, args1); + vaf1.fmt = fmt1; + vaf1.va = &args1_copy; + + vaf2.fmt = fmt2; + vaf2.va = &args2; + + emit_log_message(priority, module, prefix, &vaf1, &vaf2); + + va_end(args1_copy); + va_end(args2); +} + +int uds_vlog_strerror(int priority, int errnum, const char *module, const char *format, + va_list args) +{ + char errbuf[UDS_MAX_ERROR_MESSAGE_SIZE]; + const char *message = uds_string_error(errnum, errbuf, sizeof(errbuf)); + + uds_log_embedded_message(priority, module, NULL, format, args, ": %s (%d)", + message, errnum); + return errnum; +} + +int __uds_log_strerror(int priority, int errnum, const char *module, const char *format, ...) +{ + va_list args; + + va_start(args, format); + uds_vlog_strerror(priority, errnum, module, format, args); + va_end(args); + return errnum; +} + +void uds_log_backtrace(int priority) +{ + if (priority > uds_get_log_level()) + return; + + dump_stack(); +} + +void __uds_log_message(int priority, const char *module, const char *format, ...) +{ + va_list args; + + va_start(args, format); + uds_log_embedded_message(priority, module, NULL, format, args, "%s", ""); + va_end(args); +} + +/* + * Sleep or delay a few milliseconds in an attempt to allow the log buffers to be flushed lest they + * be overrun. + */ +void uds_pause_for_logger(void) +{ + fsleep(4000); +} diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h new file mode 100644 index 000000000000..246aa43db9ee --- /dev/null +++ b/drivers/md/dm-vdo/logger.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_LOGGER_H +#define UDS_LOGGER_H + +#include +#include + +/* Custom logging utilities for UDS */ + +#define UDS_LOG_EMERG 0 +#define UDS_LOG_ALERT 1 +#define UDS_LOG_CRIT 2 +#define UDS_LOG_ERR 3 +#define UDS_LOG_WARNING 4 +#define UDS_LOG_NOTICE 5 +#define UDS_LOG_INFO 6 +#define UDS_LOG_DEBUG 7 + +#if defined(MODULE) +#define UDS_LOGGING_MODULE_NAME THIS_MODULE->name +#else /* compiled into the kernel */ +#define UDS_LOGGING_MODULE_NAME "vdo" +#endif + +/* Apply a rate limiter to a log method call. */ +#define uds_log_ratelimit(log_fn, ...) \ + do { \ + static DEFINE_RATELIMIT_STATE(_rs, \ + DEFAULT_RATELIMIT_INTERVAL, \ + DEFAULT_RATELIMIT_BURST); \ + if (__ratelimit(&_rs)) { \ + log_fn(__VA_ARGS__); \ + } \ + } while (0) + +int uds_get_log_level(void); + +void uds_set_log_level(int new_log_level); + +int uds_log_string_to_priority(const char *string); + +const char *uds_log_priority_to_string(int priority); + +void uds_log_embedded_message(int priority, const char *module, const char *prefix, + const char *fmt1, va_list args1, const char *fmt2, ...) + __printf(4, 0) __printf(6, 7); + +void uds_log_backtrace(int priority); + +/* All log functions will preserve the caller's value of errno. */ + +#define uds_log_strerror(priority, errnum, ...) \ + __uds_log_strerror(priority, errnum, UDS_LOGGING_MODULE_NAME, __VA_ARGS__) + +int __uds_log_strerror(int priority, int errnum, const char *module, + const char *format, ...) + __printf(4, 5); + +int uds_vlog_strerror(int priority, int errnum, const char *module, const char *format, + va_list args) + __printf(4, 0); + +/* Log an error prefixed with the string associated with the errnum. */ +#define uds_log_error_strerror(errnum, ...) \ + uds_log_strerror(UDS_LOG_ERR, errnum, __VA_ARGS__) + +#define uds_log_debug_strerror(errnum, ...) \ + uds_log_strerror(UDS_LOG_DEBUG, errnum, __VA_ARGS__) + +#define uds_log_info_strerror(errnum, ...) \ + uds_log_strerror(UDS_LOG_INFO, errnum, __VA_ARGS__) + +#define uds_log_notice_strerror(errnum, ...) \ + uds_log_strerror(UDS_LOG_NOTICE, errnum, __VA_ARGS__) + +#define uds_log_warning_strerror(errnum, ...) \ + uds_log_strerror(UDS_LOG_WARNING, errnum, __VA_ARGS__) + +#define uds_log_fatal_strerror(errnum, ...) \ + uds_log_strerror(UDS_LOG_CRIT, errnum, __VA_ARGS__) + +#define uds_log_message(priority, ...) \ + __uds_log_message(priority, UDS_LOGGING_MODULE_NAME, __VA_ARGS__) + +void __uds_log_message(int priority, const char *module, const char *format, ...) + __printf(3, 4); + +#define uds_log_debug(...) uds_log_message(UDS_LOG_DEBUG, __VA_ARGS__) + +#define uds_log_info(...) uds_log_message(UDS_LOG_INFO, __VA_ARGS__) + +#define uds_log_notice(...) uds_log_message(UDS_LOG_NOTICE, __VA_ARGS__) + +#define uds_log_warning(...) uds_log_message(UDS_LOG_WARNING, __VA_ARGS__) + +#define uds_log_error(...) uds_log_message(UDS_LOG_ERR, __VA_ARGS__) + +#define uds_log_fatal(...) uds_log_message(UDS_LOG_CRIT, __VA_ARGS__) + +void uds_pause_for_logger(void); +#endif /* UDS_LOGGER_H */ diff --git a/drivers/md/dm-vdo/permassert.c b/drivers/md/dm-vdo/permassert.c new file mode 100644 index 000000000000..3fa752ba0061 --- /dev/null +++ b/drivers/md/dm-vdo/permassert.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "permassert.h" + +#include "errors.h" +#include "logger.h" + +int uds_assertion_failed(const char *expression_string, const char *file_name, + int line_number, const char *format, ...) +{ + va_list args; + + va_start(args, format); + + uds_log_embedded_message(UDS_LOG_ERR, UDS_LOGGING_MODULE_NAME, "assertion \"", + format, args, "\" (%s) failed at %s:%d", + expression_string, file_name, line_number); + uds_log_backtrace(UDS_LOG_ERR); + + va_end(args); + + return UDS_ASSERTION_FAILED; +} diff --git a/drivers/md/dm-vdo/permassert.h b/drivers/md/dm-vdo/permassert.h new file mode 100644 index 000000000000..ee978bc115ec --- /dev/null +++ b/drivers/md/dm-vdo/permassert.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef PERMASSERT_H +#define PERMASSERT_H + +#include + +#include "errors.h" + +/* Utilities for asserting that certain conditions are met */ + +#define STRINGIFY(X) #X +#define STRINGIFY_VALUE(X) STRINGIFY(X) + +/* + * A hack to apply the "warn if unused" attribute to an integral expression. + * + * Since GCC doesn't propagate the warn_unused_result attribute to conditional expressions + * incorporating calls to functions with that attribute, this function can be used to wrap such an + * expression. With optimization enabled, this function contributes no additional instructions, but + * the warn_unused_result attribute still applies to the code calling it. + */ +static inline int __must_check uds_must_use(int value) +{ + return value; +} + +/* Assert that an expression is true and return an error if it is not. */ +#define ASSERT(expr, ...) uds_must_use(__UDS_ASSERT(expr, __VA_ARGS__)) + +/* Log a message if the expression is not true. */ +#define ASSERT_LOG_ONLY(expr, ...) __UDS_ASSERT(expr, __VA_ARGS__) + +#define __UDS_ASSERT(expr, ...) \ + (likely(expr) ? UDS_SUCCESS \ + : uds_assertion_failed(STRINGIFY(expr), __FILE__, __LINE__, __VA_ARGS__)) + +/* Log an assertion failure message. */ +int uds_assertion_failed(const char *expression_string, const char *file_name, + int line_number, const char *format, ...) + __printf(4, 5); + +#endif /* PERMASSERT_H */ diff --git a/drivers/md/dm-vdo/string-utils.c b/drivers/md/dm-vdo/string-utils.c new file mode 100644 index 000000000000..a584b37bb70c --- /dev/null +++ b/drivers/md/dm-vdo/string-utils.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "string-utils.h" + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "uds.h" + +char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) +{ + va_list args; + size_t n; + + va_start(args, fmt); + n = vsnprintf(buffer, buf_end - buffer, fmt, args); + if (n >= (size_t) (buf_end - buffer)) + buffer = buf_end; + else + buffer += n; + va_end(args); + + return buffer; +} diff --git a/drivers/md/dm-vdo/string-utils.h b/drivers/md/dm-vdo/string-utils.h new file mode 100644 index 000000000000..8275af582cf7 --- /dev/null +++ b/drivers/md/dm-vdo/string-utils.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_STRING_UTILS_H +#define UDS_STRING_UTILS_H + +#include +#include + +/* Utilities related to string manipulation */ + +static inline const char *uds_bool_to_string(bool value) +{ + return value ? "true" : "false"; +} + +/* Append a formatted string to the end of a buffer. */ +char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) + __printf(3, 4); + +#endif /* UDS_STRING_UTILS_H */ diff --git a/drivers/md/dm-vdo/time-utils.h b/drivers/md/dm-vdo/time-utils.h new file mode 100644 index 000000000000..5f1e850fd826 --- /dev/null +++ b/drivers/md/dm-vdo/time-utils.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_TIME_UTILS_H +#define UDS_TIME_UTILS_H + +#include +#include +#include + +static inline s64 ktime_to_seconds(ktime_t reltime) +{ + return reltime / NSEC_PER_SEC; +} + +static inline ktime_t current_time_ns(clockid_t clock) +{ + return clock == CLOCK_MONOTONIC ? ktime_get_ns() : ktime_get_real_ns(); +} + +static inline ktime_t current_time_us(void) +{ + return current_time_ns(CLOCK_REALTIME) / NSEC_PER_USEC; +} + +#endif /* UDS_TIME_UTILS_H */ From 4fcb4290dfe303de048fe01968513da6f4512147 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:46:10 -0500 Subject: [PATCH 0553/1038] dm vdo: add vdo type declarations, constants, and simple data structures Add definitions of constants defining the fixed parameters of a VDO volume, and the default and maximum values of configurable or dynamic parameters. Add definitions of internal status codes used for internal communication within the module and for logging. Add definitions of types and structs used to manage the processing of an I/O operation. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/constants.c | 15 ++ drivers/md/dm-vdo/constants.h | 102 ++++++++ drivers/md/dm-vdo/status-codes.c | 123 ++++++++++ drivers/md/dm-vdo/status-codes.h | 112 +++++++++ drivers/md/dm-vdo/types.h | 393 +++++++++++++++++++++++++++++++ drivers/md/dm-vdo/wait-queue.c | 225 ++++++++++++++++++ drivers/md/dm-vdo/wait-queue.h | 131 +++++++++++ 7 files changed, 1101 insertions(+) create mode 100644 drivers/md/dm-vdo/constants.c create mode 100644 drivers/md/dm-vdo/constants.h create mode 100644 drivers/md/dm-vdo/status-codes.c create mode 100644 drivers/md/dm-vdo/status-codes.h create mode 100644 drivers/md/dm-vdo/types.h create mode 100644 drivers/md/dm-vdo/wait-queue.c create mode 100644 drivers/md/dm-vdo/wait-queue.h diff --git a/drivers/md/dm-vdo/constants.c b/drivers/md/dm-vdo/constants.c new file mode 100644 index 000000000000..14ac047101f8 --- /dev/null +++ b/drivers/md/dm-vdo/constants.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "types.h" + +/* The maximum logical space is 4 petabytes, which is 1 terablock. */ +const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; + +/* The maximum physical space is 256 terabytes, which is 64 gigablocks. */ +const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; + +/* unit test minimum */ +const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2; diff --git a/drivers/md/dm-vdo/constants.h b/drivers/md/dm-vdo/constants.h new file mode 100644 index 000000000000..9ae3917d00de --- /dev/null +++ b/drivers/md/dm-vdo/constants.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_CONSTANTS_H +#define VDO_CONSTANTS_H + +#include + +#include "types.h" + +enum { + /* + * The maximum number of contiguous PBNs which will go to a single bio submission queue, + * assuming there is more than one queue. + */ + VDO_BIO_ROTATION_INTERVAL_LIMIT = 1024, + + /** The number of entries on a block map page */ + VDO_BLOCK_MAP_ENTRIES_PER_PAGE = 812, + + /** The origin of the flat portion of the block map */ + VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN = 1, + + /* + * The height of a block map tree. Assuming a root count of 60 and 812 entries per page, + * this is big enough to represent almost 95 PB of logical space. + */ + VDO_BLOCK_MAP_TREE_HEIGHT = 5, + + /** The default number of bio submission queues. */ + DEFAULT_VDO_BIO_SUBMIT_QUEUE_COUNT = 4, + + /** The number of contiguous PBNs to be submitted to a single bio queue. */ + DEFAULT_VDO_BIO_SUBMIT_QUEUE_ROTATE_INTERVAL = 64, + + /** The number of trees in the arboreal block map */ + DEFAULT_VDO_BLOCK_MAP_TREE_ROOT_COUNT = 60, + + /** The default size of the recovery journal, in blocks */ + DEFAULT_VDO_RECOVERY_JOURNAL_SIZE = 32 * 1024, + + /** The default size of each slab journal, in blocks */ + DEFAULT_VDO_SLAB_JOURNAL_SIZE = 224, + + /* + * The initial size of lbn_operations and pbn_operations, which is based upon the expected + * maximum number of outstanding VIOs. This value was chosen to make it highly unlikely + * that the maps would need to be resized. + */ + VDO_LOCK_MAP_CAPACITY = 10000, + + /** The maximum number of logical zones */ + MAX_VDO_LOGICAL_ZONES = 60, + + /** The maximum number of physical zones */ + MAX_VDO_PHYSICAL_ZONES = 16, + + /** The base-2 logarithm of the maximum blocks in one slab */ + MAX_VDO_SLAB_BITS = 23, + + /** The maximum number of slabs the slab depot supports */ + MAX_VDO_SLABS = 8192, + + /* + * The maximum number of block map pages to load simultaneously during recovery or rebuild. + */ + MAXIMUM_SIMULTANEOUS_VDO_BLOCK_MAP_RESTORATION_READS = 1024, + + /** The maximum number of entries in the slab summary */ + MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES = MAX_VDO_SLABS * MAX_VDO_PHYSICAL_ZONES, + + /** The maximum number of total threads in a VDO thread configuration. */ + MAXIMUM_VDO_THREADS = 100, + + /** The maximum number of VIOs in the system at once */ + MAXIMUM_VDO_USER_VIOS = 2048, + + /** The only physical block size supported by VDO */ + VDO_BLOCK_SIZE = 4096, + + /** The number of sectors per block */ + VDO_SECTORS_PER_BLOCK = (VDO_BLOCK_SIZE >> SECTOR_SHIFT), + + /** The size of a sector that will not be torn */ + VDO_SECTOR_SIZE = 512, + + /** The physical block number reserved for storing the zero block */ + VDO_ZERO_BLOCK = 0, +}; + +/** The maximum logical space is 4 petabytes, which is 1 terablock. */ +extern const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS; + +/** The maximum physical space is 256 terabytes, which is 64 gigablocks. */ +extern const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS; + +/** unit test minimum */ +extern const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS; + +#endif /* VDO_CONSTANTS_H */ diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c new file mode 100644 index 000000000000..33b1de29b205 --- /dev/null +++ b/drivers/md/dm-vdo/status-codes.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "status-codes.h" + +#include "errors.h" +#include "logger.h" +#include "permassert.h" +#include "uds-threads.h" + +const struct error_info vdo_status_list[] = { + { "VDO_NOT_IMPLEMENTED", "Not implemented" }, + { "VDO_OUT_OF_RANGE", "Out of range" }, + { "VDO_REF_COUNT_INVALID", "Reference count would become invalid" }, + { "VDO_NO_SPACE", "Out of space" }, + { "VDO_UNEXPECTED_EOF", "Unexpected EOF on block read" }, + { "VDO_BAD_CONFIGURATION", "Bad configuration option" }, + { "VDO_SOCKET_ERROR", "Socket error" }, + { "VDO_BAD_ALIGNMENT", "Mis-aligned block reference" }, + { "VDO_COMPONENT_BUSY", "Prior operation still in progress" }, + { "VDO_BAD_PAGE", "Corrupt or incorrect page" }, + { "VDO_UNSUPPORTED_VERSION", "Unsupported component version" }, + { "VDO_INCORRECT_COMPONENT", "Component id mismatch in decoder" }, + { "VDO_PARAMETER_MISMATCH", "Parameters have conflicting values" }, + { "VDO_BLOCK_SIZE_TOO_SMALL", "The block size is too small" }, + { "VDO_UNKNOWN_PARTITION", "No partition exists with a given id" }, + { "VDO_PARTITION_EXISTS", "A partition already exists with a given id" }, + { "VDO_NOT_READ_ONLY", "The device is not in read-only mode" }, + { "VDO_INCREMENT_TOO_SMALL", "Physical block growth of too few blocks" }, + { "VDO_CHECKSUM_MISMATCH", "Incorrect checksum" }, + { "VDO_RECOVERY_JOURNAL_FULL", "The recovery journal is full" }, + { "VDO_LOCK_ERROR", "A lock is held incorrectly" }, + { "VDO_READ_ONLY", "The device is in read-only mode" }, + { "VDO_SHUTTING_DOWN", "The device is shutting down" }, + { "VDO_CORRUPT_JOURNAL", "Recovery journal entries corrupted" }, + { "VDO_TOO_MANY_SLABS", "Exceeds maximum number of slabs supported" }, + { "VDO_INVALID_FRAGMENT", "Compressed block fragment is invalid" }, + { "VDO_RETRY_AFTER_REBUILD", "Retry operation after rebuilding finishes" }, + { "VDO_UNKNOWN_COMMAND", "The extended command is not known" }, + { "VDO_COMMAND_ERROR", "Bad extended command parameters" }, + { "VDO_CANNOT_DETERMINE_SIZE", "Cannot determine config sizes to fit" }, + { "VDO_BAD_MAPPING", "Invalid page mapping" }, + { "VDO_READ_CACHE_BUSY", "Read cache has no free slots" }, + { "VDO_BIO_CREATION_FAILED", "Bio creation failed" }, + { "VDO_BAD_MAGIC", "Bad magic number" }, + { "VDO_BAD_NONCE", "Bad nonce" }, + { "VDO_JOURNAL_OVERFLOW", "Journal sequence number overflow" }, + { "VDO_INVALID_ADMIN_STATE", "Invalid operation for current state" }, + { "VDO_CANT_ADD_SYSFS_NODE", "Failed to add sysfs node" }, +}; + +static atomic_t vdo_status_codes_registered = ATOMIC_INIT(0); +static int status_code_registration_result; + +static void do_status_code_registration(void) +{ + int result; + + BUILD_BUG_ON((VDO_STATUS_CODE_LAST - VDO_STATUS_CODE_BASE) != + ARRAY_SIZE(vdo_status_list)); + + result = uds_register_error_block("VDO Status", VDO_STATUS_CODE_BASE, + VDO_STATUS_CODE_BLOCK_END, vdo_status_list, + sizeof(vdo_status_list)); + /* + * The following test handles cases where libvdo is statically linked against both the test + * modules and the test driver (because multiple instances of this module call their own + * copy of this function once each, resulting in multiple calls to register_error_block + * which is shared in libuds). + */ + if (result == UDS_DUPLICATE_NAME) + result = UDS_SUCCESS; + + status_code_registration_result = (result == UDS_SUCCESS) ? VDO_SUCCESS : result; +} + +/** + * vdo_register_status_codes() - Register the VDO status codes if needed. + * Return: A success or error code. + */ +int vdo_register_status_codes(void) +{ + uds_perform_once(&vdo_status_codes_registered, do_status_code_registration); + return status_code_registration_result; +} + +/** + * vdo_map_to_system_error() - Given an error code, return a value we can return to the OS. + * @error: The error code to convert. + * + * The input error code may be a system-generated value (such as -EIO), an errno macro used in our + * code (such as EIO), or a UDS or VDO status code; the result must be something the rest of the OS + * can consume (negative errno values such as -EIO, in the case of the kernel). + * + * Return: A system error code value. + */ +int vdo_map_to_system_error(int error) +{ + char error_name[UDS_MAX_ERROR_NAME_SIZE]; + char error_message[UDS_MAX_ERROR_MESSAGE_SIZE]; + + /* 0 is success, negative a system error code */ + if (likely(error <= 0)) + return error; + if (error < 1024) + return -error; + + /* VDO or UDS error */ + switch (error) { + case VDO_NO_SPACE: + return -ENOSPC; + case VDO_READ_ONLY: + return -EIO; + default: + uds_log_info("%s: mapping internal status code %d (%s: %s) to EIO", + __func__, error, + uds_string_error_name(error, error_name, sizeof(error_name)), + uds_string_error(error, error_message, sizeof(error_message))); + return -EIO; + } +} diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h new file mode 100644 index 000000000000..c5da37ec8b55 --- /dev/null +++ b/drivers/md/dm-vdo/status-codes.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_STATUS_CODES_H +#define VDO_STATUS_CODES_H + +#include "errors.h" + +enum { + UDS_BLOCK_SIZE = UDS_ERROR_CODE_BLOCK_END - UDS_ERROR_CODE_BASE, + VDO_BLOCK_START = UDS_ERROR_CODE_BLOCK_END, + VDO_BLOCK_END = VDO_BLOCK_START + UDS_BLOCK_SIZE, + PRP_BLOCK_START = VDO_BLOCK_END, + PRP_BLOCK_END = PRP_BLOCK_START + UDS_BLOCK_SIZE, +}; + +/* VDO-specific status codes. */ +enum vdo_status_codes { + /* successful result */ + VDO_SUCCESS, + /* base of all VDO errors */ + VDO_STATUS_CODE_BASE = VDO_BLOCK_START, + /* we haven't written this yet */ + VDO_NOT_IMPLEMENTED = VDO_STATUS_CODE_BASE, + /* input out of range */ + VDO_OUT_OF_RANGE, + /* an invalid reference count would result */ + VDO_REF_COUNT_INVALID, + /* a free block could not be allocated */ + VDO_NO_SPACE, + /* unexpected EOF on block read */ + VDO_UNEXPECTED_EOF, + /* improper or missing configuration option */ + VDO_BAD_CONFIGURATION, + /* socket opening or binding problem */ + VDO_SOCKET_ERROR, + /* read or write on non-aligned offset */ + VDO_BAD_ALIGNMENT, + /* prior operation still in progress */ + VDO_COMPONENT_BUSY, + /* page contents incorrect or corrupt data */ + VDO_BAD_PAGE, + /* unsupported version of some component */ + VDO_UNSUPPORTED_VERSION, + /* component id mismatch in decoder */ + VDO_INCORRECT_COMPONENT, + /* parameters have conflicting values */ + VDO_PARAMETER_MISMATCH, + /* the block size is too small */ + VDO_BLOCK_SIZE_TOO_SMALL, + /* no partition exists with a given id */ + VDO_UNKNOWN_PARTITION, + /* a partition already exists with a given id */ + VDO_PARTITION_EXISTS, + /* the VDO is not in read-only mode */ + VDO_NOT_READ_ONLY, + /* physical block growth of too few blocks */ + VDO_INCREMENT_TOO_SMALL, + /* incorrect checksum */ + VDO_CHECKSUM_MISMATCH, + /* the recovery journal is full */ + VDO_RECOVERY_JOURNAL_FULL, + /* a lock is held incorrectly */ + VDO_LOCK_ERROR, + /* the VDO is in read-only mode */ + VDO_READ_ONLY, + /* the VDO is shutting down */ + VDO_SHUTTING_DOWN, + /* the recovery journal has corrupt entries */ + VDO_CORRUPT_JOURNAL, + /* exceeds maximum number of slabs supported */ + VDO_TOO_MANY_SLABS, + /* a compressed block fragment is invalid */ + VDO_INVALID_FRAGMENT, + /* action is unsupported while rebuilding */ + VDO_RETRY_AFTER_REBUILD, + /* the extended command is not known */ + VDO_UNKNOWN_COMMAND, + /* bad extended command parameters */ + VDO_COMMAND_ERROR, + /* cannot determine sizes to fit */ + VDO_CANNOT_DETERMINE_SIZE, + /* a block map entry is invalid */ + VDO_BAD_MAPPING, + /* read cache has no free slots */ + VDO_READ_CACHE_BUSY, + /* bio_add_page failed */ + VDO_BIO_CREATION_FAILED, + /* bad magic number */ + VDO_BAD_MAGIC, + /* bad nonce */ + VDO_BAD_NONCE, + /* sequence number overflow */ + VDO_JOURNAL_OVERFLOW, + /* the VDO is not in a state to perform an admin operation */ + VDO_INVALID_ADMIN_STATE, + /* failure adding a sysfs node */ + VDO_CANT_ADD_SYSFS_NODE, + /* one more than last error code */ + VDO_STATUS_CODE_LAST, + VDO_STATUS_CODE_BLOCK_END = VDO_BLOCK_END +}; + +extern const struct error_info vdo_status_list[]; + +int vdo_register_status_codes(void); + +int vdo_map_to_system_error(int error); + +#endif /* VDO_STATUS_CODES_H */ diff --git a/drivers/md/dm-vdo/types.h b/drivers/md/dm-vdo/types.h new file mode 100644 index 000000000000..abc6d36f6522 --- /dev/null +++ b/drivers/md/dm-vdo/types.h @@ -0,0 +1,393 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_TYPES_H +#define VDO_TYPES_H + +#include +#include +#include +#include +#include +#include + +#include "funnel-queue.h" + +/* A size type in blocks. */ +typedef u64 block_count_t; + +/* The size of a block. */ +typedef u16 block_size_t; + +/* A counter for data_vios */ +typedef u16 data_vio_count_t; + +/* A height within a tree. */ +typedef u8 height_t; + +/* The logical block number as used by the consumer. */ +typedef u64 logical_block_number_t; + +/* The type of the nonce used to identify instances of VDO. */ +typedef u64 nonce_t; + +/* A size in pages. */ +typedef u32 page_count_t; + +/* A page number. */ +typedef u32 page_number_t; + +/* + * The physical (well, less logical) block number at which the block is found on the underlying + * device. + */ +typedef u64 physical_block_number_t; + +/* A count of tree roots. */ +typedef u8 root_count_t; + +/* A number of sectors. */ +typedef u8 sector_count_t; + +/* A sequence number. */ +typedef u64 sequence_number_t; + +/* The offset of a block within a slab. */ +typedef u32 slab_block_number; + +/* A size type in slabs. */ +typedef u16 slab_count_t; + +/* A slot in a bin or block map page. */ +typedef u16 slot_number_t; + +/* typedef thread_count_t - A thread counter. */ +typedef u8 thread_count_t; + +/* typedef thread_id_t - A thread ID, vdo threads are numbered sequentially from 0. */ +typedef u8 thread_id_t; + +/* A zone counter */ +typedef u8 zone_count_t; + +/* The following enums are persisted on storage, so the values must be preserved. */ + +/* The current operating mode of the VDO. */ +enum vdo_state { + VDO_DIRTY = 0, + VDO_NEW = 1, + VDO_CLEAN = 2, + VDO_READ_ONLY_MODE = 3, + VDO_FORCE_REBUILD = 4, + VDO_RECOVERING = 5, + VDO_REPLAYING = 6, /* VDO_REPLAYING is never set anymore, but retained for upgrade */ + VDO_REBUILD_FOR_UPGRADE = 7, + + /* Keep VDO_STATE_COUNT at the bottom. */ + VDO_STATE_COUNT +}; + +/** + * vdo_state_requires_read_only_rebuild() - Check whether a vdo_state indicates + * that a read-only rebuild is required. + * @state: The vdo_state to check. + * + * Return: true if the state indicates a rebuild is required + */ +static inline bool __must_check vdo_state_requires_read_only_rebuild(enum vdo_state state) +{ + return ((state == VDO_FORCE_REBUILD) || (state == VDO_REBUILD_FOR_UPGRADE)); +} + +/** + * vdo_state_requires_recovery() - Check whether a vdo state indicates that recovery is needed. + * @state: The state to check. + * + * Return: true if the state indicates a recovery is required + */ +static inline bool __must_check vdo_state_requires_recovery(enum vdo_state state) +{ + return ((state == VDO_DIRTY) || (state == VDO_REPLAYING) || (state == VDO_RECOVERING)); +} + +/* + * The current operation on a physical block (from the point of view of the recovery journal, slab + * journals, and reference counts. + */ +enum journal_operation { + VDO_JOURNAL_DATA_REMAPPING = 0, + VDO_JOURNAL_BLOCK_MAP_REMAPPING = 1, +} __packed; + +/* Partition IDs encoded in the volume layout in the super block. */ +enum partition_id { + VDO_BLOCK_MAP_PARTITION = 0, + VDO_SLAB_DEPOT_PARTITION = 1, + VDO_RECOVERY_JOURNAL_PARTITION = 2, + VDO_SLAB_SUMMARY_PARTITION = 3, +} __packed; + +/* Metadata types for the vdo. */ +enum vdo_metadata_type { + VDO_METADATA_RECOVERY_JOURNAL = 1, + VDO_METADATA_SLAB_JOURNAL = 2, + VDO_METADATA_RECOVERY_JOURNAL_2 = 3, +} __packed; + +/* A position in the block map where a block map entry is stored. */ +struct block_map_slot { + physical_block_number_t pbn; + slot_number_t slot; +}; + +/* + * Four bits of each five-byte block map entry contain a mapping state value used to distinguish + * unmapped or trimmed logical blocks (which are treated as mapped to the zero block) from entries + * that have been mapped to a physical block, including the zero block. + * + * FIXME: these should maybe be defines. + */ +enum block_mapping_state { + VDO_MAPPING_STATE_UNMAPPED = 0, /* Must be zero to be the default value */ + VDO_MAPPING_STATE_UNCOMPRESSED = 1, /* A normal (uncompressed) block */ + VDO_MAPPING_STATE_COMPRESSED_BASE = 2, /* Compressed in slot 0 */ + VDO_MAPPING_STATE_COMPRESSED_MAX = 15, /* Compressed in slot 13 */ +}; + +enum { + VDO_MAX_COMPRESSION_SLOTS = + (VDO_MAPPING_STATE_COMPRESSED_MAX - VDO_MAPPING_STATE_COMPRESSED_BASE + 1), +}; + + +struct data_location { + physical_block_number_t pbn; + enum block_mapping_state state; +}; + +/* The configuration of a single slab derived from the configured block size and slab size. */ +struct slab_config { + /* total number of blocks in the slab */ + block_count_t slab_blocks; + /* number of blocks available for data */ + block_count_t data_blocks; + /* number of blocks for reference counts */ + block_count_t reference_count_blocks; + /* number of blocks for the slab journal */ + block_count_t slab_journal_blocks; + /* + * Number of blocks after which the slab journal starts pushing out a reference_block for + * each new entry it receives. + */ + block_count_t slab_journal_flushing_threshold; + /* + * Number of blocks after which the slab journal pushes out all reference_blocks and makes + * all vios wait. + */ + block_count_t slab_journal_blocking_threshold; + /* Number of blocks after which the slab must be scrubbed before coming online. */ + block_count_t slab_journal_scrubbing_threshold; +} __packed; + +/* + * This structure is memcmp'd for equality. Keep it packed and don't add any fields that are not + * properly set in both extant and parsed configs. + */ +struct thread_count_config { + unsigned int bio_ack_threads; + unsigned int bio_threads; + unsigned int bio_rotation_interval; + unsigned int cpu_threads; + unsigned int logical_zones; + unsigned int physical_zones; + unsigned int hash_zones; +} __packed; + +struct device_config { + struct dm_target *owning_target; + struct dm_dev *owned_device; + struct vdo *vdo; + /* All configs referencing a layer are kept on a list in the layer */ + struct list_head config_list; + char *original_string; + unsigned int version; + char *parent_device_name; + block_count_t physical_blocks; + /* + * This is the number of logical blocks from VDO's internal point of view. It is the number + * of 4K blocks regardless of the value of the logical_block_size parameter below. + */ + block_count_t logical_blocks; + unsigned int logical_block_size; + unsigned int cache_size; + unsigned int block_map_maximum_age; + bool deduplication; + bool compression; + struct thread_count_config thread_counts; + block_count_t max_discard_blocks; +}; + +enum vdo_completion_type { + /* Keep VDO_UNSET_COMPLETION_TYPE at the top. */ + VDO_UNSET_COMPLETION_TYPE, + VDO_ACTION_COMPLETION, + VDO_ADMIN_COMPLETION, + VDO_BLOCK_ALLOCATOR_COMPLETION, + VDO_DATA_VIO_POOL_COMPLETION, + VDO_DECREMENT_COMPLETION, + VDO_FLUSH_COMPLETION, + VDO_FLUSH_NOTIFICATION_COMPLETION, + VDO_GENERATION_FLUSHED_COMPLETION, + VDO_HASH_ZONE_COMPLETION, + VDO_HASH_ZONES_COMPLETION, + VDO_LOCK_COUNTER_COMPLETION, + VDO_PAGE_COMPLETION, + VDO_READ_ONLY_MODE_COMPLETION, + VDO_REPAIR_COMPLETION, + VDO_SYNC_COMPLETION, + VIO_COMPLETION, +} __packed; + +struct vdo_completion; + +/** + * typedef vdo_action_fn - An asynchronous VDO operation. + * @completion: The completion of the operation. + */ +typedef void (*vdo_action_fn)(struct vdo_completion *completion); + +enum vdo_completion_priority { + BIO_ACK_Q_ACK_PRIORITY = 0, + BIO_ACK_Q_MAX_PRIORITY = 0, + BIO_Q_COMPRESSED_DATA_PRIORITY = 0, + BIO_Q_DATA_PRIORITY = 0, + BIO_Q_FLUSH_PRIORITY = 2, + BIO_Q_HIGH_PRIORITY = 2, + BIO_Q_METADATA_PRIORITY = 1, + BIO_Q_VERIFY_PRIORITY = 1, + BIO_Q_MAX_PRIORITY = 2, + CPU_Q_COMPLETE_VIO_PRIORITY = 0, + CPU_Q_COMPLETE_READ_PRIORITY = 0, + CPU_Q_COMPRESS_BLOCK_PRIORITY = 0, + CPU_Q_EVENT_REPORTER_PRIORITY = 0, + CPU_Q_HASH_BLOCK_PRIORITY = 0, + CPU_Q_MAX_PRIORITY = 0, + UDS_Q_PRIORITY = 0, + UDS_Q_MAX_PRIORITY = 0, + VDO_DEFAULT_Q_COMPLETION_PRIORITY = 1, + VDO_DEFAULT_Q_FLUSH_PRIORITY = 2, + VDO_DEFAULT_Q_MAP_BIO_PRIORITY = 0, + VDO_DEFAULT_Q_SYNC_PRIORITY = 2, + VDO_DEFAULT_Q_VIO_CALLBACK_PRIORITY = 1, + VDO_DEFAULT_Q_MAX_PRIORITY = 2, + /* The maximum allowable priority */ + VDO_WORK_Q_MAX_PRIORITY = 2, + /* A value which must be out of range for a valid priority */ + VDO_WORK_Q_DEFAULT_PRIORITY = VDO_WORK_Q_MAX_PRIORITY + 1, +}; + +struct vdo_completion { + /* The type of completion this is */ + enum vdo_completion_type type; + + /* + * true once the processing of the operation is complete. This flag should not + * be used by waiters external to the VDO base as it is used to gate calling the callback. + */ + bool complete; + + /* + * If true, queue this completion on the next callback invocation, even if it is already + * running on the correct thread. + */ + bool requeue; + + /* The ID of the thread which should run the next callback */ + thread_id_t callback_thread_id; + + /* The result of the operation */ + int result; + + /* The VDO on which this completion operates */ + struct vdo *vdo; + + /* The callback which will be called once the operation is complete */ + vdo_action_fn callback; + + /* Callback which, if set, will be called if an error result is set */ + vdo_action_fn error_handler; + + /* The parent object, if any, that spawned this completion */ + void *parent; + + /* Entry link for lock-free work queue */ + struct funnel_queue_entry work_queue_entry_link; + enum vdo_completion_priority priority; + struct vdo_work_queue *my_queue; +}; + +struct block_allocator; +struct data_vio; +struct vdo; +struct vdo_config; + +/* vio types for statistics and instrumentation. */ +enum vio_type { + VIO_TYPE_UNINITIALIZED = 0, + VIO_TYPE_DATA, + VIO_TYPE_BLOCK_ALLOCATOR, + VIO_TYPE_BLOCK_MAP, + VIO_TYPE_BLOCK_MAP_INTERIOR, + VIO_TYPE_GEOMETRY, + VIO_TYPE_PARTITION_COPY, + VIO_TYPE_RECOVERY_JOURNAL, + VIO_TYPE_SLAB_JOURNAL, + VIO_TYPE_SLAB_SUMMARY, + VIO_TYPE_SUPER_BLOCK, +} __packed; + +/* Priority levels for asynchronous I/O operations performed on a vio. */ +enum vio_priority { + VIO_PRIORITY_LOW = 0, + VIO_PRIORITY_DATA = VIO_PRIORITY_LOW, + VIO_PRIORITY_COMPRESSED_DATA = VIO_PRIORITY_DATA, + VIO_PRIORITY_METADATA, + VIO_PRIORITY_HIGH, +} __packed; + +/* + * A wrapper for a bio. All I/O to the storage below a vdo is conducted via vios. + */ +struct vio { + /* The completion for this vio */ + struct vdo_completion completion; + + /* The bio zone in which I/O should be processed */ + zone_count_t bio_zone; + + /* The queueing priority of the vio operation */ + enum vio_priority priority; + + /* The vio type is used for statistics and instrumentation. */ + enum vio_type type; + + /* The size of this vio in blocks */ + unsigned int block_count; + + /* The data being read or written. */ + char *data; + + /* The VDO-owned bio to use for all IO for this vio */ + struct bio *bio; + + /* + * A list of enqueued bios with consecutive block numbers, stored by vdo_submit_bio() under + * the first-enqueued vio. The other vios are found via their bio entries in this list, and + * are not added to the work queue as separate completions. + */ + struct bio_list bios_merged; +}; + +#endif /* VDO_TYPES_H */ diff --git a/drivers/md/dm-vdo/wait-queue.c b/drivers/md/dm-vdo/wait-queue.c new file mode 100644 index 000000000000..8acc24e79d2b --- /dev/null +++ b/drivers/md/dm-vdo/wait-queue.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "wait-queue.h" + +#include + +#include "permassert.h" + +#include "status-codes.h" + +/** + * vdo_enqueue_waiter() - Add a waiter to the tail end of a wait queue. + * @queue: The queue to which to add the waiter. + * @waiter: The waiter to add to the queue. + * + * The waiter must not already be waiting in a queue. + * + * Return: VDO_SUCCESS or an error code. + */ +void vdo_enqueue_waiter(struct wait_queue *queue, struct waiter *waiter) +{ + BUG_ON(waiter->next_waiter != NULL); + + if (queue->last_waiter == NULL) { + /* + * The queue is empty, so form the initial circular list by self-linking the + * initial waiter. + */ + waiter->next_waiter = waiter; + } else { + /* Splice the new waiter in at the end of the queue. */ + waiter->next_waiter = queue->last_waiter->next_waiter; + queue->last_waiter->next_waiter = waiter; + } + + /* In both cases, the waiter we added to the ring becomes the last waiter. */ + queue->last_waiter = waiter; + queue->queue_length += 1; +} + +/** + * vdo_transfer_all_waiters() - Transfer all waiters from one wait queue to a second queue, + * emptying the first queue. + * @from_queue: The queue containing the waiters to move. + * @to_queue: The queue that will receive the waiters from the first queue. + */ +void vdo_transfer_all_waiters(struct wait_queue *from_queue, struct wait_queue *to_queue) +{ + /* If the source queue is empty, there's nothing to do. */ + if (!vdo_has_waiters(from_queue)) + return; + + if (vdo_has_waiters(to_queue)) { + /* + * Both queues are non-empty. Splice the two circular lists together by swapping + * the next (head) pointers in the list tails. + */ + struct waiter *from_head = from_queue->last_waiter->next_waiter; + struct waiter *to_head = to_queue->last_waiter->next_waiter; + + to_queue->last_waiter->next_waiter = from_head; + from_queue->last_waiter->next_waiter = to_head; + } + + to_queue->last_waiter = from_queue->last_waiter; + to_queue->queue_length += from_queue->queue_length; + vdo_initialize_wait_queue(from_queue); +} + +/** + * vdo_notify_all_waiters() - Notify all the entries waiting in a queue. + * @queue: The wait queue containing the waiters to notify. + * @callback: The function to call to notify each waiter, or NULL to invoke the callback field + * registered in each waiter. + * @context: The context to pass to the callback function. + * + * Notifies all the entries waiting in a queue to continue execution by invoking a callback + * function on each of them in turn. The queue is copied and emptied before invoking any callbacks, + * and only the waiters that were in the queue at the start of the call will be notified. + */ +void vdo_notify_all_waiters(struct wait_queue *queue, waiter_callback_fn callback, + void *context) +{ + /* + * Copy and empty the queue first, avoiding the possibility of an infinite loop if entries + * are returned to the queue by the callback function. + */ + struct wait_queue waiters; + + vdo_initialize_wait_queue(&waiters); + vdo_transfer_all_waiters(queue, &waiters); + + /* Drain the copied queue, invoking the callback on every entry. */ + while (vdo_has_waiters(&waiters)) + vdo_notify_next_waiter(&waiters, callback, context); +} + +/** + * vdo_get_first_waiter() - Return the waiter that is at the head end of a wait queue. + * @queue: The queue from which to get the first waiter. + * + * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. + */ +struct waiter *vdo_get_first_waiter(const struct wait_queue *queue) +{ + struct waiter *last_waiter = queue->last_waiter; + + if (last_waiter == NULL) { + /* There are no waiters, so we're done. */ + return NULL; + } + + /* The queue is circular, so the last entry links to the head of the queue. */ + return last_waiter->next_waiter; +} + +/** + * vdo_dequeue_matching_waiters() - Remove all waiters that match based on the specified matching + * method and append them to a wait_queue. + * @queue: The wait queue to process. + * @match_method: The method to determine matching. + * @match_context: Contextual info for the match method. + * @matched_queue: A wait_queue to store matches. + */ +void vdo_dequeue_matching_waiters(struct wait_queue *queue, waiter_match_fn match_method, + void *match_context, struct wait_queue *matched_queue) +{ + struct wait_queue matched_waiters, iteration_queue; + + vdo_initialize_wait_queue(&matched_waiters); + + vdo_initialize_wait_queue(&iteration_queue); + vdo_transfer_all_waiters(queue, &iteration_queue); + while (vdo_has_waiters(&iteration_queue)) { + struct waiter *waiter = vdo_dequeue_next_waiter(&iteration_queue); + + vdo_enqueue_waiter((match_method(waiter, match_context) ? + &matched_waiters : queue), waiter); + } + + vdo_transfer_all_waiters(&matched_waiters, matched_queue); +} + +/** + * vdo_dequeue_next_waiter() - Remove the first waiter from the head end of a wait queue. + * @queue: The wait queue from which to remove the first entry. + * + * The caller will be responsible for waking the waiter by invoking the correct callback function + * to resume its execution. + * + * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. + */ +struct waiter *vdo_dequeue_next_waiter(struct wait_queue *queue) +{ + struct waiter *first_waiter = vdo_get_first_waiter(queue); + struct waiter *last_waiter = queue->last_waiter; + + if (first_waiter == NULL) + return NULL; + + if (first_waiter == last_waiter) { + /* The queue has a single entry, so just empty it out by nulling the tail. */ + queue->last_waiter = NULL; + } else { + /* + * The queue has more than one entry, so splice the first waiter out of the + * circular queue. + */ + last_waiter->next_waiter = first_waiter->next_waiter; + } + + /* The waiter is no longer in a wait queue. */ + first_waiter->next_waiter = NULL; + queue->queue_length -= 1; + + return first_waiter; +} + +/** + * vdo_notify_next_waiter() - Notify the next entry waiting in a queue. + * @queue: The wait queue containing the waiter to notify. + * @callback: The function to call to notify the waiter, or NULL to invoke the callback field + * registered in the waiter. + * @context: The context to pass to the callback function. + * + * Notifies the next entry waiting in a queue to continue execution by invoking a callback function + * on it after removing it from the queue. + * + * Return: true if there was a waiter in the queue. + */ +bool vdo_notify_next_waiter(struct wait_queue *queue, waiter_callback_fn callback, + void *context) +{ + struct waiter *waiter = vdo_dequeue_next_waiter(queue); + + if (waiter == NULL) + return false; + + if (callback == NULL) + callback = waiter->callback; + (*callback)(waiter, context); + + return true; +} + +/** + * vdo_get_next_waiter() - Get the waiter after this one, for debug iteration. + * @queue: The wait queue. + * @waiter: A waiter. + * + * Return: The next waiter, or NULL. + */ +const struct waiter *vdo_get_next_waiter(const struct wait_queue *queue, + const struct waiter *waiter) +{ + struct waiter *first_waiter = vdo_get_first_waiter(queue); + + if (waiter == NULL) + return first_waiter; + + return ((waiter->next_waiter != first_waiter) ? waiter->next_waiter : NULL); +} diff --git a/drivers/md/dm-vdo/wait-queue.h b/drivers/md/dm-vdo/wait-queue.h new file mode 100644 index 000000000000..50f1e2a1ea67 --- /dev/null +++ b/drivers/md/dm-vdo/wait-queue.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_WAIT_QUEUE_H +#define VDO_WAIT_QUEUE_H + +#include +#include + +/** + * DOC: Wait queues. + * + * A wait queue is a circular list of entries waiting to be notified of a change in a condition. + * Keeping a circular list allows the queue structure to simply be a pointer to the tail (newest) + * entry in the queue, supporting constant-time enqueue and dequeue operations. A null pointer is + * an empty queue. + * + * An empty queue: + * queue0.last_waiter -> NULL + * + * A singleton queue: + * queue1.last_waiter -> entry1 -> entry1 -> [...] + * + * A three-element queue: + * queue2.last_waiter -> entry3 -> entry1 -> entry2 -> entry3 -> [...] + */ + +struct waiter; + +struct wait_queue { + /* The tail of the queue, the last (most recently added) entry */ + struct waiter *last_waiter; + /* The number of waiters currently in the queue */ + size_t queue_length; +}; + +/** + * typedef waiter_callback_fn - Callback type for functions which will be called to resume + * processing of a waiter after it has been removed from its wait + * queue. + */ +typedef void (*waiter_callback_fn)(struct waiter *waiter, void *context); + +/** + * typedef waiter_match_fn - Method type for waiter matching methods. + * + * A waiter_match_fn method returns false if the waiter does not match. + */ +typedef bool (*waiter_match_fn)(struct waiter *waiter, void *context); + +/* The queue entry structure for entries in a wait_queue. */ +struct waiter { + /* + * The next waiter in the queue. If this entry is the last waiter, then this is actually a + * pointer back to the head of the queue. + */ + struct waiter *next_waiter; + + /* Optional waiter-specific callback to invoke when waking this waiter. */ + waiter_callback_fn callback; +}; + +/** + * is_waiting() - Check whether a waiter is waiting. + * @waiter: The waiter to check. + * + * Return: true if the waiter is on some wait_queue. + */ +static inline bool vdo_is_waiting(struct waiter *waiter) +{ + return (waiter->next_waiter != NULL); +} + +/** + * initialize_wait_queue() - Initialize a wait queue. + * @queue: The queue to initialize. + */ +static inline void vdo_initialize_wait_queue(struct wait_queue *queue) +{ + *queue = (struct wait_queue) { + .last_waiter = NULL, + .queue_length = 0, + }; +} + +/** + * has_waiters() - Check whether a wait queue has any entries waiting in it. + * @queue: The queue to query. + * + * Return: true if there are any waiters in the queue. + */ +static inline bool __must_check vdo_has_waiters(const struct wait_queue *queue) +{ + return (queue->last_waiter != NULL); +} + +void vdo_enqueue_waiter(struct wait_queue *queue, struct waiter *waiter); + +void vdo_notify_all_waiters(struct wait_queue *queue, waiter_callback_fn callback, + void *context); + +bool vdo_notify_next_waiter(struct wait_queue *queue, waiter_callback_fn callback, + void *context); + +void vdo_transfer_all_waiters(struct wait_queue *from_queue, + struct wait_queue *to_queue); + +struct waiter *vdo_get_first_waiter(const struct wait_queue *queue); + +void vdo_dequeue_matching_waiters(struct wait_queue *queue, waiter_match_fn match_method, + void *match_context, struct wait_queue *matched_queue); + +struct waiter *vdo_dequeue_next_waiter(struct wait_queue *queue); + +/** + * count_waiters() - Count the number of waiters in a wait queue. + * @queue: The wait queue to query. + * + * Return: The number of waiters in the queue. + */ +static inline size_t __must_check vdo_count_waiters(const struct wait_queue *queue) +{ + return queue->queue_length; +} + +const struct waiter * __must_check vdo_get_next_waiter(const struct wait_queue *queue, + const struct waiter *waiter); + +#endif /* VDO_WAIT_QUEUE_H */ From 89f9b701f5b800daa76506d5bba197624e97ee44 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:47:35 -0500 Subject: [PATCH 0554/1038] dm vdo: add thread and synchronization utilities This patch adds utilities for managing and using named threads, as well as several locking and synchronization utilities. These utilities help dm-vdo minimize thread transitions and manage interactions between threads. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/thread-cond-var.c | 46 +++++++ drivers/md/dm-vdo/thread-device.c | 36 ++++++ drivers/md/dm-vdo/thread-device.h | 20 +++ drivers/md/dm-vdo/thread-registry.c | 92 ++++++++++++++ drivers/md/dm-vdo/thread-registry.h | 32 +++++ drivers/md/dm-vdo/uds-threads.c | 183 ++++++++++++++++++++++++++++ drivers/md/dm-vdo/uds-threads.h | 115 +++++++++++++++++ 7 files changed, 524 insertions(+) create mode 100644 drivers/md/dm-vdo/thread-cond-var.c create mode 100644 drivers/md/dm-vdo/thread-device.c create mode 100644 drivers/md/dm-vdo/thread-device.h create mode 100644 drivers/md/dm-vdo/thread-registry.c create mode 100644 drivers/md/dm-vdo/thread-registry.h create mode 100644 drivers/md/dm-vdo/uds-threads.c create mode 100644 drivers/md/dm-vdo/uds-threads.h diff --git a/drivers/md/dm-vdo/thread-cond-var.c b/drivers/md/dm-vdo/thread-cond-var.c new file mode 100644 index 000000000000..ed7f0b79ca0a --- /dev/null +++ b/drivers/md/dm-vdo/thread-cond-var.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include +#include + +#include "errors.h" +#include "time-utils.h" +#include "uds-threads.h" + +int uds_init_cond(struct cond_var *cv) +{ + init_waitqueue_head(&cv->wait_queue); + return UDS_SUCCESS; +} + +int uds_signal_cond(struct cond_var *cv) +{ + wake_up(&cv->wait_queue); + return UDS_SUCCESS; +} + +int uds_broadcast_cond(struct cond_var *cv) +{ + wake_up_all(&cv->wait_queue); + return UDS_SUCCESS; +} + +int uds_wait_cond(struct cond_var *cv, struct mutex *mutex) +{ + DEFINE_WAIT(__wait); + + prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); + uds_unlock_mutex(mutex); + schedule(); + finish_wait(&cv->wait_queue, &__wait); + uds_lock_mutex(mutex); + return UDS_SUCCESS; +} + +int uds_destroy_cond(struct cond_var *cv) +{ + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/thread-device.c b/drivers/md/dm-vdo/thread-device.c new file mode 100644 index 000000000000..b87de448a83b --- /dev/null +++ b/drivers/md/dm-vdo/thread-device.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "thread-device.h" + +#include "thread-registry.h" + +/* A registry of threads associated with device id numbers. */ +static struct thread_registry device_id_thread_registry; + +/* Any registered thread must be unregistered. */ +void uds_register_thread_device_id(struct registered_thread *new_thread, + unsigned int *id_ptr) +{ + uds_register_thread(&device_id_thread_registry, new_thread, id_ptr); +} + +void uds_unregister_thread_device_id(void) +{ + uds_unregister_thread(&device_id_thread_registry); +} + +int uds_get_thread_device_id(void) +{ + const unsigned int *pointer; + + pointer = uds_lookup_thread(&device_id_thread_registry); + return (pointer != NULL) ? *pointer : -1; +} + +void uds_initialize_thread_device_registry(void) +{ + uds_initialize_thread_registry(&device_id_thread_registry); +} diff --git a/drivers/md/dm-vdo/thread-device.h b/drivers/md/dm-vdo/thread-device.h new file mode 100644 index 000000000000..428b2908541d --- /dev/null +++ b/drivers/md/dm-vdo/thread-device.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_THREAD_DEVICE_H +#define UDS_THREAD_DEVICE_H + +#include "thread-registry.h" + +void uds_register_thread_device_id(struct registered_thread *new_thread, + unsigned int *id_ptr); + +void uds_unregister_thread_device_id(void); + +int uds_get_thread_device_id(void); + +void uds_initialize_thread_device_registry(void); + +#endif /* UDS_THREAD_DEVICE_H */ diff --git a/drivers/md/dm-vdo/thread-registry.c b/drivers/md/dm-vdo/thread-registry.c new file mode 100644 index 000000000000..8c887158c224 --- /dev/null +++ b/drivers/md/dm-vdo/thread-registry.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "thread-registry.h" + +#include + +#include "permassert.h" + +/* + * We need to be careful when using other facilities that may use thread registry functions in + * their normal operation. For example, we do not want to invoke the logger while holding a lock. + */ + +void uds_initialize_thread_registry(struct thread_registry *registry) +{ + INIT_LIST_HEAD(®istry->links); + spin_lock_init(®istry->lock); +} + +/* Register the current thread and associate it with a data pointer. */ +void uds_register_thread(struct thread_registry *registry, + struct registered_thread *new_thread, const void *pointer) +{ + struct registered_thread *thread; + bool found_it = false; + + INIT_LIST_HEAD(&new_thread->links); + new_thread->pointer = pointer; + new_thread->task = current; + + spin_lock(®istry->lock); + list_for_each_entry(thread, ®istry->links, links) { + if (thread->task == current) { + /* There should be no existing entry. */ + list_del_rcu(&thread->links); + found_it = true; + break; + } + } + list_add_tail_rcu(&new_thread->links, ®istry->links); + spin_unlock(®istry->lock); + + ASSERT_LOG_ONLY(!found_it, "new thread not already in registry"); + if (found_it) { + /* Ensure no RCU iterators see it before re-initializing. */ + synchronize_rcu(); + INIT_LIST_HEAD(&thread->links); + } +} + +void uds_unregister_thread(struct thread_registry *registry) +{ + struct registered_thread *thread; + bool found_it = false; + + spin_lock(®istry->lock); + list_for_each_entry(thread, ®istry->links, links) { + if (thread->task == current) { + list_del_rcu(&thread->links); + found_it = true; + break; + } + } + spin_unlock(®istry->lock); + + ASSERT_LOG_ONLY(found_it, "thread found in registry"); + if (found_it) { + /* Ensure no RCU iterators see it before re-initializing. */ + synchronize_rcu(); + INIT_LIST_HEAD(&thread->links); + } +} + +const void *uds_lookup_thread(struct thread_registry *registry) +{ + struct registered_thread *thread; + const void *result = NULL; + + rcu_read_lock(); + list_for_each_entry_rcu(thread, ®istry->links, links) { + if (thread->task == current) { + result = thread->pointer; + break; + } + } + rcu_read_unlock(); + + return result; +} diff --git a/drivers/md/dm-vdo/thread-registry.h b/drivers/md/dm-vdo/thread-registry.h new file mode 100644 index 000000000000..f70f755568a1 --- /dev/null +++ b/drivers/md/dm-vdo/thread-registry.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_THREAD_REGISTRY_H +#define UDS_THREAD_REGISTRY_H + +#include +#include + +struct thread_registry { + struct list_head links; + spinlock_t lock; +}; + +struct registered_thread { + struct list_head links; + const void *pointer; + struct task_struct *task; +}; + +void uds_initialize_thread_registry(struct thread_registry *registry); + +void uds_register_thread(struct thread_registry *registry, + struct registered_thread *new_thread, const void *pointer); + +void uds_unregister_thread(struct thread_registry *registry); + +const void *uds_lookup_thread(struct thread_registry *registry); + +#endif /* UDS_THREAD_REGISTRY_H */ diff --git a/drivers/md/dm-vdo/uds-threads.c b/drivers/md/dm-vdo/uds-threads.c new file mode 100644 index 000000000000..e7524617f9f8 --- /dev/null +++ b/drivers/md/dm-vdo/uds-threads.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "uds-threads.h" + +#include +#include +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" + +static struct hlist_head thread_list; +static struct mutex thread_mutex; +static atomic_t thread_once = ATOMIC_INIT(0); + +struct thread { + void (*thread_function)(void *thread_data); + void *thread_data; + struct hlist_node thread_links; + struct task_struct *thread_task; + struct completion thread_done; +}; + +enum { + ONCE_NOT_DONE = 0, + ONCE_IN_PROGRESS = 1, + ONCE_COMPLETE = 2, +}; + +/* Run a function once only, and record that fact in the atomic value. */ +void uds_perform_once(atomic_t *once, void (*function)(void)) +{ + for (;;) { + switch (atomic_cmpxchg(once, ONCE_NOT_DONE, ONCE_IN_PROGRESS)) { + case ONCE_NOT_DONE: + function(); + atomic_set_release(once, ONCE_COMPLETE); + return; + case ONCE_IN_PROGRESS: + cond_resched(); + break; + case ONCE_COMPLETE: + return; + default: + return; + } + } +} + +static void thread_init(void) +{ + mutex_init(&thread_mutex); +} + +static int thread_starter(void *arg) +{ + struct registered_thread allocating_thread; + struct thread *thread = arg; + + thread->thread_task = current; + uds_perform_once(&thread_once, thread_init); + mutex_lock(&thread_mutex); + hlist_add_head(&thread->thread_links, &thread_list); + mutex_unlock(&thread_mutex); + uds_register_allocating_thread(&allocating_thread, NULL); + thread->thread_function(thread->thread_data); + uds_unregister_allocating_thread(); + complete(&thread->thread_done); + return 0; +} + +int uds_create_thread(void (*thread_function)(void *), void *thread_data, + const char *name, struct thread **new_thread) +{ + char *name_colon = strchr(name, ':'); + char *my_name_colon = strchr(current->comm, ':'); + struct task_struct *task; + struct thread *thread; + int result; + + result = uds_allocate(1, struct thread, __func__, &thread); + if (result != UDS_SUCCESS) { + uds_log_warning("Error allocating memory for %s", name); + return result; + } + + thread->thread_function = thread_function; + thread->thread_data = thread_data; + init_completion(&thread->thread_done); + /* + * Start the thread, with an appropriate thread name. + * + * If the name supplied contains a colon character, use that name. This causes uds module + * threads to have names like "uds:callbackW" and the main test runner thread to be named + * "zub:runtest". + * + * Otherwise if the current thread has a name containing a colon character, prefix the name + * supplied with the name of the current thread up to (and including) the colon character. + * Thus when the "kvdo0:dedupeQ" thread opens an index session, all the threads associated + * with that index will have names like "kvdo0:foo". + * + * Otherwise just use the name supplied. This should be a rare occurrence. + */ + if ((name_colon == NULL) && (my_name_colon != NULL)) { + task = kthread_run(thread_starter, thread, "%.*s:%s", + (int) (my_name_colon - current->comm), current->comm, + name); + } else { + task = kthread_run(thread_starter, thread, "%s", name); + } + + if (IS_ERR(task)) { + uds_free(thread); + return PTR_ERR(task); + } + + *new_thread = thread; + return UDS_SUCCESS; +} + +int uds_join_threads(struct thread *thread) +{ + while (wait_for_completion_interruptible(&thread->thread_done) != 0) + /* empty loop */ + ; + + mutex_lock(&thread_mutex); + hlist_del(&thread->thread_links); + mutex_unlock(&thread_mutex); + uds_free(thread); + return UDS_SUCCESS; +} + +int uds_initialize_barrier(struct barrier *barrier, unsigned int thread_count) +{ + int result; + + result = uds_initialize_semaphore(&barrier->mutex, 1); + if (result != UDS_SUCCESS) + return result; + + barrier->arrived = 0; + barrier->thread_count = thread_count; + return uds_initialize_semaphore(&barrier->wait, 0); +} + +int uds_destroy_barrier(struct barrier *barrier) +{ + int result; + + result = uds_destroy_semaphore(&barrier->mutex); + if (result != UDS_SUCCESS) + return result; + + return uds_destroy_semaphore(&barrier->wait); +} + +int uds_enter_barrier(struct barrier *barrier) +{ + bool last_thread; + + uds_acquire_semaphore(&barrier->mutex); + last_thread = (++barrier->arrived == barrier->thread_count); + if (last_thread) { + int i; + + for (i = 1; i < barrier->thread_count; i++) + uds_release_semaphore(&barrier->wait); + + barrier->arrived = 0; + uds_release_semaphore(&barrier->mutex); + } else { + uds_release_semaphore(&barrier->mutex); + uds_acquire_semaphore(&barrier->wait); + } + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/uds-threads.h b/drivers/md/dm-vdo/uds-threads.h new file mode 100644 index 000000000000..9f3bf7991383 --- /dev/null +++ b/drivers/md/dm-vdo/uds-threads.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_THREADS_H +#define UDS_THREADS_H + +#include +#include +#include +#include +#include +#include + +#include "errors.h" +#include "time-utils.h" + +/* Thread and synchronization utilities for UDS */ + +struct cond_var { + wait_queue_head_t wait_queue; +}; + +struct thread; + +struct barrier { + /* Mutex for this barrier object */ + struct semaphore mutex; + /* Semaphore for threads waiting at the barrier */ + struct semaphore wait; + /* Number of threads which have arrived */ + int arrived; + /* Total number of threads using this barrier */ + int thread_count; +}; + +int __must_check uds_create_thread(void (*thread_function)(void *), void *thread_data, + const char *name, struct thread **new_thread); + +void uds_perform_once(atomic_t *once_state, void (*function) (void)); + +int uds_join_threads(struct thread *thread); + +int __must_check uds_initialize_barrier(struct barrier *barrier, + unsigned int thread_count); +int uds_destroy_barrier(struct barrier *barrier); +int uds_enter_barrier(struct barrier *barrier); + +int __must_check uds_init_cond(struct cond_var *cond); +int uds_signal_cond(struct cond_var *cond); +int uds_broadcast_cond(struct cond_var *cond); +int uds_wait_cond(struct cond_var *cond, struct mutex *mutex); +int uds_destroy_cond(struct cond_var *cond); + +static inline int __must_check uds_init_mutex(struct mutex *mutex) +{ + mutex_init(mutex); + return UDS_SUCCESS; +} + +static inline int uds_destroy_mutex(struct mutex *mutex) +{ + return UDS_SUCCESS; +} + +static inline void uds_lock_mutex(struct mutex *mutex) +{ + mutex_lock(mutex); +} + +static inline void uds_unlock_mutex(struct mutex *mutex) +{ + mutex_unlock(mutex); +} + +static inline int __must_check uds_initialize_semaphore(struct semaphore *semaphore, + unsigned int value) +{ + sema_init(semaphore, value); + return UDS_SUCCESS; +} + +static inline int uds_destroy_semaphore(struct semaphore *semaphore) +{ + return UDS_SUCCESS; +} + +static inline void uds_acquire_semaphore(struct semaphore *semaphore) +{ + /* + * Do not use down(semaphore). Instead use down_interruptible so that + * we do not get 120 second stall messages in kern.log. + */ + while (down_interruptible(semaphore) != 0) { + /* + * If we're called from a user-mode process (e.g., "dmsetup + * remove") while waiting for an operation that may take a + * while (e.g., UDS index save), and a signal is sent (SIGINT, + * SIGUSR2), then down_interruptible will not block. If that + * happens, sleep briefly to avoid keeping the CPU locked up in + * this loop. We could just call cond_resched, but then we'd + * still keep consuming CPU time slices and swamp other threads + * trying to do computational work. [VDO-4980] + */ + fsleep(1000); + } +} + +static inline void uds_release_semaphore(struct semaphore *semaphore) +{ + up(semaphore); +} + +#endif /* UDS_THREADS_H */ From d9e894d9b2ec9f3117f4d16400a3981c48bb0639 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:48:41 -0500 Subject: [PATCH 0555/1038] dm vdo: add specialized request queueing functionality This patch adds funnel_queue, a mostly lock-free multi-producer, single-consumer queue. It also adds the request queue used by the dm-vdo deduplication index, and the work_queue used by the dm-vdo data store. Both of these are built on top of funnel queue and are intended to support the dispatching of many short-running tasks. The work_queue also supports priorities. Finally, this patch adds vdo_completion, the structure which is enqueued on work_queues. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/completion.c | 140 ++++++ drivers/md/dm-vdo/completion.h | 152 ++++++ drivers/md/dm-vdo/cpu.h | 59 +++ drivers/md/dm-vdo/funnel-queue.c | 171 +++++++ drivers/md/dm-vdo/funnel-queue.h | 110 ++++ drivers/md/dm-vdo/funnel-requestqueue.c | 283 +++++++++++ drivers/md/dm-vdo/funnel-requestqueue.h | 31 ++ drivers/md/dm-vdo/funnel-workqueue.c | 638 ++++++++++++++++++++++++ drivers/md/dm-vdo/funnel-workqueue.h | 51 ++ 9 files changed, 1635 insertions(+) create mode 100644 drivers/md/dm-vdo/completion.c create mode 100644 drivers/md/dm-vdo/completion.h create mode 100644 drivers/md/dm-vdo/cpu.h create mode 100644 drivers/md/dm-vdo/funnel-queue.c create mode 100644 drivers/md/dm-vdo/funnel-queue.h create mode 100644 drivers/md/dm-vdo/funnel-requestqueue.c create mode 100644 drivers/md/dm-vdo/funnel-requestqueue.h create mode 100644 drivers/md/dm-vdo/funnel-workqueue.c create mode 100644 drivers/md/dm-vdo/funnel-workqueue.h diff --git a/drivers/md/dm-vdo/completion.c b/drivers/md/dm-vdo/completion.c new file mode 100644 index 000000000000..9e2381dc3683 --- /dev/null +++ b/drivers/md/dm-vdo/completion.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "completion.h" + +#include + +#include "logger.h" +#include "permassert.h" + +#include "status-codes.h" +#include "types.h" +#include "vio.h" +#include "vdo.h" + +/** + * DOC: vdo completions. + * + * Most of vdo's data structures are lock free, each either belonging to a single "zone," or + * divided into a number of zones whose accesses to the structure do not overlap. During normal + * operation, at most one thread will be operating in any given zone. Each zone has a + * vdo_work_queue which holds vdo_completions that are to be run in that zone. A completion may + * only be enqueued on one queue or operating in a single zone at a time. + * + * At each step of a multi-threaded operation, the completion performing the operation is given a + * callback, error handler, and thread id for the next step. A completion is "run" when it is + * operating on the correct thread (as specified by its callback_thread_id). If the value of its + * "result" field is an error (i.e. not VDO_SUCCESS), the function in its "error_handler" will be + * invoked. If the error_handler is NULL, or there is no error, the function set as its "callback" + * will be invoked. Generally, a completion will not be run directly, but rather will be + * "launched." In this case, it will check whether it is operating on the correct thread. If it is, + * it will run immediately. Otherwise, it will be enqueue on the vdo_work_queue associated with the + * completion's "callback_thread_id". When it is dequeued, it will be on the correct thread, and + * will get run. In some cases, the completion should get queued instead of running immediately, + * even if it is being launched from the correct thread. This is usually in cases where there is a + * long chain of callbacks, all on the same thread, which could overflow the stack. In such cases, + * the completion's "requeue" field should be set to true. Doing so will skip the current thread + * check and simply enqueue the completion. + * + * A completion may be "finished," in which case its "complete" field will be set to true before it + * is next run. It is a bug to attempt to set the result or re-finish a finished completion. + * Because a completion's fields are not safe to examine from any thread other than the one on + * which the completion is currently operating, this field is used only to aid in detecting + * programming errors. It can not be used for cross-thread checking on the status of an operation. + * A completion must be "reset" before it can be reused after it has been finished. Resetting will + * also clear any error from the result field. + **/ + +void vdo_initialize_completion(struct vdo_completion *completion, + struct vdo *vdo, + enum vdo_completion_type type) +{ + memset(completion, 0, sizeof(*completion)); + completion->vdo = vdo; + completion->type = type; + vdo_reset_completion(completion); +} + +static inline void assert_incomplete(struct vdo_completion *completion) +{ + ASSERT_LOG_ONLY(!completion->complete, "completion is not complete"); +} + +/** + * vdo_set_completion_result() - Set the result of a completion. + * + * Older errors will not be masked. + */ +void vdo_set_completion_result(struct vdo_completion *completion, int result) +{ + assert_incomplete(completion); + if (completion->result == VDO_SUCCESS) + completion->result = result; +} + +/** + * vdo_launch_completion_with_priority() - Run or enqueue a completion. + * @priority: The priority at which to enqueue the completion. + * + * If called on the correct thread (i.e. the one specified in the completion's callback_thread_id + * field) and not marked for requeue, the completion will be run immediately. Otherwise, the + * completion will be enqueued on the specified thread. + */ +void vdo_launch_completion_with_priority(struct vdo_completion *completion, + enum vdo_completion_priority priority) +{ + thread_id_t callback_thread = completion->callback_thread_id; + + if (completion->requeue || (callback_thread != vdo_get_callback_thread_id())) { + vdo_enqueue_completion(completion, priority); + return; + } + + vdo_run_completion(completion); +} + +/** vdo_finish_completion() - Mark a completion as complete and then launch it. */ +void vdo_finish_completion(struct vdo_completion *completion) +{ + assert_incomplete(completion); + completion->complete = true; + if (completion->callback != NULL) + vdo_launch_completion(completion); +} + +void vdo_enqueue_completion(struct vdo_completion *completion, + enum vdo_completion_priority priority) +{ + struct vdo *vdo = completion->vdo; + thread_id_t thread_id = completion->callback_thread_id; + + if (ASSERT(thread_id < vdo->thread_config.thread_count, + "thread_id %u (completion type %d) is less than thread count %u", + thread_id, completion->type, + vdo->thread_config.thread_count) != UDS_SUCCESS) + BUG(); + + completion->requeue = false; + completion->priority = priority; + completion->my_queue = NULL; + vdo_enqueue_work_queue(vdo->threads[thread_id].queue, completion); +} + +/** + * vdo_requeue_completion_if_needed() - Requeue a completion if not called on the specified thread. + * + * Return: True if the completion was requeued; callers may not access the completion in this case. + */ +bool vdo_requeue_completion_if_needed(struct vdo_completion *completion, + thread_id_t callback_thread_id) +{ + if (vdo_get_callback_thread_id() == callback_thread_id) + return false; + + completion->callback_thread_id = callback_thread_id; + vdo_enqueue_completion(completion, VDO_WORK_Q_DEFAULT_PRIORITY); + return true; +} diff --git a/drivers/md/dm-vdo/completion.h b/drivers/md/dm-vdo/completion.h new file mode 100644 index 000000000000..aa145d73a686 --- /dev/null +++ b/drivers/md/dm-vdo/completion.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_COMPLETION_H +#define VDO_COMPLETION_H + +#include "permassert.h" + +#include "status-codes.h" +#include "types.h" + +/** + * vdo_run_completion() - Run a completion's callback or error handler on the current thread. + * + * Context: This function must be called from the correct callback thread. + */ +static inline void vdo_run_completion(struct vdo_completion *completion) +{ + if ((completion->result != VDO_SUCCESS) && (completion->error_handler != NULL)) { + completion->error_handler(completion); + return; + } + + completion->callback(completion); +} + +void vdo_set_completion_result(struct vdo_completion *completion, int result); + +void vdo_initialize_completion(struct vdo_completion *completion, struct vdo *vdo, + enum vdo_completion_type type); + +/** + * vdo_reset_completion() - Reset a completion to a clean state, while keeping the type, vdo and + * parent information. + */ +static inline void vdo_reset_completion(struct vdo_completion *completion) +{ + completion->result = VDO_SUCCESS; + completion->complete = false; +} + +void vdo_launch_completion_with_priority(struct vdo_completion *completion, + enum vdo_completion_priority priority); + +/** + * vdo_launch_completion() - Launch a completion with default priority. + */ +static inline void vdo_launch_completion(struct vdo_completion *completion) +{ + vdo_launch_completion_with_priority(completion, VDO_WORK_Q_DEFAULT_PRIORITY); +} + +/** + * vdo_continue_completion() - Continue processing a completion. + * @result: The current result (will not mask older errors). + * + * Continue processing a completion by setting the current result and calling + * vdo_launch_completion(). + */ +static inline void vdo_continue_completion(struct vdo_completion *completion, int result) +{ + vdo_set_completion_result(completion, result); + vdo_launch_completion(completion); +} + +void vdo_finish_completion(struct vdo_completion *completion); + +/** + * vdo_fail_completion() - Set the result of a completion if it does not already have an error, + * then finish it. + */ +static inline void vdo_fail_completion(struct vdo_completion *completion, int result) +{ + vdo_set_completion_result(completion, result); + vdo_finish_completion(completion); +} + +/** + * vdo_assert_completion_type() - Assert that a completion is of the correct type. + * + * Return: VDO_SUCCESS or an error + */ +static inline int vdo_assert_completion_type(struct vdo_completion *completion, + enum vdo_completion_type expected) +{ + return ASSERT(expected == completion->type, + "completion type should be %u, not %u", expected, + completion->type); +} + +static inline void vdo_set_completion_callback(struct vdo_completion *completion, + vdo_action_fn callback, + thread_id_t callback_thread_id) +{ + completion->callback = callback; + completion->callback_thread_id = callback_thread_id; +} + +/** + * vdo_launch_completion_callback() - Set the callback for a completion and launch it immediately. + */ +static inline void vdo_launch_completion_callback(struct vdo_completion *completion, + vdo_action_fn callback, + thread_id_t callback_thread_id) +{ + vdo_set_completion_callback(completion, callback, callback_thread_id); + vdo_launch_completion(completion); +} + +/** + * vdo_prepare_completion() - Prepare a completion for launch. + * + * Resets the completion, and then sets its callback, error handler, callback thread, and parent. + */ +static inline void vdo_prepare_completion(struct vdo_completion *completion, + vdo_action_fn callback, + vdo_action_fn error_handler, + thread_id_t callback_thread_id, void *parent) +{ + vdo_reset_completion(completion); + vdo_set_completion_callback(completion, callback, callback_thread_id); + completion->error_handler = error_handler; + completion->parent = parent; +} + +/** + * vdo_prepare_completion_for_requeue() - Prepare a completion for launch ensuring that it will + * always be requeued. + * + * Resets the completion, and then sets its callback, error handler, callback thread, and parent. + */ +static inline void vdo_prepare_completion_for_requeue(struct vdo_completion *completion, + vdo_action_fn callback, + vdo_action_fn error_handler, + thread_id_t callback_thread_id, + void *parent) +{ + vdo_prepare_completion(completion, callback, error_handler, + callback_thread_id, parent); + completion->requeue = true; +} + +void vdo_enqueue_completion(struct vdo_completion *completion, + enum vdo_completion_priority priority); + + +bool vdo_requeue_completion_if_needed(struct vdo_completion *completion, + thread_id_t callback_thread_id); + +#endif /* VDO_COMPLETION_H */ diff --git a/drivers/md/dm-vdo/cpu.h b/drivers/md/dm-vdo/cpu.h new file mode 100644 index 000000000000..d6a2615ba657 --- /dev/null +++ b/drivers/md/dm-vdo/cpu.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_CPU_H +#define UDS_CPU_H + +#include + +/** + * uds_prefetch_address() - Minimize cache-miss latency by attempting to move data into a CPU cache + * before it is accessed. + * + * @address: the address to fetch (may be invalid) + * @for_write: must be constant at compile time--false if for reading, true if for writing + */ +static inline void uds_prefetch_address(const void *address, bool for_write) +{ + /* + * for_write won't be a constant if we are compiled with optimization turned off, in which + * case prefetching really doesn't matter. clang can't figure out that if for_write is a + * constant, it can be passed as the second, mandatorily constant argument to prefetch(), + * at least currently on llvm 12. + */ + if (__builtin_constant_p(for_write)) { + if (for_write) + __builtin_prefetch(address, true); + else + __builtin_prefetch(address, false); + } +} + +/** + * uds_prefetch_range() - Minimize cache-miss latency by attempting to move a range of addresses + * into a CPU cache before they are accessed. + * + * @start: the starting address to fetch (may be invalid) + * @size: the number of bytes in the address range + * @for_write: must be constant at compile time--false if for reading, true if for writing + */ +static inline void uds_prefetch_range(const void *start, unsigned int size, + bool for_write) +{ + /* + * Count the number of cache lines to fetch, allowing for the address range to span an + * extra cache line boundary due to address alignment. + */ + const char *address = (const char *) start; + unsigned int offset = ((uintptr_t) address % L1_CACHE_BYTES); + unsigned int cache_lines = (1 + ((size + offset) / L1_CACHE_BYTES)); + + while (cache_lines-- > 0) { + uds_prefetch_address(address, for_write); + address += L1_CACHE_BYTES; + } +} + +#endif /* UDS_CPU_H */ diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c new file mode 100644 index 000000000000..6940b282086d --- /dev/null +++ b/drivers/md/dm-vdo/funnel-queue.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "funnel-queue.h" + +#include "cpu.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "uds.h" + +int uds_make_funnel_queue(struct funnel_queue **queue_ptr) +{ + int result; + struct funnel_queue *queue; + + result = uds_allocate(1, struct funnel_queue, "funnel queue", &queue); + if (result != UDS_SUCCESS) + return result; + + /* + * Initialize the stub entry and put it in the queue, establishing the invariant that + * queue->newest and queue->oldest are never null. + */ + queue->stub.next = NULL; + queue->newest = &queue->stub; + queue->oldest = &queue->stub; + + *queue_ptr = queue; + return UDS_SUCCESS; +} + +void uds_free_funnel_queue(struct funnel_queue *queue) +{ + uds_free(queue); +} + +static struct funnel_queue_entry *get_oldest(struct funnel_queue *queue) +{ + /* + * Barrier requirements: We need a read barrier between reading a "next" field pointer + * value and reading anything it points to. There's an accompanying barrier in + * uds_funnel_queue_put() between its caller setting up the entry and making it visible. + */ + struct funnel_queue_entry *oldest = queue->oldest; + struct funnel_queue_entry *next = READ_ONCE(oldest->next); + + if (oldest == &queue->stub) { + /* + * When the oldest entry is the stub and it has no successor, the queue is + * logically empty. + */ + if (next == NULL) + return NULL; + /* + * The stub entry has a successor, so the stub can be dequeued and ignored without + * breaking the queue invariants. + */ + oldest = next; + queue->oldest = oldest; + next = READ_ONCE(oldest->next); + } + + /* + * We have a non-stub candidate to dequeue. If it lacks a successor, we'll need to put the + * stub entry back on the queue first. + */ + if (next == NULL) { + struct funnel_queue_entry *newest = READ_ONCE(queue->newest); + + if (oldest != newest) { + /* + * Another thread has already swung queue->newest atomically, but not yet + * assigned previous->next. The queue is really still empty. + */ + return NULL; + } + + /* + * Put the stub entry back on the queue, ensuring a successor will eventually be + * seen. + */ + uds_funnel_queue_put(queue, &queue->stub); + + /* Check again for a successor. */ + next = READ_ONCE(oldest->next); + if (next == NULL) { + /* + * We lost a race with a producer who swapped queue->newest before we did, + * but who hasn't yet updated previous->next. Try again later. + */ + return NULL; + } + } + + return oldest; +} + +/* + * Poll a queue, removing the oldest entry if the queue is not empty. This function must only be + * called from a single consumer thread. + */ +struct funnel_queue_entry *uds_funnel_queue_poll(struct funnel_queue *queue) +{ + struct funnel_queue_entry *oldest = get_oldest(queue); + + if (oldest == NULL) + return oldest; + + /* + * Dequeue the oldest entry and return it. Only one consumer thread may call this function, + * so no locking, atomic operations, or fences are needed; queue->oldest is owned by the + * consumer and oldest->next is never used by a producer thread after it is swung from NULL + * to non-NULL. + */ + queue->oldest = READ_ONCE(oldest->next); + /* + * Make sure the caller sees the proper stored data for this entry. Since we've already + * fetched the entry pointer we stored in "queue->oldest", this also ensures that on entry + * to the next call we'll properly see the dependent data. + */ + smp_rmb(); + /* + * If "oldest" is a very light-weight work item, we'll be looking for the next one very + * soon, so prefetch it now. + */ + uds_prefetch_address(queue->oldest, true); + WRITE_ONCE(oldest->next, NULL); + return oldest; +} + +/* + * Check whether the funnel queue is empty or not. If the queue is in a transition state with one + * or more entries being added such that the list view is incomplete, this function will report the + * queue as empty. + */ +bool uds_is_funnel_queue_empty(struct funnel_queue *queue) +{ + return get_oldest(queue) == NULL; +} + +/* + * Check whether the funnel queue is idle or not. If the queue has entries available to be + * retrieved, it is not idle. If the queue is in a transition state with one or more entries being + * added such that the list view is incomplete, it may not be possible to retrieve an entry with + * the uds_funnel_queue_poll() function, but the queue will not be considered idle. + */ +bool uds_is_funnel_queue_idle(struct funnel_queue *queue) +{ + /* + * Oldest is not the stub, so there's another entry, though if next is NULL we can't + * retrieve it yet. + */ + if (queue->oldest != &queue->stub) + return false; + + /* + * Oldest is the stub, but newest has been updated by _put(); either there's another, + * retrievable entry in the list, or the list is officially empty but in the intermediate + * state of having an entry added. + * + * Whether anything is retrievable depends on whether stub.next has been updated and become + * visible to us, but for idleness we don't care. And due to memory ordering in _put(), the + * update to newest would be visible to us at the same time or sooner. + */ + if (READ_ONCE(queue->newest) != &queue->stub) + return false; + + return true; +} diff --git a/drivers/md/dm-vdo/funnel-queue.h b/drivers/md/dm-vdo/funnel-queue.h new file mode 100644 index 000000000000..88a30c593fdc --- /dev/null +++ b/drivers/md/dm-vdo/funnel-queue.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_FUNNEL_QUEUE_H +#define UDS_FUNNEL_QUEUE_H + +#include +#include + +/* + * A funnel queue is a simple (almost) lock-free queue that accepts entries from multiple threads + * (multi-producer) and delivers them to a single thread (single-consumer). "Funnel" is an attempt + * to evoke the image of requests from more than one producer being "funneled down" to a single + * consumer. + * + * This is an unsynchronized but thread-safe data structure when used as intended. There is no + * mechanism to ensure that only one thread is consuming from the queue. If more than one thread + * attempts to consume from the queue, the resulting behavior is undefined. Clients must not + * directly access or manipulate the internals of the queue, which are only exposed for the purpose + * of allowing the very simple enqueue operation to be inlined. + * + * The implementation requires that a funnel_queue_entry structure (a link pointer) is embedded in + * the queue entries, and pointers to those structures are used exclusively by the queue. No macros + * are defined to template the queue, so the offset of the funnel_queue_entry in the records placed + * in the queue must all be the same so the client can derive their structure pointer from the + * entry pointer returned by uds_funnel_queue_poll(). + * + * Callers are wholly responsible for allocating and freeing the entries. Entries may be freed as + * soon as they are returned since this queue is not susceptible to the "ABA problem" present in + * many lock-free data structures. The queue is dynamically allocated to ensure cache-line + * alignment, but no other dynamic allocation is used. + * + * The algorithm is not actually 100% lock-free. There is a single point in uds_funnel_queue_put() + * at which a preempted producer will prevent the consumers from seeing items added to the queue by + * later producers, and only if the queue is short enough or the consumer fast enough for it to + * reach what was the end of the queue at the time of the preemption. + * + * The consumer function, uds_funnel_queue_poll(), will return NULL when the queue is empty. To + * wait for data to consume, spin (if safe) or combine the queue with a struct event_count to + * signal the presence of new entries. + */ + +/* This queue link structure must be embedded in client entries. */ +struct funnel_queue_entry { + /* The next (newer) entry in the queue. */ + struct funnel_queue_entry *next; +}; + +/* + * The dynamically allocated queue structure, which is allocated on a cache line boundary so the + * producer and consumer fields in the structure will land on separate cache lines. This should be + * consider opaque but it is exposed here so uds_funnel_queue_put() can be inlined. + */ +struct __aligned(L1_CACHE_BYTES) funnel_queue { + /* + * The producers' end of the queue, an atomically exchanged pointer that will never be + * NULL. + */ + struct funnel_queue_entry *newest; + + /* The consumer's end of the queue, which is owned by the consumer and never NULL. */ + struct funnel_queue_entry *oldest __aligned(L1_CACHE_BYTES); + + /* A dummy entry used to provide the non-NULL invariants above. */ + struct funnel_queue_entry stub; +}; + +int __must_check uds_make_funnel_queue(struct funnel_queue **queue_ptr); + +void uds_free_funnel_queue(struct funnel_queue *queue); + +/* + * Put an entry on the end of the queue. + * + * The entry pointer must be to the struct funnel_queue_entry embedded in the caller's data + * structure. The caller must be able to derive the address of the start of their data structure + * from the pointer that passed in here, so every entry in the queue must have the struct + * funnel_queue_entry at the same offset within the client's structure. + */ +static inline void uds_funnel_queue_put(struct funnel_queue *queue, + struct funnel_queue_entry *entry) +{ + struct funnel_queue_entry *previous; + + /* + * Barrier requirements: All stores relating to the entry ("next" pointer, containing data + * structure fields) must happen before the previous->next store making it visible to the + * consumer. Also, the entry's "next" field initialization to NULL must happen before any + * other producer threads can see the entry (the xchg) and try to update the "next" field. + * + * xchg implements a full barrier. + */ + WRITE_ONCE(entry->next, NULL); + previous = xchg(&queue->newest, entry); + /* + * Preemptions between these two statements hide the rest of the queue from the consumer, + * preventing consumption until the following assignment runs. + */ + WRITE_ONCE(previous->next, entry); +} + +struct funnel_queue_entry *__must_check uds_funnel_queue_poll(struct funnel_queue *queue); + +bool __must_check uds_is_funnel_queue_empty(struct funnel_queue *queue); + +bool __must_check uds_is_funnel_queue_idle(struct funnel_queue *queue); + +#endif /* UDS_FUNNEL_QUEUE_H */ diff --git a/drivers/md/dm-vdo/funnel-requestqueue.c b/drivers/md/dm-vdo/funnel-requestqueue.c new file mode 100644 index 000000000000..c8ba04c1089c --- /dev/null +++ b/drivers/md/dm-vdo/funnel-requestqueue.c @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "funnel-requestqueue.h" + +#include +#include +#include + +#include "funnel-queue.h" +#include "logger.h" +#include "memory-alloc.h" +#include "uds-threads.h" + +/* + * This queue will attempt to handle requests in reasonably sized batches instead of reacting + * immediately to each new request. The wait time between batches is dynamically adjusted up or + * down to try to balance responsiveness against wasted thread run time. + * + * If the wait time becomes long enough, the queue will become dormant and must be explicitly + * awoken when a new request is enqueued. The enqueue operation updates "newest" in the funnel + * queue via xchg (which is a memory barrier), and later checks "dormant" to decide whether to do a + * wakeup of the worker thread. + * + * When deciding to go to sleep, the worker thread sets "dormant" and then examines "newest" to + * decide if the funnel queue is idle. In dormant mode, the last examination of "newest" before + * going to sleep is done inside the wait_event_interruptible() macro, after a point where one or + * more memory barriers have been issued. (Preparing to sleep uses spin locks.) Even if the funnel + * queue's "next" field update isn't visible yet to make the entry accessible, its existence will + * kick the worker thread out of dormant mode and back into timer-based mode. + * + * Unbatched requests are used to communicate between different zone threads and will also cause + * the queue to awaken immediately. + */ + +enum { + NANOSECOND = 1, + MICROSECOND = 1000 * NANOSECOND, + MILLISECOND = 1000 * MICROSECOND, + DEFAULT_WAIT_TIME = 20 * MICROSECOND, + MINIMUM_WAIT_TIME = DEFAULT_WAIT_TIME / 2, + MAXIMUM_WAIT_TIME = MILLISECOND, + MINIMUM_BATCH = 32, + MAXIMUM_BATCH = 64, +}; + +struct uds_request_queue { + /* Wait queue for synchronizing producers and consumer */ + struct wait_queue_head wait_head; + /* Function to process a request */ + uds_request_queue_processor_fn processor; + /* Queue of new incoming requests */ + struct funnel_queue *main_queue; + /* Queue of old requests to retry */ + struct funnel_queue *retry_queue; + /* The thread id of the worker thread */ + struct thread *thread; + /* True if the worker was started */ + bool started; + /* When true, requests can be enqueued */ + bool running; + /* A flag set when the worker is waiting without a timeout */ + atomic_t dormant; +}; + +static inline struct uds_request *poll_queues(struct uds_request_queue *queue) +{ + struct funnel_queue_entry *entry; + + entry = uds_funnel_queue_poll(queue->retry_queue); + if (entry != NULL) + return container_of(entry, struct uds_request, queue_link); + + entry = uds_funnel_queue_poll(queue->main_queue); + if (entry != NULL) + return container_of(entry, struct uds_request, queue_link); + + return NULL; +} + +static inline bool are_queues_idle(struct uds_request_queue *queue) +{ + return uds_is_funnel_queue_idle(queue->retry_queue) && + uds_is_funnel_queue_idle(queue->main_queue); +} + +/* + * Determine if there is a next request to process, and return it if there is. Also return flags + * indicating whether the worker thread can sleep (for the use of wait_event() macros) and whether + * the thread did sleep before returning a new request. + */ +static inline bool dequeue_request(struct uds_request_queue *queue, + struct uds_request **request_ptr, bool *waited_ptr) +{ + struct uds_request *request = poll_queues(queue); + + if (request != NULL) { + *request_ptr = request; + return true; + } + + if (!READ_ONCE(queue->running)) { + /* Wake the worker thread so it can exit. */ + *request_ptr = NULL; + return true; + } + + *request_ptr = NULL; + *waited_ptr = true; + return false; +} + +static void wait_for_request(struct uds_request_queue *queue, bool dormant, + unsigned long timeout, struct uds_request **request, + bool *waited) +{ + if (dormant) { + wait_event_interruptible(queue->wait_head, + (dequeue_request(queue, request, waited) || + !are_queues_idle(queue))); + return; + } + + wait_event_interruptible_hrtimeout(queue->wait_head, + dequeue_request(queue, request, waited), + ns_to_ktime(timeout)); +} + +static void request_queue_worker(void *arg) +{ + struct uds_request_queue *queue = arg; + struct uds_request *request = NULL; + unsigned long time_batch = DEFAULT_WAIT_TIME; + bool dormant = atomic_read(&queue->dormant); + bool waited = false; + long current_batch = 0; + + for (;;) { + wait_for_request(queue, dormant, time_batch, &request, &waited); + if (likely(request != NULL)) { + current_batch++; + queue->processor(request); + } else if (!READ_ONCE(queue->running)) { + break; + } + + if (dormant) { + /* + * The queue has been roused from dormancy. Clear the flag so enqueuers can + * stop broadcasting. No fence is needed for this transition. + */ + atomic_set(&queue->dormant, false); + dormant = false; + time_batch = DEFAULT_WAIT_TIME; + } else if (waited) { + /* + * We waited for this request to show up. Adjust the wait time to smooth + * out the batch size. + */ + if (current_batch < MINIMUM_BATCH) { + /* + * If the last batch of requests was too small, increase the wait + * time. + */ + time_batch += time_batch / 4; + if (time_batch >= MAXIMUM_WAIT_TIME) { + atomic_set(&queue->dormant, true); + dormant = true; + } + } else if (current_batch > MAXIMUM_BATCH) { + /* + * If the last batch of requests was too large, decrease the wait + * time. + */ + time_batch -= time_batch / 4; + if (time_batch < MINIMUM_WAIT_TIME) + time_batch = MINIMUM_WAIT_TIME; + } + current_batch = 0; + } + } + + /* + * Ensure that we process any remaining requests that were enqueued before trying to shut + * down. The corresponding write barrier is in uds_request_queue_finish(). + */ + smp_rmb(); + while ((request = poll_queues(queue)) != NULL) + queue->processor(request); +} + +int uds_make_request_queue(const char *queue_name, + uds_request_queue_processor_fn processor, + struct uds_request_queue **queue_ptr) +{ + int result; + struct uds_request_queue *queue; + + result = uds_allocate(1, struct uds_request_queue, __func__, &queue); + if (result != UDS_SUCCESS) + return result; + + queue->processor = processor; + queue->running = true; + atomic_set(&queue->dormant, false); + init_waitqueue_head(&queue->wait_head); + + result = uds_make_funnel_queue(&queue->main_queue); + if (result != UDS_SUCCESS) { + uds_request_queue_finish(queue); + return result; + } + + result = uds_make_funnel_queue(&queue->retry_queue); + if (result != UDS_SUCCESS) { + uds_request_queue_finish(queue); + return result; + } + + result = uds_create_thread(request_queue_worker, queue, queue_name, + &queue->thread); + if (result != UDS_SUCCESS) { + uds_request_queue_finish(queue); + return result; + } + + queue->started = true; + *queue_ptr = queue; + return UDS_SUCCESS; +} + +static inline void wake_up_worker(struct uds_request_queue *queue) +{ + if (wq_has_sleeper(&queue->wait_head)) + wake_up(&queue->wait_head); +} + +void uds_request_queue_enqueue(struct uds_request_queue *queue, + struct uds_request *request) +{ + struct funnel_queue *sub_queue; + bool unbatched = request->unbatched; + + sub_queue = request->requeued ? queue->retry_queue : queue->main_queue; + uds_funnel_queue_put(sub_queue, &request->queue_link); + + /* + * We must wake the worker thread when it is dormant. A read fence isn't needed here since + * we know the queue operation acts as one. + */ + if (atomic_read(&queue->dormant) || unbatched) + wake_up_worker(queue); +} + +void uds_request_queue_finish(struct uds_request_queue *queue) +{ + int result; + + if (queue == NULL) + return; + + /* + * This memory barrier ensures that any requests we queued will be seen. The point is that + * when dequeue_request() sees the following update to the running flag, it will also be + * able to see any change we made to a next field in the funnel queue entry. The + * corresponding read barrier is in request_queue_worker(). + */ + smp_wmb(); + WRITE_ONCE(queue->running, false); + + if (queue->started) { + wake_up_worker(queue); + result = uds_join_threads(queue->thread); + if (result != UDS_SUCCESS) + uds_log_warning_strerror(result, "Failed to join worker thread"); + } + + uds_free_funnel_queue(queue->main_queue); + uds_free_funnel_queue(queue->retry_queue); + uds_free(queue); +} diff --git a/drivers/md/dm-vdo/funnel-requestqueue.h b/drivers/md/dm-vdo/funnel-requestqueue.h new file mode 100644 index 000000000000..e74c231fe269 --- /dev/null +++ b/drivers/md/dm-vdo/funnel-requestqueue.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_REQUEST_QUEUE_H +#define UDS_REQUEST_QUEUE_H + +#include "uds.h" + +/* + * A simple request queue which will handle new requests in the order in which they are received, + * and will attempt to handle requeued requests before new ones. However, the nature of the + * implementation means that it cannot guarantee this ordering; the prioritization is merely a + * hint. + */ + +struct uds_request_queue; + +typedef void (*uds_request_queue_processor_fn)(struct uds_request *); + +int __must_check uds_make_request_queue(const char *queue_name, + uds_request_queue_processor_fn processor, + struct uds_request_queue **queue_ptr); + +void uds_request_queue_enqueue(struct uds_request_queue *queue, + struct uds_request *request); + +void uds_request_queue_finish(struct uds_request_queue *queue); + +#endif /* UDS_REQUEST_QUEUE_H */ diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c new file mode 100644 index 000000000000..8f0ada13e549 --- /dev/null +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -0,0 +1,638 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "funnel-workqueue.h" + +#include +#include +#include +#include +#include +#include + +#include "funnel-queue.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" + +#include "completion.h" +#include "status-codes.h" + +static DEFINE_PER_CPU(unsigned int, service_queue_rotor); + +/** + * DOC: Work queue definition. + * + * There are two types of work queues: simple, with one worker thread, and round-robin, which uses + * a group of the former to do the work, and assigns work to them in round-robin fashion (roughly). + * Externally, both are represented via the same common sub-structure, though there's actually not + * a great deal of overlap between the two types internally. + */ +struct vdo_work_queue { + /* Name of just the work queue (e.g., "cpuQ12") */ + char *name; + bool round_robin_mode; + struct vdo_thread *owner; + /* Life cycle functions, etc */ + const struct vdo_work_queue_type *type; +}; + +struct simple_work_queue { + struct vdo_work_queue common; + struct funnel_queue *priority_lists[VDO_WORK_Q_MAX_PRIORITY + 1]; + void *private; + + /* + * The fields above are unchanged after setup but often read, and are good candidates for + * caching -- and if the max priority is 2, just fit in one x86-64 cache line if aligned. + * The fields below are often modified as we sleep and wake, so we want a separate cache + * line for performance. + */ + + /* Any (0 or 1) worker threads waiting for new work to do */ + wait_queue_head_t waiting_worker_threads ____cacheline_aligned; + /* Hack to reduce wakeup calls if the worker thread is running */ + atomic_t idle; + + /* These are infrequently used so in terms of performance we don't care where they land. */ + struct task_struct *thread; + /* Notify creator once worker has initialized */ + struct completion *started; +}; + +struct round_robin_work_queue { + struct vdo_work_queue common; + struct simple_work_queue **service_queues; + unsigned int num_service_queues; +}; + +static inline struct simple_work_queue *as_simple_work_queue(struct vdo_work_queue *queue) +{ + return ((queue == NULL) ? + NULL : container_of(queue, struct simple_work_queue, common)); +} + +static inline struct round_robin_work_queue *as_round_robin_work_queue(struct vdo_work_queue *queue) +{ + return ((queue == NULL) ? + NULL : + container_of(queue, struct round_robin_work_queue, common)); +} + +/* Processing normal completions. */ + +/* + * Dequeue and return the next waiting completion, if any. + * + * We scan the funnel queues from highest priority to lowest, once; there is therefore a race + * condition where a high-priority completion can be enqueued followed by a lower-priority one, and + * we'll grab the latter (but we'll catch the high-priority item on the next call). If strict + * enforcement of priorities becomes necessary, this function will need fixing. + */ +static struct vdo_completion *poll_for_completion(struct simple_work_queue *queue) +{ + int i; + + for (i = queue->common.type->max_priority; i >= 0; i--) { + struct funnel_queue_entry *link = uds_funnel_queue_poll(queue->priority_lists[i]); + + if (link != NULL) + return container_of(link, struct vdo_completion, work_queue_entry_link); + } + + return NULL; +} + +static void enqueue_work_queue_completion(struct simple_work_queue *queue, + struct vdo_completion *completion) +{ + ASSERT_LOG_ONLY(completion->my_queue == NULL, + "completion %px (fn %px) to enqueue (%px) is not already queued (%px)", + completion, completion->callback, queue, completion->my_queue); + if (completion->priority == VDO_WORK_Q_DEFAULT_PRIORITY) + completion->priority = queue->common.type->default_priority; + + if (ASSERT(completion->priority <= queue->common.type->max_priority, + "priority is in range for queue") != VDO_SUCCESS) + completion->priority = 0; + + completion->my_queue = &queue->common; + + /* Funnel queue handles the synchronization for the put. */ + uds_funnel_queue_put(queue->priority_lists[completion->priority], + &completion->work_queue_entry_link); + + /* + * Due to how funnel queue synchronization is handled (just atomic operations), the + * simplest safe implementation here would be to wake-up any waiting threads after + * enqueueing each item. Even if the funnel queue is not empty at the time of adding an + * item to the queue, the consumer thread may not see this since it is not guaranteed to + * have the same view of the queue as a producer thread. + * + * However, the above is wasteful so instead we attempt to minimize the number of thread + * wakeups. Using an idle flag, and careful ordering using memory barriers, we should be + * able to determine when the worker thread might be asleep or going to sleep. We use + * cmpxchg to try to take ownership (vs other producer threads) of the responsibility for + * waking the worker thread, so multiple wakeups aren't tried at once. + * + * This was tuned for some x86 boxes that were handy; it's untested whether doing the read + * first is any better or worse for other platforms, even other x86 configurations. + */ + smp_mb(); + if ((atomic_read(&queue->idle) != 1) || (atomic_cmpxchg(&queue->idle, 1, 0) != 1)) + return; + + /* There's a maximum of one thread in this list. */ + wake_up(&queue->waiting_worker_threads); +} + +static void run_start_hook(struct simple_work_queue *queue) +{ + if (queue->common.type->start != NULL) + queue->common.type->start(queue->private); +} + +static void run_finish_hook(struct simple_work_queue *queue) +{ + if (queue->common.type->finish != NULL) + queue->common.type->finish(queue->private); +} + +/* + * Wait for the next completion to process, or until kthread_should_stop indicates that it's time + * for us to shut down. + * + * If kthread_should_stop says it's time to stop but we have pending completions return a + * completion. + * + * Also update statistics relating to scheduler interactions. + */ +static struct vdo_completion *wait_for_next_completion(struct simple_work_queue *queue) +{ + struct vdo_completion *completion; + DEFINE_WAIT(wait); + + while (true) { + prepare_to_wait(&queue->waiting_worker_threads, &wait, + TASK_INTERRUPTIBLE); + /* + * Don't set the idle flag until a wakeup will not be lost. + * + * Force synchronization between setting the idle flag and checking the funnel + * queue; the producer side will do them in the reverse order. (There's still a + * race condition we've chosen to allow, because we've got a timeout below that + * unwedges us if we hit it, but this may narrow the window a little.) + */ + atomic_set(&queue->idle, 1); + smp_mb(); /* store-load barrier between "idle" and funnel queue */ + + completion = poll_for_completion(queue); + if (completion != NULL) + break; + + /* + * We need to check for thread-stop after setting TASK_INTERRUPTIBLE state up + * above. Otherwise, schedule() will put the thread to sleep and might miss a + * wakeup from kthread_stop() call in vdo_finish_work_queue(). + */ + if (kthread_should_stop()) + break; + + schedule(); + + /* + * Most of the time when we wake, it should be because there's work to do. If it + * was a spurious wakeup, continue looping. + */ + completion = poll_for_completion(queue); + if (completion != NULL) + break; + } + + finish_wait(&queue->waiting_worker_threads, &wait); + atomic_set(&queue->idle, 0); + + return completion; +} + +static void process_completion(struct simple_work_queue *queue, + struct vdo_completion *completion) +{ + if (ASSERT(completion->my_queue == &queue->common, + "completion %px from queue %px marked as being in this queue (%px)", + completion, queue, completion->my_queue) == UDS_SUCCESS) + completion->my_queue = NULL; + + vdo_run_completion(completion); +} + +static void service_work_queue(struct simple_work_queue *queue) +{ + run_start_hook(queue); + + while (true) { + struct vdo_completion *completion = poll_for_completion(queue); + + if (completion == NULL) + completion = wait_for_next_completion(queue); + + if (completion == NULL) { + /* No completions but kthread_should_stop() was triggered. */ + break; + } + + process_completion(queue, completion); + + /* + * Be friendly to a CPU that has other work to do, if the kernel has told us to. + * This speeds up some performance tests; that "other work" might include other VDO + * threads. + */ + if (need_resched()) + cond_resched(); + } + + run_finish_hook(queue); +} + +static int work_queue_runner(void *ptr) +{ + struct simple_work_queue *queue = ptr; + + complete(queue->started); + service_work_queue(queue); + return 0; +} + +/* Creation & teardown */ + +static void free_simple_work_queue(struct simple_work_queue *queue) +{ + unsigned int i; + + for (i = 0; i <= VDO_WORK_Q_MAX_PRIORITY; i++) + uds_free_funnel_queue(queue->priority_lists[i]); + uds_free(queue->common.name); + uds_free(queue); +} + +static void free_round_robin_work_queue(struct round_robin_work_queue *queue) +{ + struct simple_work_queue **queue_table = queue->service_queues; + unsigned int count = queue->num_service_queues; + unsigned int i; + + queue->service_queues = NULL; + + for (i = 0; i < count; i++) + free_simple_work_queue(queue_table[i]); + uds_free(queue_table); + uds_free(queue->common.name); + uds_free(queue); +} + +void vdo_free_work_queue(struct vdo_work_queue *queue) +{ + if (queue == NULL) + return; + + vdo_finish_work_queue(queue); + + if (queue->round_robin_mode) + free_round_robin_work_queue(as_round_robin_work_queue(queue)); + else + free_simple_work_queue(as_simple_work_queue(queue)); +} + +static int make_simple_work_queue(const char *thread_name_prefix, const char *name, + struct vdo_thread *owner, void *private, + const struct vdo_work_queue_type *type, + struct simple_work_queue **queue_ptr) +{ + DECLARE_COMPLETION_ONSTACK(started); + struct simple_work_queue *queue; + int i; + struct task_struct *thread = NULL; + int result; + + ASSERT_LOG_ONLY((type->max_priority <= VDO_WORK_Q_MAX_PRIORITY), + "queue priority count %u within limit %u", type->max_priority, + VDO_WORK_Q_MAX_PRIORITY); + + result = uds_allocate(1, struct simple_work_queue, "simple work queue", &queue); + if (result != UDS_SUCCESS) + return result; + + queue->private = private; + queue->started = &started; + queue->common.type = type; + queue->common.owner = owner; + init_waitqueue_head(&queue->waiting_worker_threads); + + result = uds_duplicate_string(name, "queue name", &queue->common.name); + if (result != VDO_SUCCESS) { + uds_free(queue); + return -ENOMEM; + } + + for (i = 0; i <= type->max_priority; i++) { + result = uds_make_funnel_queue(&queue->priority_lists[i]); + if (result != UDS_SUCCESS) { + free_simple_work_queue(queue); + return result; + } + } + + thread = kthread_run(work_queue_runner, queue, "%s:%s", thread_name_prefix, + queue->common.name); + if (IS_ERR(thread)) { + free_simple_work_queue(queue); + return (int) PTR_ERR(thread); + } + + queue->thread = thread; + + /* + * If we don't wait to ensure the thread is running VDO code, a quick kthread_stop (due to + * errors elsewhere) could cause it to never get as far as running VDO, skipping the + * cleanup code. + * + * Eventually we should just make that path safe too, and then we won't need this + * synchronization. + */ + wait_for_completion(&started); + + *queue_ptr = queue; + return UDS_SUCCESS; +} + +/** + * vdo_make_work_queue() - Create a work queue; if multiple threads are requested, completions will + * be distributed to them in round-robin fashion. + * + * Each queue is associated with a struct vdo_thread which has a single vdo thread id. Regardless + * of the actual number of queues and threads allocated here, code outside of the queue + * implementation will treat this as a single zone. + */ +int vdo_make_work_queue(const char *thread_name_prefix, const char *name, + struct vdo_thread *owner, const struct vdo_work_queue_type *type, + unsigned int thread_count, void *thread_privates[], + struct vdo_work_queue **queue_ptr) +{ + struct round_robin_work_queue *queue; + int result; + char thread_name[TASK_COMM_LEN]; + unsigned int i; + + if (thread_count == 1) { + struct simple_work_queue *simple_queue; + void *context = ((thread_privates != NULL) ? thread_privates[0] : NULL); + + result = make_simple_work_queue(thread_name_prefix, name, owner, context, + type, &simple_queue); + if (result == VDO_SUCCESS) + *queue_ptr = &simple_queue->common; + return result; + } + + result = uds_allocate(1, struct round_robin_work_queue, "round-robin work queue", + &queue); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(thread_count, struct simple_work_queue *, + "subordinate work queues", &queue->service_queues); + if (result != UDS_SUCCESS) { + uds_free(queue); + return result; + } + + queue->num_service_queues = thread_count; + queue->common.round_robin_mode = true; + queue->common.owner = owner; + + result = uds_duplicate_string(name, "queue name", &queue->common.name); + if (result != VDO_SUCCESS) { + uds_free(queue->service_queues); + uds_free(queue); + return -ENOMEM; + } + + *queue_ptr = &queue->common; + + for (i = 0; i < thread_count; i++) { + void *context = ((thread_privates != NULL) ? thread_privates[i] : NULL); + + snprintf(thread_name, sizeof(thread_name), "%s%u", name, i); + result = make_simple_work_queue(thread_name_prefix, thread_name, owner, + context, type, &queue->service_queues[i]); + if (result != VDO_SUCCESS) { + queue->num_service_queues = i; + /* Destroy previously created subordinates. */ + vdo_free_work_queue(uds_forget(*queue_ptr)); + return result; + } + } + + return VDO_SUCCESS; +} + +static void finish_simple_work_queue(struct simple_work_queue *queue) +{ + if (queue->thread == NULL) + return; + + /* Tells the worker thread to shut down and waits for it to exit. */ + kthread_stop(queue->thread); + queue->thread = NULL; +} + +static void finish_round_robin_work_queue(struct round_robin_work_queue *queue) +{ + struct simple_work_queue **queue_table = queue->service_queues; + unsigned int count = queue->num_service_queues; + unsigned int i; + + for (i = 0; i < count; i++) + finish_simple_work_queue(queue_table[i]); +} + +/* No enqueueing of completions should be done once this function is called. */ +void vdo_finish_work_queue(struct vdo_work_queue *queue) +{ + if (queue == NULL) + return; + + if (queue->round_robin_mode) + finish_round_robin_work_queue(as_round_robin_work_queue(queue)); + else + finish_simple_work_queue(as_simple_work_queue(queue)); +} + +/* Debugging dumps */ + +static void dump_simple_work_queue(struct simple_work_queue *queue) +{ + const char *thread_status = "no threads"; + char task_state_report = '-'; + + if (queue->thread != NULL) { + task_state_report = task_state_to_char(queue->thread); + thread_status = atomic_read(&queue->idle) ? "idle" : "running"; + } + + uds_log_info("workQ %px (%s) %s (%c)", &queue->common, queue->common.name, + thread_status, task_state_report); + + /* ->waiting_worker_threads wait queue status? anyone waiting? */ +} + +/* + * Write to the buffer some info about the completion, for logging. Since the common use case is + * dumping info about a lot of completions to syslog all at once, the format favors brevity over + * readability. + */ +void vdo_dump_work_queue(struct vdo_work_queue *queue) +{ + if (queue->round_robin_mode) { + struct round_robin_work_queue *round_robin = as_round_robin_work_queue(queue); + unsigned int i; + + for (i = 0; i < round_robin->num_service_queues; i++) + dump_simple_work_queue(round_robin->service_queues[i]); + } else { + dump_simple_work_queue(as_simple_work_queue(queue)); + } +} + +static void get_function_name(void *pointer, char *buffer, size_t buffer_length) +{ + if (pointer == NULL) { + /* + * Format "%ps" logs a null pointer as "(null)" with a bunch of leading spaces. We + * sometimes use this when logging lots of data; don't be so verbose. + */ + strscpy(buffer, "-", buffer_length); + } else { + /* + * Use a pragma to defeat gcc's format checking, which doesn't understand that + * "%ps" actually does support a precision spec in Linux kernel code. + */ + char *space; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" + snprintf(buffer, buffer_length, "%.*ps", buffer_length - 1, pointer); +#pragma GCC diagnostic pop + + space = strchr(buffer, ' '); + if (space != NULL) + *space = '\0'; + } +} + +void vdo_dump_completion_to_buffer(struct vdo_completion *completion, char *buffer, + size_t length) +{ + size_t current_length = + scnprintf(buffer, length, "%.*s/", TASK_COMM_LEN, + (completion->my_queue == NULL ? "-" : completion->my_queue->name)); + + if (current_length < length - 1) { + get_function_name((void *) completion->callback, buffer + current_length, + length - current_length); + } +} + +/* Completion submission */ +/* + * If the completion has a timeout that has already passed, the timeout handler function may be + * invoked by this function. + */ +void vdo_enqueue_work_queue(struct vdo_work_queue *queue, + struct vdo_completion *completion) +{ + /* + * Convert the provided generic vdo_work_queue to the simple_work_queue to actually queue + * on. + */ + struct simple_work_queue *simple_queue = NULL; + + if (!queue->round_robin_mode) { + simple_queue = as_simple_work_queue(queue); + } else { + struct round_robin_work_queue *round_robin = as_round_robin_work_queue(queue); + + /* + * It shouldn't be a big deal if the same rotor gets used for multiple work queues. + * Any patterns that might develop are likely to be disrupted by random ordering of + * multiple completions and migration between cores, unless the load is so light as + * to be regular in ordering of tasks and the threads are confined to individual + * cores; with a load that light we won't care. + */ + unsigned int rotor = this_cpu_inc_return(service_queue_rotor); + unsigned int index = rotor % round_robin->num_service_queues; + + simple_queue = round_robin->service_queues[index]; + } + + enqueue_work_queue_completion(simple_queue, completion); +} + +/* Misc */ + +/* + * Return the work queue pointer recorded at initialization time in the work-queue stack handle + * initialized on the stack of the current thread, if any. + */ +static struct simple_work_queue *get_current_thread_work_queue(void) +{ + /* + * In interrupt context, if a vdo thread is what got interrupted, the calls below will find + * the queue for the thread which was interrupted. However, the interrupted thread may have + * been processing a completion, in which case starting to process another would violate + * our concurrency assumptions. + */ + if (in_interrupt()) + return NULL; + + if (kthread_func(current) != work_queue_runner) + /* Not a VDO work queue thread. */ + return NULL; + + return kthread_data(current); +} + +struct vdo_work_queue *vdo_get_current_work_queue(void) +{ + struct simple_work_queue *queue = get_current_thread_work_queue(); + + return (queue == NULL) ? NULL : &queue->common; +} + +struct vdo_thread *vdo_get_work_queue_owner(struct vdo_work_queue *queue) +{ + return queue->owner; +} + +/** + * vdo_get_work_queue_private_data() - Returns the private data for the current thread's work + * queue, or NULL if none or if the current thread is not a + * work queue thread. + */ +void *vdo_get_work_queue_private_data(void) +{ + struct simple_work_queue *queue = get_current_thread_work_queue(); + + return (queue != NULL) ? queue->private : NULL; +} + +bool vdo_work_queue_type_is(struct vdo_work_queue *queue, + const struct vdo_work_queue_type *type) +{ + return (queue->type == type); +} diff --git a/drivers/md/dm-vdo/funnel-workqueue.h b/drivers/md/dm-vdo/funnel-workqueue.h new file mode 100644 index 000000000000..b5be6e9e83bc --- /dev/null +++ b/drivers/md/dm-vdo/funnel-workqueue.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_WORK_QUEUE_H +#define VDO_WORK_QUEUE_H + +#include /* for TASK_COMM_LEN */ + +#include "types.h" + +enum { + MAX_VDO_WORK_QUEUE_NAME_LEN = TASK_COMM_LEN, +}; + +struct vdo_work_queue_type { + void (*start)(void *context); + void (*finish)(void *context); + enum vdo_completion_priority max_priority; + enum vdo_completion_priority default_priority; +}; + +struct vdo_completion; +struct vdo_thread; +struct vdo_work_queue; + +int vdo_make_work_queue(const char *thread_name_prefix, const char *name, + struct vdo_thread *owner, const struct vdo_work_queue_type *type, + unsigned int thread_count, void *thread_privates[], + struct vdo_work_queue **queue_ptr); + +void vdo_enqueue_work_queue(struct vdo_work_queue *queue, struct vdo_completion *completion); + +void vdo_finish_work_queue(struct vdo_work_queue *queue); + +void vdo_free_work_queue(struct vdo_work_queue *queue); + +void vdo_dump_work_queue(struct vdo_work_queue *queue); + +void vdo_dump_completion_to_buffer(struct vdo_completion *completion, char *buffer, + size_t length); + +void *vdo_get_work_queue_private_data(void); +struct vdo_work_queue *vdo_get_current_work_queue(void); +struct vdo_thread *vdo_get_work_queue_owner(struct vdo_work_queue *queue); + +bool __must_check vdo_work_queue_type_is(struct vdo_work_queue *queue, + const struct vdo_work_queue_type *type); + +#endif /* VDO_WORK_QUEUE_H */ From cc46b9554b3f6d2f09b1111386b2706e5b4f56c8 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:49:57 -0500 Subject: [PATCH 0556/1038] dm vdo: add basic hash map data structures This patch adds two hash maps, one keyed by integers, the other by pointers, and also a priority heap. The integer map is used for locking of logical and physical addresses. The pointer map is used for managing concurrent writes of the same data, ensuring that those writes are deduplicated. The priority heap is used to minimize the search time for free blocks. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/int-map.c | 718 +++++++++++++++++++++++++++++ drivers/md/dm-vdo/int-map.h | 40 ++ drivers/md/dm-vdo/pointer-map.c | 696 ++++++++++++++++++++++++++++ drivers/md/dm-vdo/pointer-map.h | 81 ++++ drivers/md/dm-vdo/priority-table.c | 226 +++++++++ drivers/md/dm-vdo/priority-table.h | 47 ++ 6 files changed, 1808 insertions(+) create mode 100644 drivers/md/dm-vdo/int-map.c create mode 100644 drivers/md/dm-vdo/int-map.h create mode 100644 drivers/md/dm-vdo/pointer-map.c create mode 100644 drivers/md/dm-vdo/pointer-map.h create mode 100644 drivers/md/dm-vdo/priority-table.c create mode 100644 drivers/md/dm-vdo/priority-table.h diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c new file mode 100644 index 000000000000..463a4951e0d1 --- /dev/null +++ b/drivers/md/dm-vdo/int-map.c @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/** + * DOC: + * + * Hash table implementation of a map from integers to pointers, implemented using the Hopscotch + * Hashing algorithm by Herlihy, Shavit, and Tzafrir (see + * http://en.wikipedia.org/wiki/Hopscotch_hashing). This implementation does not contain any of the + * locking/concurrency features of the algorithm, just the collision resolution scheme. + * + * Hopscotch Hashing is based on hashing with open addressing and linear probing. All the entries + * are stored in a fixed array of buckets, with no dynamic allocation for collisions. Unlike linear + * probing, all the entries that hash to a given bucket are stored within a fixed neighborhood + * starting at that bucket. Chaining is effectively represented as a bit vector relative to each + * bucket instead of as pointers or explicit offsets. + * + * When an empty bucket cannot be found within a given neighborhood, subsequent neighborhoods are + * searched, and one or more entries will "hop" into those neighborhoods. When this process works, + * an empty bucket will move into the desired neighborhood, allowing the entry to be added. When + * that process fails (typically when the buckets are around 90% full), the table must be resized + * and the all entries rehashed and added to the expanded table. + * + * Unlike linear probing, the number of buckets that must be searched in the worst case has a fixed + * upper bound (the size of the neighborhood). Those entries occupy a small number of memory cache + * lines, leading to improved use of the cache (fewer misses on both successful and unsuccessful + * searches). Hopscotch hashing outperforms linear probing at much higher load factors, so even + * with the increased memory burden for maintaining the hop vectors, less memory is needed to + * achieve that performance. Hopscotch is also immune to "contamination" from deleting entries + * since entries are genuinely removed instead of being replaced by a placeholder. + * + * The published description of the algorithm used a bit vector, but the paper alludes to an offset + * scheme which is used by this implementation. Since the entries in the neighborhood are within N + * entries of the hash bucket at the start of the neighborhood, a pair of small offset fields each + * log2(N) bits wide is all that's needed to maintain the hops as a linked list. In order to encode + * "no next hop" (i.e. NULL) as the natural initial value of zero, the offsets are biased by one + * (i.e. 0 => NULL, 1 => offset=0, 2 => offset=1, etc.) We can represent neighborhoods of up to 255 + * entries with just 8+8=16 bits per entry. The hop list is sorted by hop offset so the first entry + * in the list is always the bucket closest to the start of the neighborhood. + * + * While individual accesses tend to be very fast, the table resize operations are very, very + * expensive. If an upper bound on the latency of adding an entry to the table is needed, we either + * need to ensure the table is pre-sized to be large enough so no resize is ever needed, or we'll + * need to develop an approach to incrementally resize the table. + */ + +#include "int-map.h" + +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" + +enum { + DEFAULT_CAPACITY = 16, /* the number of neighborhoods in a new table */ + NEIGHBORHOOD = 255, /* the number of buckets in each neighborhood */ + MAX_PROBES = 1024, /* limit on the number of probes for a free bucket */ + NULL_HOP_OFFSET = 0, /* the hop offset value terminating the hop list */ + DEFAULT_LOAD = 75 /* a compromise between memory use and performance */ +}; + +/** + * struct bucket - hash bucket + * + * Buckets are packed together to reduce memory usage and improve cache efficiency. It would be + * tempting to encode the hop offsets separately and maintain alignment of key/value pairs, but + * it's crucial to keep the hop fields near the buckets that they use them so they'll tend to share + * cache lines. + */ +struct __packed bucket { + /** + * @first_hop: The biased offset of the first entry in the hop list of the neighborhood + * that hashes to this bucket. + */ + u8 first_hop; + /** @next_hop: The biased offset of the next bucket in the hop list. */ + u8 next_hop; + /** @key: The key stored in this bucket. */ + u64 key; + /** @value: The value stored in this bucket (NULL if empty). */ + void *value; +}; + +/** + * struct int_map - The concrete definition of the opaque int_map type. + * + * To avoid having to wrap the neighborhoods of the last entries back around to the start of the + * bucket array, we allocate a few more buckets at the end of the array instead, which is why + * capacity and bucket_count are different. + */ +struct int_map { + /** @size: The number of entries stored in the map. */ + size_t size; + /** @capacity: The number of neighborhoods in the map. */ + size_t capacity; + /* @bucket_count: The number of buckets in the bucket array. */ + size_t bucket_count; + /** @buckets: The array of hash buckets. */ + struct bucket *buckets; +}; + +/** + * mix() - The Google CityHash 16-byte hash mixing function. + * @input1: The first input value. + * @input2: The second input value. + * + * Return: A hash of the two inputs. + */ +static u64 mix(u64 input1, u64 input2) +{ + static const u64 CITY_MULTIPLIER = 0x9ddfea08eb382d69ULL; + u64 hash = (input1 ^ input2); + + hash *= CITY_MULTIPLIER; + hash ^= (hash >> 47); + hash ^= input2; + hash *= CITY_MULTIPLIER; + hash ^= (hash >> 47); + hash *= CITY_MULTIPLIER; + return hash; +} + +/** + * hash_key() - Calculate a 64-bit non-cryptographic hash value for the provided 64-bit integer + * key. + * @key: The mapping key. + * + * The implementation is based on Google's CityHash, only handling the specific case of an 8-byte + * input. + * + * Return: The hash of the mapping key. + */ +static u64 hash_key(u64 key) +{ + /* + * Aliasing restrictions forbid us from casting pointer types, so use a union to convert a + * single u64 to two u32 values. + */ + union { + u64 u64; + u32 u32[2]; + } pun = {.u64 = key}; + + return mix(sizeof(key) + (((u64) pun.u32[0]) << 3), pun.u32[1]); +} + +/** + * allocate_buckets() - Initialize an int_map. + * @map: The map to initialize. + * @capacity: The initial capacity of the map. + * + * Return: UDS_SUCCESS or an error code. + */ +static int allocate_buckets(struct int_map *map, size_t capacity) +{ + map->size = 0; + map->capacity = capacity; + + /* + * Allocate NEIGHBORHOOD - 1 extra buckets so the last bucket can have a full neighborhood + * without have to wrap back around to element zero. + */ + map->bucket_count = capacity + (NEIGHBORHOOD - 1); + return uds_allocate(map->bucket_count, struct bucket, + "struct int_map buckets", &map->buckets); +} + +/** + * vdo_make_int_map() - Allocate and initialize an int_map. + * @initial_capacity: The number of entries the map should initially be capable of holding (zero + * tells the map to use its own small default). + * @initial_load: The load factor of the map, expressed as an integer percentage (typically in the + * range 50 to 90, with zero telling the map to use its own default). + * @map_ptr: Output, a pointer to hold the new int_map. + * + * Return: UDS_SUCCESS or an error code. + */ +int vdo_make_int_map(size_t initial_capacity, unsigned int initial_load, struct int_map **map_ptr) +{ + struct int_map *map; + int result; + size_t capacity; + + /* Use the default initial load if the caller did not specify one. */ + if (initial_load == 0) + initial_load = DEFAULT_LOAD; + if (initial_load > 100) + return UDS_INVALID_ARGUMENT; + + result = uds_allocate(1, struct int_map, "struct int_map", &map); + if (result != UDS_SUCCESS) + return result; + + /* Use the default capacity if the caller did not specify one. */ + capacity = (initial_capacity > 0) ? initial_capacity : DEFAULT_CAPACITY; + + /* + * Scale up the capacity by the specified initial load factor. (i.e to hold 1000 entries at + * 80% load we need a capacity of 1250) + */ + capacity = capacity * 100 / initial_load; + + result = allocate_buckets(map, capacity); + if (result != UDS_SUCCESS) { + vdo_free_int_map(uds_forget(map)); + return result; + } + + *map_ptr = map; + return UDS_SUCCESS; +} + +/** + * vdo_free_int_map() - Free an int_map. + * @map: The int_map to free. + * + * NOTE: The map does not own the pointer values stored in the map and they are not freed by this + * call. + */ +void vdo_free_int_map(struct int_map *map) +{ + if (map == NULL) + return; + + uds_free(uds_forget(map->buckets)); + uds_free(uds_forget(map)); +} + +/** + * vdo_int_map_size() - Get the number of entries stored in an int_map. + * @map: The int_map to query. + * + * Return: The number of entries in the map. + */ +size_t vdo_int_map_size(const struct int_map *map) +{ + return map->size; +} + +/** + * dereference_hop() - Convert a biased hop offset within a neighborhood to a pointer to the bucket + * it references. + * @neighborhood: The first bucket in the neighborhood. + * @hop_offset: The biased hop offset to the desired bucket. + * + * Return: NULL if hop_offset is zero, otherwise a pointer to the bucket in the neighborhood at + * hop_offset - 1. + */ +static struct bucket *dereference_hop(struct bucket *neighborhood, unsigned int hop_offset) +{ + BUILD_BUG_ON(NULL_HOP_OFFSET != 0); + if (hop_offset == NULL_HOP_OFFSET) + return NULL; + + return &neighborhood[hop_offset - 1]; +} + +/** + * insert_in_hop_list() - Add a bucket into the hop list for the neighborhood. + * @neighborhood: The first bucket in the neighborhood. + * @new_bucket: The bucket to add to the hop list. + * + * The bucket is inserted it into the list so the hop list remains sorted by hop offset. + */ +static void insert_in_hop_list(struct bucket *neighborhood, struct bucket *new_bucket) +{ + /* Zero indicates a NULL hop offset, so bias the hop offset by one. */ + int hop_offset = 1 + (new_bucket - neighborhood); + + /* Handle the special case of adding a bucket at the start of the list. */ + int next_hop = neighborhood->first_hop; + + if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { + new_bucket->next_hop = next_hop; + neighborhood->first_hop = hop_offset; + return; + } + + /* Search the hop list for the insertion point that maintains the sort order. */ + for (;;) { + struct bucket *bucket = dereference_hop(neighborhood, next_hop); + + next_hop = bucket->next_hop; + + if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { + new_bucket->next_hop = next_hop; + bucket->next_hop = hop_offset; + return; + } + } +} + +/** + * select_bucket() - Select and return the hash bucket for a given search key. + * @map: The map to search. + * @key: The mapping key. + */ +static struct bucket *select_bucket(const struct int_map *map, u64 key) +{ + /* + * Calculate a good hash value for the provided key. We want exactly 32 bits, so mask the + * result. + */ + u64 hash = hash_key(key) & 0xFFFFFFFF; + + /* + * Scale the 32-bit hash to a bucket index by treating it as a binary fraction and + * multiplying that by the capacity. If the hash is uniformly distributed over [0 .. + * 2^32-1], then (hash * capacity / 2^32) should be uniformly distributed over [0 .. + * capacity-1]. The multiply and shift is much faster than a divide (modulus) on X86 CPUs. + */ + return &map->buckets[(hash * map->capacity) >> 32]; +} + +/** + * search_hop_list() - Search the hop list associated with given hash bucket for a given search + * key. + * @map: The map being searched. + * @bucket: The map bucket to search for the key. + * @key: The mapping key. + * @previous_ptr: Output. if not NULL, a pointer in which to store the bucket in the list preceding + * the one that had the matching key + * + * If the key is found, returns a pointer to the entry (bucket or collision), otherwise returns + * NULL. + * + * Return: An entry that matches the key, or NULL if not found. + */ +static struct bucket *search_hop_list(struct int_map *map __always_unused, + struct bucket *bucket, + u64 key, + struct bucket **previous_ptr) +{ + struct bucket *previous = NULL; + unsigned int next_hop = bucket->first_hop; + + while (next_hop != NULL_HOP_OFFSET) { + /* + * Check the neighboring bucket indexed by the offset for the + * desired key. + */ + struct bucket *entry = dereference_hop(bucket, next_hop); + + if ((key == entry->key) && (entry->value != NULL)) { + if (previous_ptr != NULL) + *previous_ptr = previous; + return entry; + } + next_hop = entry->next_hop; + previous = entry; + } + + return NULL; +} + +/** + * vdo_int_map_get() - Retrieve the value associated with a given key from the int_map. + * @map: The int_map to query. + * @key: The key to look up. + * + * Return: The value associated with the given key, or NULL if the key is not mapped to any value. + */ +void *vdo_int_map_get(struct int_map *map, u64 key) +{ + struct bucket *match = search_hop_list(map, select_bucket(map, key), key, NULL); + + return ((match != NULL) ? match->value : NULL); +} + +/** + * resize_buckets() - Increase the number of hash buckets. + * @map: The map to resize. + * + * Resizes and rehashes all the existing entries, storing them in the new buckets. + * + * Return: UDS_SUCCESS or an error code. + */ +static int resize_buckets(struct int_map *map) +{ + int result; + size_t i; + + /* Copy the top-level map data to the stack. */ + struct int_map old_map = *map; + + /* Re-initialize the map to be empty and 50% larger. */ + size_t new_capacity = map->capacity / 2 * 3; + + uds_log_info("%s: attempting resize from %zu to %zu, current size=%zu", + __func__, map->capacity, new_capacity, map->size); + result = allocate_buckets(map, new_capacity); + if (result != UDS_SUCCESS) { + *map = old_map; + return result; + } + + /* Populate the new hash table from the entries in the old bucket array. */ + for (i = 0; i < old_map.bucket_count; i++) { + struct bucket *entry = &old_map.buckets[i]; + + if (entry->value == NULL) + continue; + + result = vdo_int_map_put(map, entry->key, entry->value, true, NULL); + if (result != UDS_SUCCESS) { + /* Destroy the new partial map and restore the map from the stack. */ + uds_free(uds_forget(map->buckets)); + *map = old_map; + return result; + } + } + + /* Destroy the old bucket array. */ + uds_free(uds_forget(old_map.buckets)); + return UDS_SUCCESS; +} + +/** + * find_empty_bucket() - Probe the bucket array starting at the given bucket for the next empty + * bucket, returning a pointer to it. + * @map: The map containing the buckets to search. + * @bucket: The bucket at which to start probing. + * @max_probes: The maximum number of buckets to search. + * + * NULL will be returned if the search reaches the end of the bucket array or if the number of + * linear probes exceeds a specified limit. + * + * Return: The next empty bucket, or NULL if the search failed. + */ +static struct bucket * +find_empty_bucket(struct int_map *map, struct bucket *bucket, unsigned int max_probes) +{ + /* + * Limit the search to either the nearer of the end of the bucket array or a fixed distance + * beyond the initial bucket. + */ + ptrdiff_t remaining = &map->buckets[map->bucket_count] - bucket; + struct bucket *sentinel = &bucket[min_t(ptrdiff_t, remaining, max_probes)]; + struct bucket *entry; + + for (entry = bucket; entry < sentinel; entry++) { + if (entry->value == NULL) + return entry; + } + + return NULL; +} + +/** + * move_empty_bucket() - Move an empty bucket closer to the start of the bucket array. + * @map: The map containing the bucket. + * @hole: The empty bucket to fill with an entry that precedes it in one of its enclosing + * neighborhoods. + * + * This searches the neighborhoods that contain the empty bucket for a non-empty bucket closer to + * the start of the array. If such a bucket is found, this swaps the two buckets by moving the + * entry to the empty bucket. + * + * Return: The bucket that was vacated by moving its entry to the provided hole, or NULL if no + * entry could be moved. + */ +static struct bucket *move_empty_bucket(struct int_map *map __always_unused, struct bucket *hole) +{ + /* + * Examine every neighborhood that the empty bucket is part of, starting with the one in + * which it is the last bucket. No boundary check is needed for the negative array + * arithmetic since this function is only called when hole is at least NEIGHBORHOOD cells + * deeper into the array than a valid bucket. + */ + struct bucket *bucket; + + for (bucket = &hole[1 - NEIGHBORHOOD]; bucket < hole; bucket++) { + /* + * Find the entry that is nearest to the bucket, which means it will be nearest to + * the hash bucket whose neighborhood is full. + */ + struct bucket *new_hole = dereference_hop(bucket, bucket->first_hop); + + if (new_hole == NULL) { + /* + * There are no buckets in this neighborhood that are in use by this one + * (they must all be owned by overlapping neighborhoods). + */ + continue; + } + + /* + * Skip this bucket if its first entry is actually further away than the hole that + * we're already trying to fill. + */ + if (hole < new_hole) + continue; + + /* + * We've found an entry in this neighborhood that we can "hop" further away, moving + * the hole closer to the hash bucket, if not all the way into its neighborhood. + */ + + /* + * The entry that will be the new hole is the first bucket in the list, so setting + * first_hop is all that's needed remove it from the list. + */ + bucket->first_hop = new_hole->next_hop; + new_hole->next_hop = NULL_HOP_OFFSET; + + /* Move the entry into the original hole. */ + hole->key = new_hole->key; + hole->value = new_hole->value; + new_hole->value = NULL; + + /* Insert the filled hole into the hop list for the neighborhood. */ + insert_in_hop_list(bucket, hole); + return new_hole; + } + + /* We couldn't find an entry to relocate to the hole. */ + return NULL; +} + +/** + * update_mapping() - Find and update any existing mapping for a given key, returning the value + * associated with the key in the provided pointer. + * @map: The int_map to attempt to modify. + * @neighborhood: The first bucket in the neighborhood that would contain the search key + * @key: The key with which to associate the new value. + * @new_value: The value to be associated with the key. + * @update: Whether to overwrite an existing value. + * @old_value_ptr: a pointer in which to store the old value (unmodified if no mapping was found) + * + * Return: true if the map contains a mapping for the key, false if it does not. + */ +static bool update_mapping(struct int_map *map, + struct bucket *neighborhood, + u64 key, + void *new_value, + bool update, + void **old_value_ptr) +{ + struct bucket *bucket = search_hop_list(map, neighborhood, key, NULL); + + if (bucket == NULL) { + /* There is no bucket containing the key in the neighborhood. */ + return false; + } + + /* + * Return the value of the current mapping (if desired) and update the mapping with the new + * value (if desired). + */ + if (old_value_ptr != NULL) + *old_value_ptr = bucket->value; + if (update) + bucket->value = new_value; + return true; +} + +/** + * find_or_make_vacancy() - Find an empty bucket. + * @map: The int_map to search or modify. + * @neighborhood: The first bucket in the neighborhood in which an empty bucket is needed for a new + * mapping. + * + * Find an empty bucket in a specified neighborhood for a new mapping or attempt to re-arrange + * mappings so there is such a bucket. This operation may fail (returning NULL) if an empty bucket + * is not available or could not be relocated to the neighborhood. + * + * Return: a pointer to an empty bucket in the desired neighborhood, or NULL if a vacancy could not + * be found or arranged. + */ +static struct bucket *find_or_make_vacancy(struct int_map *map, struct bucket *neighborhood) +{ + /* Probe within and beyond the neighborhood for the first empty bucket. */ + struct bucket *hole = find_empty_bucket(map, neighborhood, MAX_PROBES); + + /* + * Keep trying until the empty bucket is in the bucket's neighborhood or we are unable to + * move it any closer by swapping it with a filled bucket. + */ + while (hole != NULL) { + int distance = hole - neighborhood; + + if (distance < NEIGHBORHOOD) { + /* + * We've found or relocated an empty bucket close enough to the initial + * hash bucket to be referenced by its hop vector. + */ + return hole; + } + + /* + * The nearest empty bucket isn't within the neighborhood that must contain the new + * entry, so try to swap it with bucket that is closer. + */ + hole = move_empty_bucket(map, hole); + } + + return NULL; +} + +/** + * vdo_int_map_put() - Try to associate a value with an integer. + * @map: The int_map to attempt to modify. + * @key: The key with which to associate the new value. + * @new_value: The value to be associated with the key. + * @update: Whether to overwrite an existing value. + * @old_value_ptr: A pointer in which to store either the old value (if the key was already mapped) + * or NULL if the map did not contain the key; NULL may be provided if the caller + * does not need to know the old value + * + * Try to associate a value (a pointer) with an integer in an int_map. If the map already contains + * a mapping for the provided key, the old value is only replaced with the specified value if + * update is true. In either case the old value is returned. If the map does not already contain a + * value for the specified key, the new value is added regardless of the value of update. + * + * Return: UDS_SUCCESS or an error code. + */ +int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, void **old_value_ptr) +{ + struct bucket *neighborhood, *bucket; + + if (new_value == NULL) + return UDS_INVALID_ARGUMENT; + + /* + * Select the bucket at the start of the neighborhood that must contain any entry for the + * provided key. + */ + neighborhood = select_bucket(map, key); + + /* + * Check whether the neighborhood already contains an entry for the key, in which case we + * optionally update it, returning the old value. + */ + if (update_mapping(map, neighborhood, key, new_value, update, old_value_ptr)) + return UDS_SUCCESS; + + /* + * Find an empty bucket in the desired neighborhood for the new entry or re-arrange entries + * in the map so there is such a bucket. This operation will usually succeed; the loop body + * will only be executed on the rare occasions that we have to resize the map. + */ + while ((bucket = find_or_make_vacancy(map, neighborhood)) == NULL) { + int result; + + /* + * There is no empty bucket in which to put the new entry in the current map, so + * we're forced to allocate a new bucket array with a larger capacity, re-hash all + * the entries into those buckets, and try again (a very expensive operation for + * large maps). + */ + result = resize_buckets(map); + if (result != UDS_SUCCESS) + return result; + + /* + * Resizing the map invalidates all pointers to buckets, so recalculate the + * neighborhood pointer. + */ + neighborhood = select_bucket(map, key); + } + + /* Put the new entry in the empty bucket, adding it to the neighborhood. */ + bucket->key = key; + bucket->value = new_value; + insert_in_hop_list(neighborhood, bucket); + map->size += 1; + + /* There was no existing entry, so there was no old value to be returned. */ + if (old_value_ptr != NULL) + *old_value_ptr = NULL; + return UDS_SUCCESS; +} + +/** + * vdo_int_map_remove() - Remove the mapping for a given key from the int_map. + * @map: The int_map from which to remove the mapping. + * @key: The key whose mapping is to be removed. + * + * Return: the value that was associated with the key, or NULL if it was not mapped. + */ +void *vdo_int_map_remove(struct int_map *map, u64 key) +{ + void *value; + + /* Select the bucket to search and search it for an existing entry. */ + struct bucket *bucket = select_bucket(map, key); + struct bucket *previous; + struct bucket *victim = search_hop_list(map, bucket, key, &previous); + + if (victim == NULL) { + /* There is no matching entry to remove. */ + return NULL; + } + + /* + * We found an entry to remove. Save the mapped value to return later and empty the bucket. + */ + map->size -= 1; + value = victim->value; + victim->value = NULL; + victim->key = 0; + + /* The victim bucket is now empty, but it still needs to be spliced out of the hop list. */ + if (previous == NULL) { + /* The victim is the head of the list, so swing first_hop. */ + bucket->first_hop = victim->next_hop; + } else { + previous->next_hop = victim->next_hop; + } + + victim->next_hop = NULL_HOP_OFFSET; + return value; +} diff --git a/drivers/md/dm-vdo/int-map.h b/drivers/md/dm-vdo/int-map.h new file mode 100644 index 000000000000..e85f12b30edc --- /dev/null +++ b/drivers/md/dm-vdo/int-map.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_INT_MAP_H +#define VDO_INT_MAP_H + +#include +#include + +/** + * DOC: int_map + * + * An int_map associates pointers (void *) with integer keys (u64). NULL pointer values are + * not supported. + * + * The map is implemented as hash table, which should provide constant-time insert, query, and + * remove operations, although the insert may occasionally grow the table, which is linear in the + * number of entries in the map. The table will grow as needed to hold new entries, but will not + * shrink as entries are removed. + */ + +struct int_map; + +int __must_check +vdo_make_int_map(size_t initial_capacity, unsigned int initial_load, struct int_map **map_ptr); + +void vdo_free_int_map(struct int_map *map); + +size_t vdo_int_map_size(const struct int_map *map); + +void *vdo_int_map_get(struct int_map *map, u64 key); + +int __must_check +vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, void **old_value_ptr); + +void *vdo_int_map_remove(struct int_map *map, u64 key); + +#endif /* VDO_INT_MAP_H */ diff --git a/drivers/md/dm-vdo/pointer-map.c b/drivers/md/dm-vdo/pointer-map.c new file mode 100644 index 000000000000..61eb2d3cd8e5 --- /dev/null +++ b/drivers/md/dm-vdo/pointer-map.c @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/** + * DOC: + * + * Hash table implementation of a map from integers to pointers, implemented using the Hopscotch + * Hashing algorithm by Herlihy, Shavit, and Tzafrir (see + * http://en.wikipedia.org/wiki/Hopscotch_hashing). This implementation does not contain any of the + * locking/concurrency features of the algorithm, just the collision resolution scheme. + * + * Hopscotch Hashing is based on hashing with open addressing and linear probing. All the entries + * are stored in a fixed array of buckets, with no dynamic allocation for collisions. Unlike linear + * probing, all the entries that hash to a given bucket are stored within a fixed neighborhood + * starting at that bucket. Chaining is effectively represented as a bit vector relative to each + * bucket instead of as pointers or explicit offsets. + * + * When an empty bucket cannot be found within a given neighborhood, subsequent neighborhoods are + * searched, and one or more entries will "hop" into those neighborhoods. When this process works, + * an empty bucket will move into the desired neighborhood, allowing the entry to be added. When + * that process fails (typically when the buckets are around 90% full), the table must be resized + * and the all entries rehashed and added to the expanded table. + * + * Unlike linear probing, the number of buckets that must be searched in the worst case has a fixed + * upper bound (the size of the neighborhood). Those entries occupy a small number of memory cache + * lines, leading to improved use of the cache (fewer misses on both successful and unsuccessful + * searches). Hopscotch hashing outperforms linear probing at much higher load factors, so even + * with the increased memory burden for maintaining the hop vectors, less memory is needed to + * achieve that performance. Hopscotch is also immune to "contamination" from deleting entries + * since entries are genuinely removed instead of being replaced by a placeholder. + * + * The published description of the algorithm used a bit vector, but the paper alludes to an offset + * scheme which is used by this implementation. Since the entries in the neighborhood are within N + * entries of the hash bucket at the start of the neighborhood, a pair of small offset fields each + * log2(N) bits wide is all that's needed to maintain the hops as a linked list. In order to encode + * "no next hop" (i.e. NULL) as the natural initial value of zero, the offsets are biased by one + * (i.e. 0 => NULL, 1 => offset=0, 2 => offset=1, etc.) We can represent neighborhoods of up to 255 + * entries with just 8+8=16 bits per entry. The hop list is sorted by hop offset so the first entry + * in the list is always the bucket closest to the start of the neighborhood. + * + * While individual accesses tend to be very fast, the table resize operations are very, very + * expensive. If an upper bound on the latency of adding an entry to the table is needed, we either + * need to ensure the table is pre-sized to be large enough so no resize is ever needed, or we'll + * need to develop an approach to incrementally resize the table. + */ + +#include "pointer-map.h" + +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" + +enum { + DEFAULT_CAPACITY = 16, /* the number of neighborhoods in a new table */ + NEIGHBORHOOD = 255, /* the number of buckets in each neighborhood */ + MAX_PROBES = 1024, /* limit on the number of probes for a free bucket */ + NULL_HOP_OFFSET = 0, /* the hop offset value terminating the hop list */ + DEFAULT_LOAD = 75 /* a compromise between memory use and performance */ +}; + +/** + * struct bucket - Hash buckets. + * + * Buckets are packed together to reduce memory usage and improve cache efficiency. It would be + * tempting to encode the hop offsets separately and maintain alignment of key/value pairs, but + * it's crucial to keep the hop fields near the buckets that they use them so they'll tend to share + * cache lines. + */ +struct __packed bucket { + /** + * @first_hop: The biased offset of the first entry in the hop list of the neighborhood + * that hashes to this bucket. + */ + u8 first_hop; + /** @next_hop: the biased offset of the next bucket in the hop list. */ + u8 next_hop; + /** @key: The key stored in this bucket. */ + const void *key; + /** @value: The value stored in this bucket (NULL if empty). */ + void *value; +}; + +/** + * struct pointer_map - The concrete definition of the opaque pointer_map type. + * + * To avoid having to wrap the neighborhoods of the last entries back around to the start of the + * bucket array, we allocate a few more buckets at the end of the array instead, which is why + * capacity and bucket_count are different. + */ +struct pointer_map { + /** @size: The number of entries stored in the map. */ + size_t size; + /** @capacity: The number of neighborhoods in the map. */ + size_t capacity; + /** @bucket_count: The number of buckets in the bucket array. */ + size_t bucket_count; + /** @buckets: The array of hash buckets. */ + struct bucket *buckets; + /** @comparator: The function for comparing keys for equality. */ + pointer_key_comparator *comparator; + /** @hasher: The function for getting a hash code from a key. */ + pointer_key_hasher *hasher; +}; + +/** + * allocate_buckets() - Initialize a pointer_map. + * @map: The map to initialize. + * @capacity: The initial capacity of the map. + * + * Return: UDS_SUCCESS or an error code. + */ +static int allocate_buckets(struct pointer_map *map, size_t capacity) +{ + map->size = 0; + map->capacity = capacity; + + /* + * Allocate NEIGHBORHOOD - 1 extra buckets so the last bucket can have a full neighborhood + * without have to wrap back around to element zero. + */ + map->bucket_count = capacity + (NEIGHBORHOOD - 1); + return uds_allocate(map->bucket_count, + struct bucket, + "pointer_map buckets", + &map->buckets); +} + +/** + * vdo_make_pointer_map() - Allocate and initialize a pointer_map. + * @initial_capacity: The number of entries the map should initially be capable of holding (zero + * tells the map to use its own small default). + * @initial_load: The load factor of the map, expressed as an integer percentage (typically in the + * range 50 to 90, with zero telling the map to use its own default). + * @comparator: The function to use to compare the referents of two pointer keys for equality. + * @hasher: The function to use obtain the hash code associated with each pointer key + * @map_ptr: A pointer to hold the new pointer_map. + * + * Return: UDS_SUCCESS or an error code. + */ +int vdo_make_pointer_map(size_t initial_capacity, + unsigned int initial_load, + pointer_key_comparator comparator, + pointer_key_hasher hasher, + struct pointer_map **map_ptr) +{ + int result; + struct pointer_map *map; + size_t capacity; + + /* Use the default initial load if the caller did not specify one. */ + if (initial_load == 0) + initial_load = DEFAULT_LOAD; + if (initial_load > 100) + return UDS_INVALID_ARGUMENT; + + result = uds_allocate(1, struct pointer_map, "pointer_map", &map); + if (result != UDS_SUCCESS) + return result; + + map->hasher = hasher; + map->comparator = comparator; + + /* Use the default capacity if the caller did not specify one. */ + capacity = (initial_capacity > 0) ? initial_capacity : DEFAULT_CAPACITY; + + /* + * Scale up the capacity by the specified initial load factor. (i.e to hold 1000 entries at + * 80% load we need a capacity of 1250) + */ + capacity = capacity * 100 / initial_load; + + result = allocate_buckets(map, capacity); + if (result != UDS_SUCCESS) { + vdo_free_pointer_map(uds_forget(map)); + return result; + } + + *map_ptr = map; + return UDS_SUCCESS; +} + +/** + * vdo_free_pointer_map() - Free a pointer_map. + * @map: The pointer_map to free. + * + * The map does not own the pointer keys and values stored in the map and they are not freed by + * this call. + */ +void vdo_free_pointer_map(struct pointer_map *map) +{ + if (map == NULL) + return; + + uds_free(uds_forget(map->buckets)); + uds_free(uds_forget(map)); +} + +/** + * vdo_pointer_map_size() - Get the number of entries stored in a pointer_map. + * @map: The pointer_map to query. + * + * Return: The number of entries in the map. + */ +size_t vdo_pointer_map_size(const struct pointer_map *map) +{ + return map->size; +} + +/** + * dereference_hop() - Convert a biased hop offset within a neighborhood to a pointer to the bucket + * it references. + * @neighborhood: The first bucket in the neighborhood. + * @hop_offset: The biased hop offset to the desired bucket. + * + * Return: NULL if hop_offset is zero, otherwise a pointer to the bucket in the neighborhood at + * hop_offset - 1. + */ +static struct bucket *dereference_hop(struct bucket *neighborhood, unsigned int hop_offset) +{ + BUILD_BUG_ON(NULL_HOP_OFFSET != 0); + if (hop_offset == NULL_HOP_OFFSET) + return NULL; + + return &neighborhood[hop_offset - 1]; +} + +/** + * insert_in_hop_list() - Add a bucket into the hop list for the neighborhood, inserting it into + * the list so the hop list remains sorted by hop offset. + * @neighborhood: The first bucket in the neighborhood. + * @new_bucket: The bucket to add to the hop list. + */ +static void insert_in_hop_list(struct bucket *neighborhood, struct bucket *new_bucket) +{ + /* Zero indicates a NULL hop offset, so bias the hop offset by one. */ + int hop_offset = 1 + (new_bucket - neighborhood); + + /* Handle the special case of adding a bucket at the start of the list. */ + int next_hop = neighborhood->first_hop; + + if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { + new_bucket->next_hop = next_hop; + neighborhood->first_hop = hop_offset; + return; + } + + /* Search the hop list for the insertion point that maintains the sort order. */ + for (;;) { + struct bucket *bucket = dereference_hop(neighborhood, next_hop); + + next_hop = bucket->next_hop; + + if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { + new_bucket->next_hop = next_hop; + bucket->next_hop = hop_offset; + return; + } + } +} + +/** + * select_bucket() - Select and return the hash bucket for a given search key. + * @map: The map to search. + * @key: The mapping key. + */ +static struct bucket *select_bucket(const struct pointer_map *map, const void *key) +{ + /* + * Scale the 32-bit hash to a bucket index by treating it as a binary fraction and + * multiplying that by the capacity. If the hash is uniformly distributed over [0 .. + * 2^32-1], then (hash * capacity / 2^32) should be uniformly distributed over [0 .. + * capacity-1]. The multiply and shift is much faster than a divide (modulus) on X86 CPUs. + */ + u64 hash = map->hasher(key); + + return &map->buckets[(hash * map->capacity) >> 32]; +} + +/** + * search_hop_list() - Search the hop list. + * @map: The map being searched. + * @bucket: The map bucket to search for the key. + * @key: The mapping key. + * @previous_ptr: if not NULL, a pointer in which to store the bucket in the list preceding the one + * that had the matching key. + * + * Searches the hop list associated with given hash bucket for a given search key. If the key is + * found, returns a pointer to the entry (bucket or collision), otherwise returns NULL. + * + * Return: an entry that matches the key, or NULL if not found. + */ +static struct bucket *search_hop_list(struct pointer_map *map, + struct bucket *bucket, + const void *key, + struct bucket **previous_ptr) +{ + struct bucket *previous = NULL; + unsigned int next_hop = bucket->first_hop; + + while (next_hop != NULL_HOP_OFFSET) { + /* Check the neighboring bucket indexed by the offset for the desired key. */ + struct bucket *entry = dereference_hop(bucket, next_hop); + + if ((entry->value != NULL) && map->comparator(key, entry->key)) { + if (previous_ptr != NULL) + *previous_ptr = previous; + return entry; + } + next_hop = entry->next_hop; + previous = entry; + } + return NULL; +} + +/** + * vdo_pointer_map_get() - Retrieve the value associated with a given key from the pointer_map. + * @map: The pointer_map to query. + * @key: The key to look up (may be NULL if the comparator and hasher functions support it). + * + * Return: the value associated with the given key, or NULL if the key is not mapped to any value. + */ +void *vdo_pointer_map_get(struct pointer_map *map, const void *key) +{ + struct bucket *match = search_hop_list(map, select_bucket(map, key), key, NULL); + + return ((match != NULL) ? match->value : NULL); +} + +/** + * resize_buckets() - Increase the number of hash buckets and rehash all the existing entries, + * storing them in the new buckets. + * @map: The map to resize. + */ +static int resize_buckets(struct pointer_map *map) +{ + int result; + size_t i; + + /* Copy the top-level map data to the stack. */ + struct pointer_map old_map = *map; + + /* Re-initialize the map to be empty and 50% larger. */ + size_t new_capacity = map->capacity / 2 * 3; + + uds_log_info("%s: attempting resize from %zu to %zu, current size=%zu", + __func__, + map->capacity, + new_capacity, + map->size); + result = allocate_buckets(map, new_capacity); + if (result != UDS_SUCCESS) { + *map = old_map; + return result; + } + + /* Populate the new hash table from the entries in the old bucket array. */ + for (i = 0; i < old_map.bucket_count; i++) { + struct bucket *entry = &old_map.buckets[i]; + + if (entry->value == NULL) + continue; + + result = vdo_pointer_map_put(map, entry->key, entry->value, true, NULL); + if (result != UDS_SUCCESS) { + /* Destroy the new partial map and restore the map from the stack. */ + uds_free(uds_forget(map->buckets)); + *map = old_map; + return result; + } + } + + /* Destroy the old bucket array. */ + uds_free(uds_forget(old_map.buckets)); + return UDS_SUCCESS; +} + +/** + * find_empty_bucket() - Probe the bucket array starting at the given bucket for the next empty + * bucket, returning a pointer to it. + * @map: The map containing the buckets to search. + * @bucket: The bucket at which to start probing. + * @max_probes: The maximum number of buckets to search. + * + * NULL will be returned if the search reaches the end of the bucket array or if the number of + * linear probes exceeds a specified limit. + * + * Return: The next empty bucket, or NULL if the search failed. + */ +static struct bucket * +find_empty_bucket(struct pointer_map *map, struct bucket *bucket, unsigned int max_probes) +{ + /* + * Limit the search to either the nearer of the end of the bucket array or a fixed distance + * beyond the initial bucket. + */ + ptrdiff_t remaining = &map->buckets[map->bucket_count] - bucket; + struct bucket *sentinel = &bucket[min_t(ptrdiff_t, remaining, max_probes)]; + struct bucket *entry; + + for (entry = bucket; entry < sentinel; entry++) + if (entry->value == NULL) + return entry; + return NULL; +} + +/** + * move_empty_bucket() - Move an empty bucket closer to the start of the bucket array. + * @map: The map containing the bucket. + + * @hole: The empty bucket to fill with an entry that precedes it in one of its enclosing + * neighborhoods. + * + * This searches the neighborhoods that contain the empty bucket for a non-empty bucket closer to + * the start of the array. If such a bucket is found, this swaps the two buckets by moving the + * entry to the empty bucket. + * + * Return: The bucket that was vacated by moving its entry to the provided hole, or NULL if no + * entry could be moved. + */ +static struct bucket * +move_empty_bucket(struct pointer_map *map __always_unused, struct bucket *hole) +{ + /* + * Examine every neighborhood that the empty bucket is part of, starting with the one in + * which it is the last bucket. No boundary check is needed for the negative array + * arithmetic since this function is only called when hole is at least NEIGHBORHOOD cells + * deeper into the array than a valid bucket. + */ + struct bucket *bucket; + + for (bucket = &hole[1 - NEIGHBORHOOD]; bucket < hole; bucket++) { + /* + * Find the entry that is nearest to the bucket, which means it will be nearest to + * the hash bucket whose neighborhood is full. + */ + struct bucket *new_hole = dereference_hop(bucket, bucket->first_hop); + + if (new_hole == NULL) { + /* + * There are no buckets in this neighborhood that are in use by this one + * (they must all be owned by overlapping neighborhoods). + */ + continue; + } + + /* + * Skip this bucket if its first entry is actually further away than the hole that + * we're already trying to fill. + */ + if (hole < new_hole) + continue; + + /* + * We've found an entry in this neighborhood that we can "hop" further away, moving + * the hole closer to the hash bucket, if not all the way into its neighborhood. + */ + + /* + * The entry that will be the new hole is the first bucket in the list, so setting + * first_hop is all that's needed remove it from the list. + */ + bucket->first_hop = new_hole->next_hop; + new_hole->next_hop = NULL_HOP_OFFSET; + + /* Move the entry into the original hole. */ + hole->key = new_hole->key; + hole->value = new_hole->value; + new_hole->value = NULL; + + /* Insert the filled hole into the hop list for the neighborhood. */ + insert_in_hop_list(bucket, hole); + return new_hole; + } + + /* We couldn't find an entry to relocate to the hole. */ + return NULL; +} + +/** + * update_mapping() - Find and update any existing mapping for a given key, returning the value + * associated with the key in the provided pointer. + * @map: The pointer_map to attempt to modify. + * @neighborhood: The first bucket in the neighborhood that would contain the search key. + * @key: The key with which to associate the new value. + * @new_value: The value to be associated with the key. + * @update: Whether to overwrite an existing value. + * @old_value_ptr: A pointer in which to store the old value (unmodified if no mapping was found). + * + * Return: true if the map contains a mapping for the key, false if it does not. + */ +static bool update_mapping(struct pointer_map *map, + struct bucket *neighborhood, + const void *key, + void *new_value, + bool update, + void **old_value_ptr) +{ + struct bucket *bucket = search_hop_list(map, neighborhood, key, NULL); + + if (bucket == NULL) { + /* There is no bucket containing the key in the neighborhood. */ + return false; + } + + /* + * Return the value of the current mapping (if desired) and update the mapping with the new + * value (if desired). + */ + if (old_value_ptr != NULL) + *old_value_ptr = bucket->value; + if (update) { + /* + * We're dropping the old key pointer on the floor here, assuming it's a property + * of the value or that it's otherwise safe to just forget. + */ + bucket->key = key; + bucket->value = new_value; + } + return true; +} + +/** + * find_or_make_vacancy() - Find an empty bucket in a specified neighborhood for a new mapping or + * attempt to re-arrange mappings so there is such a bucket. + * @map: The pointer_map to search or modify. + * @neighborhood: The first bucket in the neighborhood in which an empty bucket is needed for a new + * mapping. + * + * This operation may fail (returning NULL) if an empty bucket is not available or could not be + * relocated to the neighborhood. + * + * Return: A pointer to an empty bucket in the desired neighborhood, or NULL if a vacancy could not + * be found or arranged. + */ +static struct bucket *find_or_make_vacancy(struct pointer_map *map, struct bucket *neighborhood) +{ + /* Probe within and beyond the neighborhood for the first empty bucket. */ + struct bucket *hole = find_empty_bucket(map, neighborhood, MAX_PROBES); + + /* + * Keep trying until the empty bucket is in the bucket's neighborhood or we are unable to + * move it any closer by swapping it with a filled bucket. + */ + while (hole != NULL) { + int distance = hole - neighborhood; + + if (distance < NEIGHBORHOOD) { + /* + * We've found or relocated an empty bucket close enough to the initial + * hash bucket to be referenced by its hop vector. + */ + return hole; + } + + /* + * The nearest empty bucket isn't within the neighborhood that must contain the new + * entry, so try to swap it with bucket that is closer. + */ + hole = move_empty_bucket(map, hole); + } + + return NULL; +} + +/** + * vdo_pointer_map_put() - Try to associate a value (a pointer) with an integer in a pointer_map. + * @map: The pointer_map to attempt to modify. + * @key: The key with which to associate the new value (may be NULL if the comparator and hasher + * functions support it). + * @new_value: The value to be associated with the key. + * @update: Whether to overwrite an existing value. + * @old_value_ptr: A pointer in which to store either the old value (if the key was already mapped) + * or NULL if the map did not contain the key; NULL may be provided if the caller + * does not need to know the old value. + * + * If the map already contains a mapping for the provided key, the old value is only replaced with + * the specified value if update is true. In either case the old value is returned. If the map does + * not already contain a value for the specified key, the new value is added regardless of the + * value of update. + * + * If the value stored in the map is updated, then the key stored in the map will also be updated + * with the key provided by this call. The old key will not be returned due to the memory + * management assumptions described in the interface header comment. + * + * Return: UDS_SUCCESS or an error code. + */ +int vdo_pointer_map_put(struct pointer_map *map, + const void *key, + void *new_value, + bool update, + void **old_value_ptr) +{ + struct bucket *neighborhood, *bucket; + + if (new_value == NULL) + return UDS_INVALID_ARGUMENT; + + /* + * Select the bucket at the start of the neighborhood that must contain any entry for the + * provided key. + */ + neighborhood = select_bucket(map, key); + + /* + * Check whether the neighborhood already contains an entry for the key, in which case we + * optionally update it, returning the old value. + */ + if (update_mapping(map, neighborhood, key, new_value, update, old_value_ptr)) + return UDS_SUCCESS; + + /* + * Find an empty bucket in the desired neighborhood for the new entry or re-arrange entries + * in the map so there is such a bucket. This operation will usually succeed; the loop body + * will only be executed on the rare occasions that we have to resize the map. + */ + while ((bucket = find_or_make_vacancy(map, neighborhood)) == NULL) { + /* + * There is no empty bucket in which to put the new entry in the current map, so + * we're forced to allocate a new bucket array with a larger capacity, re-hash all + * the entries into those buckets, and try again (a very expensive operation for + * large maps). + */ + int result = resize_buckets(map); + + if (result != UDS_SUCCESS) + return result; + + /* + * Resizing the map invalidates all pointers to buckets, so + * recalculate the neighborhood pointer. + */ + neighborhood = select_bucket(map, key); + } + + /* Put the new entry in the empty bucket, adding it to the neighborhood. */ + bucket->key = key; + bucket->value = new_value; + insert_in_hop_list(neighborhood, bucket); + map->size += 1; + + /* + * There was no existing entry, so there was no old value to be + * returned. + */ + if (old_value_ptr != NULL) + *old_value_ptr = NULL; + return UDS_SUCCESS; +} + +/** + * vdo_pointer_map_remove() - Remove the mapping for a given key from the pointer_map. + * @map: The pointer_map from which to remove the mapping. + * @key: The key whose mapping is to be removed (may be NULL if the comparator and hasher functions + * support it). + * + * Return: the value that was associated with the key, or NULL if it was not mapped. + */ +void *vdo_pointer_map_remove(struct pointer_map *map, const void *key) +{ + void *value; + + /* Select the bucket to search and search it for an existing entry. */ + struct bucket *bucket = select_bucket(map, key); + struct bucket *previous; + struct bucket *victim = search_hop_list(map, bucket, key, &previous); + + if (victim == NULL) { + /* There is no matching entry to remove. */ + return NULL; + } + + /* + * We found an entry to remove. Save the mapped value to return later and empty the bucket. + */ + map->size -= 1; + value = victim->value; + victim->value = NULL; + victim->key = 0; + + /* The victim bucket is now empty, but it still needs to be spliced out of the hop list. */ + if (previous == NULL) { + /* The victim is the head of the list, so swing first_hop. */ + bucket->first_hop = victim->next_hop; + } else { + previous->next_hop = victim->next_hop; + } + + victim->next_hop = NULL_HOP_OFFSET; + return value; +} diff --git a/drivers/md/dm-vdo/pointer-map.h b/drivers/md/dm-vdo/pointer-map.h new file mode 100644 index 000000000000..83465d01bfa7 --- /dev/null +++ b/drivers/md/dm-vdo/pointer-map.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_POINTER_MAP_H +#define VDO_POINTER_MAP_H + +#include +#include + +/* + * A pointer_map associates pointer values (void *) with the data referenced by + * pointer keys (void *). NULL pointer values are not supported. A + * NULL key value is supported when the instance's key comparator and hasher functions + * support it. + * + * The map is implemented as hash table, which should provide constant-time insert, query, and + * remove operations, although the insert may occasionally grow the table, which is linear in the + * number of entries in the map. The table will grow as needed to hold new entries, but will not + * shrink as entries are removed. + * + * The key and value pointers passed to the map are retained and used by the map, but are not owned + * by the map. Freeing the map does not attempt to free the pointers. The client is entirely + * responsible for the memory management of the keys and values. The current interface and + * implementation assume that keys will be properties of the values, or that keys will not be + * memory managed, or that keys will not need to be freed as a result of being replaced when a key + * is re-mapped. + */ + +struct pointer_map; + +/** + * typedef pointer_key_comparator - The prototype of functions that compare the referents of two + * pointer keys for equality. + * @this_key: The first element to compare. + * @that_key: The second element to compare. + * + * If two keys are equal, then both keys must have the same the hash code associated with them by + * the hasher function defined below. + * + * Return: true if and only if the referents of the two key pointers are to be treated as the same + * key by the map. + */ +typedef bool pointer_key_comparator(const void *this_key, const void *that_key); + +/** + * typedef pointer_key_hasher - The prototype of functions that get or calculate a hash code + * associated with the referent of pointer key. + * @key: The pointer key to hash. + * + * The hash code must be uniformly distributed over all u32 values. The hash code associated + * with a given key must not change while the key is in the map. If the comparator function says + * two keys are equal, then this function must return the same hash code for both keys. This + * function may be called many times for a key while an entry is stored for it in the map. + * + * Return: The hash code for the key. + */ +typedef u32 pointer_key_hasher(const void *key); + +int __must_check vdo_make_pointer_map(size_t initial_capacity, + unsigned int initial_load, + pointer_key_comparator comparator, + pointer_key_hasher hasher, + struct pointer_map **map_ptr); + +void vdo_free_pointer_map(struct pointer_map *map); + +size_t vdo_pointer_map_size(const struct pointer_map *map); + +void *vdo_pointer_map_get(struct pointer_map *map, const void *key); + +int __must_check vdo_pointer_map_put(struct pointer_map *map, + const void *key, + void *new_value, + bool update, + void **old_value_ptr); + +void *vdo_pointer_map_remove(struct pointer_map *map, const void *key); + +#endif /* VDO_POINTER_MAP_H */ diff --git a/drivers/md/dm-vdo/priority-table.c b/drivers/md/dm-vdo/priority-table.c new file mode 100644 index 000000000000..9408219b5700 --- /dev/null +++ b/drivers/md/dm-vdo/priority-table.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "priority-table.h" + +#include + +#include "errors.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "status-codes.h" + +/* We use a single 64-bit search vector, so the maximum priority is 63 */ +enum { + MAX_PRIORITY = 63 +}; + +/* + * All the entries with the same priority are queued in a circular list in a bucket for that + * priority. The table is essentially an array of buckets. + */ +struct bucket { + /* + * The head of a queue of table entries, all having the same priority + */ + struct list_head queue; + /* The priority of all the entries in this bucket */ + unsigned int priority; +}; + +/* + * A priority table is an array of buckets, indexed by priority. New entries are added to the end + * of the queue in the appropriate bucket. The dequeue operation finds the highest-priority + * non-empty bucket by searching a bit vector represented as a single 8-byte word, which is very + * fast with compiler and CPU support. + */ +struct priority_table { + /* The maximum priority of entries that may be stored in this table */ + unsigned int max_priority; + /* A bit vector flagging all buckets that are currently non-empty */ + u64 search_vector; + /* The array of all buckets, indexed by priority */ + struct bucket buckets[]; +}; + +/** + * vdo_make_priority_table() - Allocate and initialize a new priority_table. + * @max_priority: The maximum priority value for table entries. + * @table_ptr: A pointer to hold the new table. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_priority_table(unsigned int max_priority, struct priority_table **table_ptr) +{ + struct priority_table *table; + int result; + unsigned int priority; + + if (max_priority > MAX_PRIORITY) + return UDS_INVALID_ARGUMENT; + + result = uds_allocate_extended(struct priority_table, max_priority + 1, + struct bucket, __func__, &table); + if (result != VDO_SUCCESS) + return result; + + for (priority = 0; priority <= max_priority; priority++) { + struct bucket *bucket = &table->buckets[priority]; + + bucket->priority = priority; + INIT_LIST_HEAD(&bucket->queue); + } + + table->max_priority = max_priority; + table->search_vector = 0; + + *table_ptr = table; + return VDO_SUCCESS; +} + +/** + * vdo_free_priority_table() - Free a priority_table. + * @table: The table to free. + * + * The table does not own the entries stored in it and they are not freed by this call. + */ +void vdo_free_priority_table(struct priority_table *table) +{ + if (table == NULL) + return; + + /* + * Unlink the buckets from any entries still in the table so the entries won't be left with + * dangling pointers to freed memory. + */ + vdo_reset_priority_table(table); + + uds_free(table); +} + +/** + * vdo_reset_priority_table() - Reset a priority table, leaving it in the same empty state as when + * newly constructed. + * @table: The table to reset. + * + * The table does not own the entries stored in it and they are not freed (or even unlinked from + * each other) by this call. + */ +void vdo_reset_priority_table(struct priority_table *table) +{ + unsigned int priority; + + table->search_vector = 0; + for (priority = 0; priority <= table->max_priority; priority++) + list_del_init(&table->buckets[priority].queue); +} + +/** + * vdo_priority_table_enqueue() - Add a new entry to the priority table, appending it to the queue + * for entries with the specified priority. + * @table: The table in which to store the entry. + * @priority: The priority of the entry. + * @entry: The list_head embedded in the entry to store in the table (the caller must have + * initialized it). + */ +void vdo_priority_table_enqueue(struct priority_table *table, unsigned int priority, + struct list_head *entry) +{ + ASSERT_LOG_ONLY((priority <= table->max_priority), + "entry priority must be valid for the table"); + + /* Append the entry to the queue in the specified bucket. */ + list_move_tail(entry, &table->buckets[priority].queue); + + /* Flag the bucket in the search vector since it must be non-empty. */ + table->search_vector |= (1ULL << priority); +} + +static inline void mark_bucket_empty(struct priority_table *table, struct bucket *bucket) +{ + table->search_vector &= ~(1ULL << bucket->priority); +} + +/** + * vdo_priority_table_dequeue() - Find the highest-priority entry in the table, remove it from the + * table, and return it. + * @table: The priority table from which to remove an entry. + * + * If there are multiple entries with the same priority, the one that has been in the table with + * that priority the longest will be returned. + * + * Return: The dequeued entry, or NULL if the table is currently empty. + */ +struct list_head *vdo_priority_table_dequeue(struct priority_table *table) +{ + struct bucket *bucket; + struct list_head *entry; + int top_priority; + + if (table->search_vector == 0) { + /* All buckets are empty. */ + return NULL; + } + + /* + * Find the highest priority non-empty bucket by finding the highest-order non-zero bit in + * the search vector. + */ + top_priority = ilog2(table->search_vector); + + /* Dequeue the first entry in the bucket. */ + bucket = &table->buckets[top_priority]; + entry = bucket->queue.next; + list_del_init(entry); + + /* Clear the bit in the search vector if the bucket has been emptied. */ + if (list_empty(&bucket->queue)) + mark_bucket_empty(table, bucket); + + return entry; +} + +/** + * vdo_priority_table_remove() - Remove a specified entry from its priority table. + * @table: The table from which to remove the entry. + * @entry: The entry to remove from the table. + */ +void vdo_priority_table_remove(struct priority_table *table, struct list_head *entry) +{ + struct list_head *next_entry; + + /* + * We can't guard against calls where the entry is on a list for a different table, but + * it's easy to deal with an entry not in any table or list. + */ + if (list_empty(entry)) + return; + + /* + * Remove the entry from the bucket list, remembering a pointer to another entry in the + * ring. + */ + next_entry = entry->next; + list_del_init(entry); + + /* + * If the rest of the list is now empty, the next node must be the list head in the bucket + * and we can use it to update the search vector. + */ + if (list_empty(next_entry)) + mark_bucket_empty(table, list_entry(next_entry, struct bucket, queue)); +} + +/** + * vdo_is_priority_table_empty() - Return whether the priority table is empty. + * @table: The table to check. + * + * Return: true if the table is empty. + */ +bool vdo_is_priority_table_empty(struct priority_table *table) +{ + return (table->search_vector == 0); +} diff --git a/drivers/md/dm-vdo/priority-table.h b/drivers/md/dm-vdo/priority-table.h new file mode 100644 index 000000000000..8b060462e3e4 --- /dev/null +++ b/drivers/md/dm-vdo/priority-table.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_PRIORITY_TABLE_H +#define VDO_PRIORITY_TABLE_H + +#include + +/* + * A priority_table is a simple implementation of a priority queue for entries with priorities that + * are small non-negative integer values. It implements the obvious priority queue operations of + * enqueuing an entry and dequeuing an entry with the maximum priority. It also supports removing + * an arbitrary entry. The priority of an entry already in the table can be changed by removing it + * and re-enqueuing it with a different priority. All operations have O(1) complexity. + * + * The links for the table entries must be embedded in the entries themselves. Lists are used to + * link entries in the table and no wrapper type is declared, so an existing list entry in an + * object can also be used to queue it in a priority_table, assuming the field is not used for + * anything else while so queued. + * + * The table is implemented as an array of queues (circular lists) indexed by priority, along with + * a hint for which queues are non-empty. Steven Skiena calls a very similar structure a "bounded + * height priority queue", but given the resemblance to a hash table, "priority table" seems both + * shorter and more apt, if somewhat novel. + */ + +struct priority_table; + +int __must_check vdo_make_priority_table(unsigned int max_priority, + struct priority_table **table_ptr); + +void vdo_free_priority_table(struct priority_table *table); + +void vdo_priority_table_enqueue(struct priority_table *table, unsigned int priority, + struct list_head *entry); + +void vdo_reset_priority_table(struct priority_table *table); + +struct list_head * __must_check vdo_priority_table_dequeue(struct priority_table *table); + +void vdo_priority_table_remove(struct priority_table *table, struct list_head *entry); + +bool __must_check vdo_is_priority_table_empty(struct priority_table *table); + +#endif /* VDO_PRIORITY_TABLE_H */ From 4390aa138b61cf257989e1f0a0deb53e46adae96 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:50:45 -0500 Subject: [PATCH 0557/1038] dm vdo: add deduplication configuration structures Add structures which record the configuration of various deduplication index parameters. This also includes facilities for saving and loading the configuration and validating its integrity. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: John Wiele Signed-off-by: John Wiele Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/config.c | 378 +++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/config.h | 124 ++++++++++++ drivers/md/dm-vdo/geometry.c | 201 +++++++++++++++++++ drivers/md/dm-vdo/geometry.h | 138 +++++++++++++ 4 files changed, 841 insertions(+) create mode 100644 drivers/md/dm-vdo/config.c create mode 100644 drivers/md/dm-vdo/config.h create mode 100644 drivers/md/dm-vdo/geometry.c create mode 100644 drivers/md/dm-vdo/geometry.h diff --git a/drivers/md/dm-vdo/config.c b/drivers/md/dm-vdo/config.c new file mode 100644 index 000000000000..96e27aeecc25 --- /dev/null +++ b/drivers/md/dm-vdo/config.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "config.h" + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "string-utils.h" +#include "uds-threads.h" + +static const u8 INDEX_CONFIG_MAGIC[] = "ALBIC"; +static const u8 INDEX_CONFIG_VERSION_6_02[] = "06.02"; +static const u8 INDEX_CONFIG_VERSION_8_02[] = "08.02"; + +enum { + DEFAULT_VOLUME_READ_THREADS = 2, + MAX_VOLUME_READ_THREADS = 16, + INDEX_CONFIG_MAGIC_LENGTH = sizeof(INDEX_CONFIG_MAGIC) - 1, + INDEX_CONFIG_VERSION_LENGTH = sizeof(INDEX_CONFIG_VERSION_6_02) - 1, +}; + +static bool is_version(const u8 *version, u8 *buffer) +{ + return memcmp(version, buffer, INDEX_CONFIG_VERSION_LENGTH) == 0; +} + +static bool are_matching_configurations(struct configuration *saved_config, + struct geometry *saved_geometry, + struct configuration *user) +{ + struct geometry *geometry = user->geometry; + bool result = true; + + if (saved_geometry->record_pages_per_chapter != geometry->record_pages_per_chapter) { + uds_log_error("Record pages per chapter (%u) does not match (%u)", + saved_geometry->record_pages_per_chapter, + geometry->record_pages_per_chapter); + result = false; + } + + if (saved_geometry->chapters_per_volume != geometry->chapters_per_volume) { + uds_log_error("Chapter count (%u) does not match (%u)", + saved_geometry->chapters_per_volume, + geometry->chapters_per_volume); + result = false; + } + + if (saved_geometry->sparse_chapters_per_volume != geometry->sparse_chapters_per_volume) { + uds_log_error("Sparse chapter count (%u) does not match (%u)", + saved_geometry->sparse_chapters_per_volume, + geometry->sparse_chapters_per_volume); + result = false; + } + + if (saved_config->cache_chapters != user->cache_chapters) { + uds_log_error("Cache size (%u) does not match (%u)", + saved_config->cache_chapters, user->cache_chapters); + result = false; + } + + if (saved_config->volume_index_mean_delta != user->volume_index_mean_delta) { + uds_log_error("Volume index mean delta (%u) does not match (%u)", + saved_config->volume_index_mean_delta, + user->volume_index_mean_delta); + result = false; + } + + if (saved_geometry->bytes_per_page != geometry->bytes_per_page) { + uds_log_error("Bytes per page value (%zu) does not match (%zu)", + saved_geometry->bytes_per_page, geometry->bytes_per_page); + result = false; + } + + if (saved_config->sparse_sample_rate != user->sparse_sample_rate) { + uds_log_error("Sparse sample rate (%u) does not match (%u)", + saved_config->sparse_sample_rate, + user->sparse_sample_rate); + result = false; + } + + if (saved_config->nonce != user->nonce) { + uds_log_error("Nonce (%llu) does not match (%llu)", + (unsigned long long) saved_config->nonce, + (unsigned long long) user->nonce); + result = false; + } + + return result; +} + +/* Read the configuration and validate it against the provided one. */ +int uds_validate_config_contents(struct buffered_reader *reader, + struct configuration *user_config) +{ + int result; + struct configuration config; + struct geometry geometry; + u8 version_buffer[INDEX_CONFIG_VERSION_LENGTH]; + u32 bytes_per_page; + u8 buffer[sizeof(struct uds_configuration_6_02)]; + size_t offset = 0; + + result = uds_verify_buffered_data(reader, INDEX_CONFIG_MAGIC, + INDEX_CONFIG_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, version_buffer, + INDEX_CONFIG_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "cannot read index config version"); + + if (!is_version(INDEX_CONFIG_VERSION_6_02, version_buffer) && + !is_version(INDEX_CONFIG_VERSION_8_02, version_buffer)) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "unsupported configuration version: '%.*s'", + INDEX_CONFIG_VERSION_LENGTH, + version_buffer); + } + + result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "cannot read config data"); + + decode_u32_le(buffer, &offset, &geometry.record_pages_per_chapter); + decode_u32_le(buffer, &offset, &geometry.chapters_per_volume); + decode_u32_le(buffer, &offset, &geometry.sparse_chapters_per_volume); + decode_u32_le(buffer, &offset, &config.cache_chapters); + offset += sizeof(u32); + decode_u32_le(buffer, &offset, &config.volume_index_mean_delta); + decode_u32_le(buffer, &offset, &bytes_per_page); + geometry.bytes_per_page = bytes_per_page; + decode_u32_le(buffer, &offset, &config.sparse_sample_rate); + decode_u64_le(buffer, &offset, &config.nonce); + + result = ASSERT(offset == sizeof(struct uds_configuration_6_02), + "%zu bytes read but not decoded", + sizeof(struct uds_configuration_6_02) - offset); + if (result != UDS_SUCCESS) + return UDS_CORRUPT_DATA; + + if (is_version(INDEX_CONFIG_VERSION_6_02, version_buffer)) { + user_config->geometry->remapped_virtual = 0; + user_config->geometry->remapped_physical = 0; + } else { + u8 remapping[sizeof(u64) + sizeof(u64)]; + + result = uds_read_from_buffered_reader(reader, remapping, + sizeof(remapping)); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "cannot read converted config"); + + offset = 0; + decode_u64_le(remapping, &offset, + &user_config->geometry->remapped_virtual); + decode_u64_le(remapping, &offset, + &user_config->geometry->remapped_physical); + } + + if (!are_matching_configurations(&config, &geometry, user_config)) { + uds_log_warning("Supplied configuration does not match save"); + return UDS_NO_INDEX; + } + + return UDS_SUCCESS; +} + +/* + * Write the configuration to stable storage. If the superblock version is < 4, write the 6.02 + * version; otherwise write the 8.02 version, indicating the configuration is for an index that has + * been reduced by one chapter. + */ +int uds_write_config_contents(struct buffered_writer *writer, + struct configuration *config, u32 version) +{ + int result; + struct geometry *geometry = config->geometry; + u8 buffer[sizeof(struct uds_configuration_8_02)]; + size_t offset = 0; + + result = uds_write_to_buffered_writer(writer, INDEX_CONFIG_MAGIC, + INDEX_CONFIG_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + /* + * If version is < 4, the index has not been reduced by a chapter so it must be written out + * as version 6.02 so that it is still compatible with older versions of UDS. + */ + if (version >= 4) { + result = uds_write_to_buffered_writer(writer, INDEX_CONFIG_VERSION_8_02, + INDEX_CONFIG_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return result; + } else { + result = uds_write_to_buffered_writer(writer, INDEX_CONFIG_VERSION_6_02, + INDEX_CONFIG_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return result; + } + + encode_u32_le(buffer, &offset, geometry->record_pages_per_chapter); + encode_u32_le(buffer, &offset, geometry->chapters_per_volume); + encode_u32_le(buffer, &offset, geometry->sparse_chapters_per_volume); + encode_u32_le(buffer, &offset, config->cache_chapters); + encode_u32_le(buffer, &offset, 0); + encode_u32_le(buffer, &offset, config->volume_index_mean_delta); + encode_u32_le(buffer, &offset, geometry->bytes_per_page); + encode_u32_le(buffer, &offset, config->sparse_sample_rate); + encode_u64_le(buffer, &offset, config->nonce); + + result = ASSERT(offset == sizeof(struct uds_configuration_6_02), + "%zu bytes encoded, of %zu expected", offset, + sizeof(struct uds_configuration_6_02)); + if (result != UDS_SUCCESS) + return result; + + if (version >= 4) { + encode_u64_le(buffer, &offset, geometry->remapped_virtual); + encode_u64_le(buffer, &offset, geometry->remapped_physical); + } + + return uds_write_to_buffered_writer(writer, buffer, offset); +} + +/* Compute configuration parameters that depend on memory size. */ +static int compute_memory_sizes(uds_memory_config_size_t mem_gb, bool sparse, + u32 *chapters_per_volume, u32 *record_pages_per_chapter, + u32 *sparse_chapters_per_volume) +{ + u32 reduced_chapters = 0; + u32 base_chapters; + + if (mem_gb == UDS_MEMORY_CONFIG_256MB) { + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_512MB) { + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 2 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_768MB) { + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 3 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if ((mem_gb >= 1) && (mem_gb <= UDS_MEMORY_CONFIG_MAX)) { + base_chapters = mem_gb * DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = DEFAULT_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_REDUCED_256MB) { + reduced_chapters = 1; + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_REDUCED_512MB) { + reduced_chapters = 1; + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 2 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_REDUCED_768MB) { + reduced_chapters = 1; + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 3 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if ((mem_gb >= 1 + UDS_MEMORY_CONFIG_REDUCED) && + (mem_gb <= UDS_MEMORY_CONFIG_REDUCED_MAX)) { + reduced_chapters = 1; + base_chapters = ((mem_gb - UDS_MEMORY_CONFIG_REDUCED) * + DEFAULT_CHAPTERS_PER_VOLUME); + *record_pages_per_chapter = DEFAULT_RECORD_PAGES_PER_CHAPTER; + } else { + uds_log_error("received invalid memory size"); + return -EINVAL; + } + + if (sparse) { + /* Make 95% of chapters sparse, allowing 10x more records. */ + *sparse_chapters_per_volume = (19 * base_chapters) / 2; + base_chapters *= 10; + } else { + *sparse_chapters_per_volume = 0; + } + + *chapters_per_volume = base_chapters - reduced_chapters; + return UDS_SUCCESS; +} + +static unsigned int __must_check normalize_zone_count(unsigned int requested) +{ + unsigned int zone_count = requested; + + if (zone_count == 0) + zone_count = num_online_cpus() / 2; + + if (zone_count < 1) + zone_count = 1; + + if (zone_count > MAX_ZONES) + zone_count = MAX_ZONES; + + uds_log_info("Using %u indexing zone%s for concurrency.", + zone_count, zone_count == 1 ? "" : "s"); + return zone_count; +} + +static unsigned int __must_check normalize_read_threads(unsigned int requested) +{ + unsigned int read_threads = requested; + + if (read_threads < 1) + read_threads = DEFAULT_VOLUME_READ_THREADS; + + if (read_threads > MAX_VOLUME_READ_THREADS) + read_threads = MAX_VOLUME_READ_THREADS; + + return read_threads; +} + +int uds_make_configuration(const struct uds_parameters *params, + struct configuration **config_ptr) +{ + struct configuration *config; + u32 chapters_per_volume = 0; + u32 record_pages_per_chapter = 0; + u32 sparse_chapters_per_volume = 0; + int result; + + result = compute_memory_sizes(params->memory_size, params->sparse, + &chapters_per_volume, &record_pages_per_chapter, + &sparse_chapters_per_volume); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(1, struct configuration, __func__, &config); + if (result != UDS_SUCCESS) + return result; + + result = uds_make_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter, + chapters_per_volume, sparse_chapters_per_volume, 0, 0, + &config->geometry); + if (result != UDS_SUCCESS) { + uds_free_configuration(config); + return result; + } + + config->zone_count = normalize_zone_count(params->zone_count); + config->read_threads = normalize_read_threads(params->read_threads); + + config->cache_chapters = DEFAULT_CACHE_CHAPTERS; + config->volume_index_mean_delta = DEFAULT_VOLUME_INDEX_MEAN_DELTA; + config->sparse_sample_rate = (params->sparse ? DEFAULT_SPARSE_SAMPLE_RATE : 0); + config->nonce = params->nonce; + config->bdev = params->bdev; + config->offset = params->offset; + config->size = params->size; + + *config_ptr = config; + return UDS_SUCCESS; +} + +void uds_free_configuration(struct configuration *config) +{ + if (config != NULL) { + uds_free_geometry(config->geometry); + uds_free(config); + } +} + +void uds_log_configuration(struct configuration *config) +{ + struct geometry *geometry = config->geometry; + + uds_log_debug("Configuration:"); + uds_log_debug(" Record pages per chapter: %10u", geometry->record_pages_per_chapter); + uds_log_debug(" Chapters per volume: %10u", geometry->chapters_per_volume); + uds_log_debug(" Sparse chapters per volume: %10u", geometry->sparse_chapters_per_volume); + uds_log_debug(" Cache size (chapters): %10u", config->cache_chapters); + uds_log_debug(" Volume index mean delta: %10u", config->volume_index_mean_delta); + uds_log_debug(" Bytes per page: %10zu", geometry->bytes_per_page); + uds_log_debug(" Sparse sample rate: %10u", config->sparse_sample_rate); + uds_log_debug(" Nonce: %llu", (unsigned long long) config->nonce); +} diff --git a/drivers/md/dm-vdo/config.h b/drivers/md/dm-vdo/config.h new file mode 100644 index 000000000000..72430d47f983 --- /dev/null +++ b/drivers/md/dm-vdo/config.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_CONFIG_H +#define UDS_CONFIG_H + +#include "geometry.h" +#include "io-factory.h" +#include "uds.h" + +/* + * The configuration records a variety of parameters used to configure a new UDS index. Some + * parameters are provided by the client, while others are fixed or derived from user-supplied + * values. It is created when an index is created, and it is recorded in the index metadata. + */ + +enum { + DEFAULT_VOLUME_INDEX_MEAN_DELTA = 4096, + DEFAULT_CACHE_CHAPTERS = 7, + DEFAULT_SPARSE_SAMPLE_RATE = 32, + MAX_ZONES = 16, +}; + +/* A set of configuration parameters for the indexer. */ +struct configuration { + /* Storage device for the index */ + struct block_device *bdev; + + /* The maximum allowable size of the index */ + size_t size; + + /* The offset where the index should start */ + off_t offset; + + /* Parameters for the volume */ + + /* The volume layout */ + struct geometry *geometry; + + /* Index owner's nonce */ + u64 nonce; + + /* The number of threads used to process index requests */ + unsigned int zone_count; + + /* The number of threads used to read volume pages */ + unsigned int read_threads; + + /* Size of the page cache and sparse chapter index cache in chapters */ + u32 cache_chapters; + + /* Parameters for the volume index */ + + /* The mean delta for the volume index */ + u32 volume_index_mean_delta; + + /* Sampling rate for sparse indexing */ + u32 sparse_sample_rate; +}; + +/* On-disk structure of data for a version 8.02 index. */ +struct uds_configuration_8_02 { + /* Smaller (16), Small (64) or large (256) indices */ + u32 record_pages_per_chapter; + /* Total number of chapters per volume */ + u32 chapters_per_volume; + /* Number of sparse chapters per volume */ + u32 sparse_chapters_per_volume; + /* Size of the page cache, in chapters */ + u32 cache_chapters; + /* Unused field */ + u32 unused; + /* The volume index mean delta to use */ + u32 volume_index_mean_delta; + /* Size of a page, used for both record pages and index pages */ + u32 bytes_per_page; + /* Sampling rate for sparse indexing */ + u32 sparse_sample_rate; + /* Index owner's nonce */ + u64 nonce; + /* Virtual chapter remapped from physical chapter 0 */ + u64 remapped_virtual; + /* New physical chapter which remapped chapter was moved to */ + u64 remapped_physical; +} __packed; + +/* On-disk structure of data for a version 6.02 index. */ +struct uds_configuration_6_02 { + /* Smaller (16), Small (64) or large (256) indices */ + u32 record_pages_per_chapter; + /* Total number of chapters per volume */ + u32 chapters_per_volume; + /* Number of sparse chapters per volume */ + u32 sparse_chapters_per_volume; + /* Size of the page cache, in chapters */ + u32 cache_chapters; + /* Unused field */ + u32 unused; + /* The volume index mean delta to use */ + u32 volume_index_mean_delta; + /* Size of a page, used for both record pages and index pages */ + u32 bytes_per_page; + /* Sampling rate for sparse indexing */ + u32 sparse_sample_rate; + /* Index owner's nonce */ + u64 nonce; +} __packed; + +int __must_check uds_make_configuration(const struct uds_parameters *params, + struct configuration **config_ptr); + +void uds_free_configuration(struct configuration *config); + +int __must_check uds_validate_config_contents(struct buffered_reader *reader, + struct configuration *config); + +int __must_check uds_write_config_contents(struct buffered_writer *writer, + struct configuration *config, u32 version); + +void uds_log_configuration(struct configuration *config); + +#endif /* UDS_CONFIG_H */ diff --git a/drivers/md/dm-vdo/geometry.c b/drivers/md/dm-vdo/geometry.c new file mode 100644 index 000000000000..1ebbef75f842 --- /dev/null +++ b/drivers/md/dm-vdo/geometry.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "geometry.h" + +#include +#include + +#include "delta-index.h" +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "uds.h" + +/* + * An index volume is divided into a fixed number of fixed-size chapters, each consisting of a + * fixed number of fixed-size pages. The volume layout is defined by two constants and four + * parameters. The constants are that index records are 32 bytes long (16-byte block name plus + * 16-byte metadata) and that open chapter index hash slots are one byte long. The four parameters + * are the number of bytes in a page, the number of record pages in a chapter, the number of + * chapters in a volume, and the number of chapters that are sparse. From these parameters, we can + * derive the rest of the layout and other index properties. + * + * The index volume is sized by its maximum memory footprint. For a dense index, the persistent + * storage is about 10 times the size of the memory footprint. For a sparse index, the persistent + * storage is about 100 times the size of the memory footprint. + * + * For a small index with a memory footprint less than 1GB, there are three possible memory + * configurations: 0.25GB, 0.5GB and 0.75GB. The default geometry for each is 1024 index records + * per 32 KB page, 1024 chapters per volume, and either 64, 128, or 192 record pages per chapter + * (resulting in 6, 13, or 20 index pages per chapter) depending on the memory configuration. For + * the VDO default of a 0.25 GB index, this yields a deduplication window of 256 GB using about 2.5 + * GB for the persistent storage and 256 MB of RAM. + * + * For a larger index with a memory footprint that is a multiple of 1 GB, the geometry is 1024 + * index records per 32 KB page, 256 record pages per chapter, 26 index pages per chapter, and 1024 + * chapters for every GB of memory footprint. For a 1 GB volume, this yields a deduplication window + * of 1 TB using about 9GB of persistent storage and 1 GB of RAM. + * + * The above numbers hold for volumes which have no sparse chapters. A sparse volume has 10 times + * as many chapters as the corresponding non-sparse volume, which provides 10 times the + * deduplication window while using 10 times as much persistent storage as the equivalent + * non-sparse volume with the same memory footprint. + * + * If the volume has been converted from a non-lvm format to an lvm volume, the number of chapters + * per volume will have been reduced by one by eliminating physical chapter 0, and the virtual + * chapter that formerly mapped to physical chapter 0 may be remapped to another physical chapter. + * This remapping is expressed by storing which virtual chapter was remapped, and which physical + * chapter it was moved to. + */ + +int uds_make_geometry(size_t bytes_per_page, + u32 record_pages_per_chapter, + u32 chapters_per_volume, + u32 sparse_chapters_per_volume, + u64 remapped_virtual, + u64 remapped_physical, + struct geometry **geometry_ptr) +{ + int result; + struct geometry *geometry; + + result = uds_allocate(1, struct geometry, "geometry", &geometry); + if (result != UDS_SUCCESS) + return result; + + geometry->bytes_per_page = bytes_per_page; + geometry->record_pages_per_chapter = record_pages_per_chapter; + geometry->chapters_per_volume = chapters_per_volume; + geometry->sparse_chapters_per_volume = sparse_chapters_per_volume; + geometry->dense_chapters_per_volume = chapters_per_volume - sparse_chapters_per_volume; + geometry->remapped_virtual = remapped_virtual; + geometry->remapped_physical = remapped_physical; + + geometry->records_per_page = bytes_per_page / BYTES_PER_RECORD; + geometry->records_per_chapter = geometry->records_per_page * record_pages_per_chapter; + geometry->records_per_volume = (u64) geometry->records_per_chapter * chapters_per_volume; + + geometry->chapter_mean_delta = 1 << DEFAULT_CHAPTER_MEAN_DELTA_BITS; + geometry->chapter_payload_bits = bits_per(record_pages_per_chapter - 1); + /* + * We want 1 delta list for every 64 records in the chapter. + * The "| 077" ensures that the chapter_delta_list_bits computation + * does not underflow. + */ + geometry->chapter_delta_list_bits = + bits_per((geometry->records_per_chapter - 1) | 077) - 6; + geometry->delta_lists_per_chapter = 1 << geometry->chapter_delta_list_bits; + /* We need enough address bits to achieve the desired mean delta. */ + geometry->chapter_address_bits = + (DEFAULT_CHAPTER_MEAN_DELTA_BITS - + geometry->chapter_delta_list_bits + + bits_per(geometry->records_per_chapter - 1)); + geometry->index_pages_per_chapter = + uds_get_delta_index_page_count(geometry->records_per_chapter, + geometry->delta_lists_per_chapter, + geometry->chapter_mean_delta, + geometry->chapter_payload_bits, + bytes_per_page); + + geometry->pages_per_chapter = geometry->index_pages_per_chapter + record_pages_per_chapter; + geometry->pages_per_volume = geometry->pages_per_chapter * chapters_per_volume; + geometry->bytes_per_volume = + bytes_per_page * (geometry->pages_per_volume + HEADER_PAGES_PER_VOLUME); + + *geometry_ptr = geometry; + return UDS_SUCCESS; +} + +int uds_copy_geometry(struct geometry *source, struct geometry **geometry_ptr) +{ + return uds_make_geometry(source->bytes_per_page, + source->record_pages_per_chapter, + source->chapters_per_volume, + source->sparse_chapters_per_volume, + source->remapped_virtual, source->remapped_physical, + geometry_ptr); +} + +void uds_free_geometry(struct geometry *geometry) +{ + uds_free(geometry); +} + +u32 __must_check uds_map_to_physical_chapter(const struct geometry *geometry, + u64 virtual_chapter) +{ + u64 delta; + + if (!uds_is_reduced_geometry(geometry)) + return virtual_chapter % geometry->chapters_per_volume; + + if (likely(virtual_chapter > geometry->remapped_virtual)) { + delta = virtual_chapter - geometry->remapped_virtual; + if (likely(delta > geometry->remapped_physical)) + return delta % geometry->chapters_per_volume; + else + return delta - 1; + } + + if (virtual_chapter == geometry->remapped_virtual) + return geometry->remapped_physical; + + delta = geometry->remapped_virtual - virtual_chapter; + if (delta < geometry->chapters_per_volume) + return geometry->chapters_per_volume - delta; + + /* This chapter is so old the answer doesn't matter. */ + return 0; +} + +/* Check whether any sparse chapters are in use. */ +bool uds_has_sparse_chapters(const struct geometry *geometry, u64 oldest_virtual_chapter, + u64 newest_virtual_chapter) +{ + return uds_is_sparse_geometry(geometry) && + ((newest_virtual_chapter - oldest_virtual_chapter + 1) > + geometry->dense_chapters_per_volume); +} + +bool uds_is_chapter_sparse(const struct geometry *geometry, u64 oldest_virtual_chapter, + u64 newest_virtual_chapter, u64 virtual_chapter_number) +{ + return uds_has_sparse_chapters(geometry, oldest_virtual_chapter, + newest_virtual_chapter) && + ((virtual_chapter_number + geometry->dense_chapters_per_volume) <= + newest_virtual_chapter); +} + +/* Calculate how many chapters to expire after opening the newest chapter. */ +u32 uds_chapters_to_expire(const struct geometry *geometry, u64 newest_chapter) +{ + /* If the index isn't full yet, don't expire anything. */ + if (newest_chapter < geometry->chapters_per_volume) + return 0; + + /* If a chapter is out of order... */ + if (geometry->remapped_physical > 0) { + u64 oldest_chapter = newest_chapter - geometry->chapters_per_volume; + + /* + * ... expire an extra chapter when expiring the moved chapter to free physical + * space for the new chapter ... + */ + if (oldest_chapter == geometry->remapped_virtual) + return 2; + + /* + * ... but don't expire anything when the new chapter will use the physical chapter + * freed by expiring the moved chapter. + */ + if (oldest_chapter == (geometry->remapped_virtual + geometry->remapped_physical)) + return 0; + } + + /* Normally, just expire one. */ + return 1; +} diff --git a/drivers/md/dm-vdo/geometry.h b/drivers/md/dm-vdo/geometry.h new file mode 100644 index 000000000000..cff69087cd79 --- /dev/null +++ b/drivers/md/dm-vdo/geometry.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_GEOMETRY_H +#define UDS_GEOMETRY_H + +#include "uds.h" + +/* + * The geometry records parameters that define the layout of a UDS index volume, and the size and + * shape of various index structures. It is created when the index is created, and is referenced by + * many index sub-components. + */ + +struct geometry { + /* Size of a chapter page, in bytes */ + size_t bytes_per_page; + /* Number of record pages in a chapter */ + u32 record_pages_per_chapter; + /* Total number of chapters in a volume */ + u32 chapters_per_volume; + /* Number of sparsely-indexed chapters in a volume */ + u32 sparse_chapters_per_volume; + /* Number of bits used to determine delta list numbers */ + u8 chapter_delta_list_bits; + /* Virtual chapter remapped from physical chapter 0 */ + u64 remapped_virtual; + /* New physical chapter where the remapped chapter can be found */ + u64 remapped_physical; + + /* + * The following properties are derived from the ones above, but they are computed and + * recorded as fields for convenience. + */ + /* Total number of pages in a volume, excluding the header */ + u32 pages_per_volume; + /* Total number of bytes in a volume, including the header */ + size_t bytes_per_volume; + /* Number of pages in a chapter */ + u32 pages_per_chapter; + /* Number of index pages in a chapter index */ + u32 index_pages_per_chapter; + /* Number of records that fit on a page */ + u32 records_per_page; + /* Number of records that fit in a chapter */ + u32 records_per_chapter; + /* Number of records that fit in a volume */ + u64 records_per_volume; + /* Number of delta lists per chapter index */ + u32 delta_lists_per_chapter; + /* Mean delta for chapter indexes */ + u32 chapter_mean_delta; + /* Number of bits needed for record page numbers */ + u8 chapter_payload_bits; + /* Number of bits used to compute addresses for chapter delta lists */ + u8 chapter_address_bits; + /* Number of densely-indexed chapters in a volume */ + u32 dense_chapters_per_volume; +}; + +enum { + /* The number of bytes in a record (name + metadata) */ + BYTES_PER_RECORD = (UDS_RECORD_NAME_SIZE + UDS_RECORD_DATA_SIZE), + + /* The default length of a page in a chapter, in bytes */ + DEFAULT_BYTES_PER_PAGE = 1024 * BYTES_PER_RECORD, + + /* The default maximum number of records per page */ + DEFAULT_RECORDS_PER_PAGE = DEFAULT_BYTES_PER_PAGE / BYTES_PER_RECORD, + + /* The default number of record pages in a chapter */ + DEFAULT_RECORD_PAGES_PER_CHAPTER = 256, + + /* The default number of record pages in a chapter for a small index */ + SMALL_RECORD_PAGES_PER_CHAPTER = 64, + + /* The default number of chapters in a volume */ + DEFAULT_CHAPTERS_PER_VOLUME = 1024, + + /* The default number of sparsely-indexed chapters in a volume */ + DEFAULT_SPARSE_CHAPTERS_PER_VOLUME = 0, + + /* The log2 of the default mean delta */ + DEFAULT_CHAPTER_MEAN_DELTA_BITS = 16, + + /* The log2 of the number of delta lists in a large chapter */ + DEFAULT_CHAPTER_DELTA_LIST_BITS = 12, + + /* The log2 of the number of delta lists in a small chapter */ + SMALL_CHAPTER_DELTA_LIST_BITS = 10, + + /* The number of header pages per volume */ + HEADER_PAGES_PER_VOLUME = 1, +}; + +int __must_check uds_make_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, + u32 chapters_per_volume, + u32 sparse_chapters_per_volume, u64 remapped_virtual, + u64 remapped_physical, + struct geometry **geometry_ptr); + +int __must_check uds_copy_geometry(struct geometry *source, + struct geometry **geometry_ptr); + +void uds_free_geometry(struct geometry *geometry); + +u32 __must_check uds_map_to_physical_chapter(const struct geometry *geometry, + u64 virtual_chapter); + +/* + * Check whether this geometry is reduced by a chapter. This will only be true if the volume was + * converted from a non-lvm volume to an lvm volume. + */ +static inline bool __must_check uds_is_reduced_geometry(const struct geometry *geometry) +{ + return !!(geometry->chapters_per_volume & 1); +} + +static inline bool __must_check uds_is_sparse_geometry(const struct geometry *geometry) +{ + return geometry->sparse_chapters_per_volume > 0; +} + +bool __must_check uds_has_sparse_chapters(const struct geometry *geometry, + u64 oldest_virtual_chapter, + u64 newest_virtual_chapter); + +bool __must_check uds_is_chapter_sparse(const struct geometry *geometry, + u64 oldest_virtual_chapter, + u64 newest_virtual_chapter, + u64 virtual_chapter_number); + +u32 __must_check uds_chapters_to_expire(const struct geometry *geometry, + u64 newest_chapter); + +#endif /* UDS_GEOMETRY_H */ From b46d79bdb82aa12bf1101c1a38f0233cefe94cb1 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:52:09 -0500 Subject: [PATCH 0558/1038] dm vdo: add deduplication index storage interface This patch adds infrastructure for managing reads and writes to the underlying storage layer for the deduplication index. The deduplication index uses dm-bufio for all of its reads and writes, so part of this infrastructure is managing the various dm-bufio clients required. It also adds the buffered reader and buffered writer abstractions, which simplify reading and writing metadata structures that span several blocks. This patch also includes structures and utilities for encoding and decoding all of the deduplication index metadata, collectively called the index layout. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: John Wiele Signed-off-by: John Wiele Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/index-layout.c | 1768 ++++++++++++++++++++++++++++++ drivers/md/dm-vdo/index-layout.h | 43 + drivers/md/dm-vdo/io-factory.c | 415 +++++++ drivers/md/dm-vdo/io-factory.h | 64 ++ drivers/md/dm-vdo/numeric.h | 78 ++ 5 files changed, 2368 insertions(+) create mode 100644 drivers/md/dm-vdo/index-layout.c create mode 100644 drivers/md/dm-vdo/index-layout.h create mode 100644 drivers/md/dm-vdo/io-factory.c create mode 100644 drivers/md/dm-vdo/io-factory.h create mode 100644 drivers/md/dm-vdo/numeric.h diff --git a/drivers/md/dm-vdo/index-layout.c b/drivers/md/dm-vdo/index-layout.c new file mode 100644 index 000000000000..4dffa82a3c43 --- /dev/null +++ b/drivers/md/dm-vdo/index-layout.c @@ -0,0 +1,1768 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "index-layout.h" + +#include + +#include "config.h" +#include "logger.h" +#include "memory-alloc.h" +#include "murmurhash3.h" +#include "numeric.h" +#include "open-chapter.h" +#include "time-utils.h" +#include "volume-index.h" + +/* + * The UDS layout on storage media is divided into a number of fixed-size regions, the sizes of + * which are computed when the index is created. Every header and region begins on 4K block + * boundary. Save regions are further sub-divided into regions of their own. + * + * Each region has a kind and an instance number. Some kinds only have one instance and therefore + * use RL_SOLE_INSTANCE (-1) as the instance number. The RL_KIND_INDEX used to use instances to + * represent sub-indices; now, however there is only ever one sub-index and therefore one instance. + * The RL_KIND_VOLUME_INDEX uses instances to record which zone is being saved. + * + * Every region header has a type and version. + * + * +-+-+---------+--------+--------+-+ + * | | | I N D E X 0 101, 0 | | + * |H|C+---------+--------+--------+S| + * |D|f| Volume | Save | Save |e| + * |R|g| Region | Region | Region |a| + * | | | 201, -1 | 202, 0 | 202, 1 |l| + * +-+-+--------+---------+--------+-+ + * + * The header contains the encoded region layout table as well as some index configuration data. + * The sub-index region and its subdivisions are maintained in the same table. + * + * There are two save regions to preserve the old state in case saving the new state is incomplete. + * They are used in alternation. Each save region is further divided into sub-regions. + * + * +-+-----+------+------+-----+-----+ + * |H| IPM | MI | MI | | OC | + * |D| | zone | zone | ... | | + * |R| 301 | 302 | 302 | | 303 | + * | | -1 | 0 | 1 | | -1 | + * +-+-----+------+------+-----+-----+ + * + * The header contains the encoded region layout table as well as index state data for that save. + * Each save also has a unique nonce. + */ + +enum { + MAGIC_SIZE = 32, + NONCE_INFO_SIZE = 32, + MAX_SAVES = 2, +}; + +enum region_kind { + RL_KIND_EMPTY = 0, + RL_KIND_HEADER = 1, + RL_KIND_CONFIG = 100, + RL_KIND_INDEX = 101, + RL_KIND_SEAL = 102, + RL_KIND_VOLUME = 201, + RL_KIND_SAVE = 202, + RL_KIND_INDEX_PAGE_MAP = 301, + RL_KIND_VOLUME_INDEX = 302, + RL_KIND_OPEN_CHAPTER = 303, +}; + +/* Some region types are historical and are no longer used. */ +enum region_type { + RH_TYPE_FREE = 0, /* unused */ + RH_TYPE_SUPER = 1, + RH_TYPE_SAVE = 2, + RH_TYPE_CHECKPOINT = 3, /* unused */ + RH_TYPE_UNSAVED = 4, +}; + +enum { + RL_SOLE_INSTANCE = 65535, +}; + +/* + * Super block version 2 is the first released version. + * + * Super block version 3 is the normal version used from RHEL 8.2 onwards. + * + * Super block versions 4 through 6 were incremental development versions and + * are not supported. + * + * Super block version 7 is used for volumes which have been reduced in size by one chapter in + * order to make room to prepend LVM metadata to a volume originally created without lvm. This + * allows the index to retain most its deduplication records. + */ +enum { + SUPER_VERSION_MINIMUM = 3, + SUPER_VERSION_CURRENT = 3, + SUPER_VERSION_MAXIMUM = 7, +}; + +static const u8 LAYOUT_MAGIC[MAGIC_SIZE] = "*ALBIREO*SINGLE*FILE*LAYOUT*001*"; +static const u64 REGION_MAGIC = 0x416c6252676e3031; /* 'AlbRgn01' */ + +struct region_header { + u64 magic; + u64 region_blocks; + u16 type; + /* Currently always version 1 */ + u16 version; + u16 region_count; + u16 payload; +}; + +struct layout_region { + u64 start_block; + u64 block_count; + u32 __unused; + u16 kind; + u16 instance; +}; + +struct region_table { + size_t encoded_size; + struct region_header header; + struct layout_region regions[]; +}; + +struct index_save_data { + u64 timestamp; + u64 nonce; + /* Currently always version 1 */ + u32 version; + u32 unused__; +}; + +struct index_state_version { + s32 signature; + s32 version_id; +}; + +static const struct index_state_version INDEX_STATE_VERSION_301 = { + .signature = -1, + .version_id = 301, +}; + +struct index_state_data301 { + struct index_state_version version; + u64 newest_chapter; + u64 oldest_chapter; + u64 last_save; + u32 unused; + u32 padding; +}; + +struct index_save_layout { + unsigned int zone_count; + struct layout_region index_save; + struct layout_region header; + struct layout_region index_page_map; + struct layout_region free_space; + struct layout_region volume_index_zones[MAX_ZONES]; + struct layout_region open_chapter; + struct index_save_data save_data; + struct index_state_data301 state_data; +}; + +struct sub_index_layout { + u64 nonce; + struct layout_region sub_index; + struct layout_region volume; + struct index_save_layout *saves; +}; + +struct super_block_data { + u8 magic_label[MAGIC_SIZE]; + u8 nonce_info[NONCE_INFO_SIZE]; + u64 nonce; + u32 version; + u32 block_size; + u16 index_count; + u16 max_saves; + /* Padding reflects a blank field on permanent storage */ + u8 padding[4]; + u64 open_chapter_blocks; + u64 page_map_blocks; + u64 volume_offset; + u64 start_offset; +}; + +struct index_layout { + struct io_factory *factory; + size_t factory_size; + off_t offset; + struct super_block_data super; + struct layout_region header; + struct layout_region config; + struct sub_index_layout index; + struct layout_region seal; + u64 total_blocks; +}; + +struct save_layout_sizes { + unsigned int save_count; + size_t block_size; + u64 volume_blocks; + u64 volume_index_blocks; + u64 page_map_blocks; + u64 open_chapter_blocks; + u64 save_blocks; + u64 sub_index_blocks; + u64 total_blocks; + size_t total_size; +}; + +static inline bool is_converted_super_block(struct super_block_data *super) +{ + return super->version == 7; +} + +static int __must_check compute_sizes(const struct configuration *config, + struct save_layout_sizes *sls) +{ + int result; + struct geometry *geometry = config->geometry; + + memset(sls, 0, sizeof(*sls)); + sls->save_count = MAX_SAVES; + sls->block_size = UDS_BLOCK_SIZE; + sls->volume_blocks = geometry->bytes_per_volume / sls->block_size; + + result = uds_compute_volume_index_save_blocks(config, sls->block_size, + &sls->volume_index_blocks); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "cannot compute index save size"); + + sls->page_map_blocks = + DIV_ROUND_UP(uds_compute_index_page_map_save_size(geometry), + sls->block_size); + sls->open_chapter_blocks = + DIV_ROUND_UP(uds_compute_saved_open_chapter_size(geometry), + sls->block_size); + sls->save_blocks = + 1 + (sls->volume_index_blocks + sls->page_map_blocks + sls->open_chapter_blocks); + sls->sub_index_blocks = sls->volume_blocks + (sls->save_count * sls->save_blocks); + sls->total_blocks = 3 + sls->sub_index_blocks; + sls->total_size = sls->total_blocks * sls->block_size; + + return UDS_SUCCESS; +} + +int uds_compute_index_size(const struct uds_parameters *parameters, u64 *index_size) +{ + int result; + struct configuration *index_config; + struct save_layout_sizes sizes; + + if (index_size == NULL) { + uds_log_error("Missing output size pointer"); + return -EINVAL; + } + + result = uds_make_configuration(parameters, &index_config); + if (result != UDS_SUCCESS) { + uds_log_error_strerror(result, "cannot compute index size"); + return uds_map_to_system_error(result); + } + + result = compute_sizes(index_config, &sizes); + uds_free_configuration(index_config); + if (result != UDS_SUCCESS) + return uds_map_to_system_error(result); + + *index_size = sizes.total_size; + return UDS_SUCCESS; +} + +/* Create unique data using the current time and a pseudorandom number. */ +static void create_unique_nonce_data(u8 *buffer) +{ + ktime_t now = current_time_ns(CLOCK_REALTIME); + u32 rand; + size_t offset = 0; + + get_random_bytes(&rand, sizeof(u32)); + memcpy(buffer + offset, &now, sizeof(now)); + offset += sizeof(now); + memcpy(buffer + offset, &rand, sizeof(rand)); + offset += sizeof(rand); + while (offset < NONCE_INFO_SIZE) { + size_t len = min(NONCE_INFO_SIZE - offset, offset); + + memcpy(buffer + offset, buffer, len); + offset += len; + } +} + +static u64 hash_stuff(u64 start, const void *data, size_t len) +{ + u32 seed = start ^ (start >> 27); + u8 hash_buffer[16]; + + murmurhash3_128(data, len, seed, hash_buffer); + return get_unaligned_le64(hash_buffer + 4); +} + +/* Generate a primary nonce from the provided data. */ +static u64 generate_primary_nonce(const void *data, size_t len) +{ + return hash_stuff(0xa1b1e0fc, data, len); +} + +/* + * Deterministically generate a secondary nonce from an existing nonce and some arbitrary data by + * hashing the original nonce and the data to produce a new nonce. + */ +static u64 generate_secondary_nonce(u64 nonce, const void *data, size_t len) +{ + return hash_stuff(nonce + 1, data, len); +} + +static int __must_check open_layout_reader(struct index_layout *layout, + struct layout_region *lr, off_t offset, + struct buffered_reader **reader_ptr) +{ + return uds_make_buffered_reader(layout->factory, lr->start_block + offset, + lr->block_count, reader_ptr); +} + +static int open_region_reader(struct index_layout *layout, struct layout_region *region, + struct buffered_reader **reader_ptr) +{ + return open_layout_reader(layout, region, -layout->super.start_offset, + reader_ptr); +} + +static int __must_check open_layout_writer(struct index_layout *layout, + struct layout_region *lr, off_t offset, + struct buffered_writer **writer_ptr) +{ + return uds_make_buffered_writer(layout->factory, lr->start_block + offset, + lr->block_count, writer_ptr); +} + +static int open_region_writer(struct index_layout *layout, struct layout_region *region, + struct buffered_writer **writer_ptr) +{ + return open_layout_writer(layout, region, -layout->super.start_offset, + writer_ptr); +} + +static void generate_super_block_data(struct save_layout_sizes *sls, + struct super_block_data *super) +{ + memset(super, 0, sizeof(*super)); + memcpy(super->magic_label, LAYOUT_MAGIC, MAGIC_SIZE); + create_unique_nonce_data(super->nonce_info); + + super->nonce = generate_primary_nonce(super->nonce_info, + sizeof(super->nonce_info)); + super->version = SUPER_VERSION_CURRENT; + super->block_size = sls->block_size; + super->index_count = 1; + super->max_saves = sls->save_count; + super->open_chapter_blocks = sls->open_chapter_blocks; + super->page_map_blocks = sls->page_map_blocks; + super->volume_offset = 0; + super->start_offset = 0; +} + +static void define_sub_index_nonce(struct index_layout *layout) +{ + struct sub_index_nonce_data { + u64 offset; + u16 index_id; + }; + struct sub_index_layout *sil = &layout->index; + u64 primary_nonce = layout->super.nonce; + u8 buffer[sizeof(struct sub_index_nonce_data)] = { 0 }; + size_t offset = 0; + + encode_u64_le(buffer, &offset, sil->sub_index.start_block); + encode_u16_le(buffer, &offset, 0); + sil->nonce = generate_secondary_nonce(primary_nonce, buffer, sizeof(buffer)); + if (sil->nonce == 0) + sil->nonce = generate_secondary_nonce(~primary_nonce + 1, buffer, + sizeof(buffer)); +} + +static void setup_sub_index(struct index_layout *layout, u64 start_block, + struct save_layout_sizes *sls) +{ + struct sub_index_layout *sil = &layout->index; + u64 next_block = start_block; + unsigned int i; + + sil->sub_index = (struct layout_region) { + .start_block = start_block, + .block_count = sls->sub_index_blocks, + .kind = RL_KIND_INDEX, + .instance = 0, + }; + + sil->volume = (struct layout_region) { + .start_block = next_block, + .block_count = sls->volume_blocks, + .kind = RL_KIND_VOLUME, + .instance = RL_SOLE_INSTANCE, + }; + + next_block += sls->volume_blocks; + + for (i = 0; i < sls->save_count; i++) { + sil->saves[i].index_save = (struct layout_region) { + .start_block = next_block, + .block_count = sls->save_blocks, + .kind = RL_KIND_SAVE, + .instance = i, + }; + + next_block += sls->save_blocks; + } + + define_sub_index_nonce(layout); +} + +static void initialize_layout(struct index_layout *layout, struct save_layout_sizes *sls) +{ + u64 next_block = layout->offset / sls->block_size; + + layout->total_blocks = sls->total_blocks; + generate_super_block_data(sls, &layout->super); + layout->header = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_HEADER, + .instance = RL_SOLE_INSTANCE, + }; + + layout->config = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_CONFIG, + .instance = RL_SOLE_INSTANCE, + }; + + setup_sub_index(layout, next_block, sls); + next_block += sls->sub_index_blocks; + + layout->seal = (struct layout_region) { + .start_block = next_block, + .block_count = 1, + .kind = RL_KIND_SEAL, + .instance = RL_SOLE_INSTANCE, + }; +} + +static int __must_check make_index_save_region_table(struct index_save_layout *isl, + struct region_table **table_ptr) +{ + int result; + unsigned int z; + struct region_table *table; + struct layout_region *lr; + u16 region_count; + size_t payload; + size_t type; + + if (isl->zone_count > 0) { + /* + * Normal save regions: header, page map, volume index zones, + * open chapter, and possibly free space. + */ + region_count = 3 + isl->zone_count; + if (isl->free_space.block_count > 0) + region_count++; + + payload = sizeof(isl->save_data) + sizeof(isl->state_data); + type = RH_TYPE_SAVE; + } else { + /* Empty save regions: header, page map, free space. */ + region_count = 3; + payload = sizeof(isl->save_data); + type = RH_TYPE_UNSAVED; + } + + result = uds_allocate_extended(struct region_table, region_count, + struct layout_region, + "layout region table for ISL", &table); + if (result != UDS_SUCCESS) + return result; + + lr = &table->regions[0]; + *lr++ = isl->header; + *lr++ = isl->index_page_map; + for (z = 0; z < isl->zone_count; z++) + *lr++ = isl->volume_index_zones[z]; + + if (isl->zone_count > 0) + *lr++ = isl->open_chapter; + + if (isl->free_space.block_count > 0) + *lr++ = isl->free_space; + + table->header = (struct region_header) { + .magic = REGION_MAGIC, + .region_blocks = isl->index_save.block_count, + .type = type, + .version = 1, + .region_count = region_count, + .payload = payload, + }; + + table->encoded_size = (sizeof(struct region_header) + payload + + region_count * sizeof(struct layout_region)); + *table_ptr = table; + return UDS_SUCCESS; +} + +static void encode_region_table(u8 *buffer, size_t *offset, struct region_table *table) +{ + unsigned int i; + + encode_u64_le(buffer, offset, REGION_MAGIC); + encode_u64_le(buffer, offset, table->header.region_blocks); + encode_u16_le(buffer, offset, table->header.type); + encode_u16_le(buffer, offset, table->header.version); + encode_u16_le(buffer, offset, table->header.region_count); + encode_u16_le(buffer, offset, table->header.payload); + + for (i = 0; i < table->header.region_count; i++) { + encode_u64_le(buffer, offset, table->regions[i].start_block); + encode_u64_le(buffer, offset, table->regions[i].block_count); + encode_u32_le(buffer, offset, 0); + encode_u16_le(buffer, offset, table->regions[i].kind); + encode_u16_le(buffer, offset, table->regions[i].instance); + } +} + +static int __must_check write_index_save_header(struct index_save_layout *isl, + struct region_table *table, + struct buffered_writer *writer) +{ + int result; + u8 *buffer; + size_t offset = 0; + + result = uds_allocate(table->encoded_size, u8, "index save data", &buffer); + if (result != UDS_SUCCESS) + return result; + + encode_region_table(buffer, &offset, table); + encode_u64_le(buffer, &offset, isl->save_data.timestamp); + encode_u64_le(buffer, &offset, isl->save_data.nonce); + encode_u32_le(buffer, &offset, isl->save_data.version); + encode_u32_le(buffer, &offset, 0); + if (isl->zone_count > 0) { + encode_u32_le(buffer, &offset, INDEX_STATE_VERSION_301.signature); + encode_u32_le(buffer, &offset, INDEX_STATE_VERSION_301.version_id); + encode_u64_le(buffer, &offset, isl->state_data.newest_chapter); + encode_u64_le(buffer, &offset, isl->state_data.oldest_chapter); + encode_u64_le(buffer, &offset, isl->state_data.last_save); + encode_u64_le(buffer, &offset, 0); + } + + result = uds_write_to_buffered_writer(writer, buffer, offset); + uds_free(buffer); + if (result != UDS_SUCCESS) + return result; + + return uds_flush_buffered_writer(writer); +} + +static int write_index_save_layout(struct index_layout *layout, + struct index_save_layout *isl) +{ + int result; + struct region_table *table; + struct buffered_writer *writer; + + result = make_index_save_region_table(isl, &table); + if (result != UDS_SUCCESS) + return result; + + result = open_region_writer(layout, &isl->header, &writer); + if (result != UDS_SUCCESS) { + uds_free(table); + return result; + } + + result = write_index_save_header(isl, table, writer); + uds_free(table); + uds_free_buffered_writer(writer); + + return result; +} + +static void reset_index_save_layout(struct index_save_layout *isl, u64 page_map_blocks) +{ + u64 free_blocks; + u64 next_block = isl->index_save.start_block; + + isl->zone_count = 0; + memset(&isl->save_data, 0, sizeof(isl->save_data)); + + isl->header = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_HEADER, + .instance = RL_SOLE_INSTANCE, + }; + + isl->index_page_map = (struct layout_region) { + .start_block = next_block, + .block_count = page_map_blocks, + .kind = RL_KIND_INDEX_PAGE_MAP, + .instance = RL_SOLE_INSTANCE, + }; + + next_block += page_map_blocks; + + free_blocks = isl->index_save.block_count - page_map_blocks - 1; + isl->free_space = (struct layout_region) { + .start_block = next_block, + .block_count = free_blocks, + .kind = RL_KIND_EMPTY, + .instance = RL_SOLE_INSTANCE, + }; +} + +static int __must_check invalidate_old_save(struct index_layout *layout, + struct index_save_layout *isl) +{ + reset_index_save_layout(isl, layout->super.page_map_blocks); + return write_index_save_layout(layout, isl); +} + +static int discard_index_state_data(struct index_layout *layout) +{ + int result; + int saved_result = UDS_SUCCESS; + unsigned int i; + + for (i = 0; i < layout->super.max_saves; i++) { + result = invalidate_old_save(layout, &layout->index.saves[i]); + if (result != UDS_SUCCESS) + saved_result = result; + } + + if (saved_result != UDS_SUCCESS) + return uds_log_error_strerror(result, + "%s: cannot destroy all index saves", + __func__); + + return UDS_SUCCESS; +} + +static int __must_check make_layout_region_table(struct index_layout *layout, + struct region_table **table_ptr) +{ + int result; + unsigned int i; + /* Regions: header, config, index, volume, saves, seal */ + u16 region_count = 5 + layout->super.max_saves; + u16 payload; + struct region_table *table; + struct layout_region *lr; + + result = uds_allocate_extended(struct region_table, region_count, + struct layout_region, "layout region table", + &table); + if (result != UDS_SUCCESS) + return result; + + lr = &table->regions[0]; + *lr++ = layout->header; + *lr++ = layout->config; + *lr++ = layout->index.sub_index; + *lr++ = layout->index.volume; + + for (i = 0; i < layout->super.max_saves; i++) + *lr++ = layout->index.saves[i].index_save; + + *lr++ = layout->seal; + + if (is_converted_super_block(&layout->super)) { + payload = sizeof(struct super_block_data); + } else { + payload = (sizeof(struct super_block_data) - + sizeof(layout->super.volume_offset) - + sizeof(layout->super.start_offset)); + } + + table->header = (struct region_header) { + .magic = REGION_MAGIC, + .region_blocks = layout->total_blocks, + .type = RH_TYPE_SUPER, + .version = 1, + .region_count = region_count, + .payload = payload, + }; + + table->encoded_size = (sizeof(struct region_header) + payload + + region_count * sizeof(struct layout_region)); + *table_ptr = table; + return UDS_SUCCESS; +} + +static int __must_check write_layout_header(struct index_layout *layout, + struct region_table *table, + struct buffered_writer *writer) +{ + int result; + u8 *buffer; + size_t offset = 0; + + result = uds_allocate(table->encoded_size, u8, "layout data", &buffer); + if (result != UDS_SUCCESS) + return result; + + encode_region_table(buffer, &offset, table); + memcpy(buffer + offset, &layout->super.magic_label, MAGIC_SIZE); + offset += MAGIC_SIZE; + memcpy(buffer + offset, &layout->super.nonce_info, NONCE_INFO_SIZE); + offset += NONCE_INFO_SIZE; + encode_u64_le(buffer, &offset, layout->super.nonce); + encode_u32_le(buffer, &offset, layout->super.version); + encode_u32_le(buffer, &offset, layout->super.block_size); + encode_u16_le(buffer, &offset, layout->super.index_count); + encode_u16_le(buffer, &offset, layout->super.max_saves); + encode_u32_le(buffer, &offset, 0); + encode_u64_le(buffer, &offset, layout->super.open_chapter_blocks); + encode_u64_le(buffer, &offset, layout->super.page_map_blocks); + + if (is_converted_super_block(&layout->super)) { + encode_u64_le(buffer, &offset, layout->super.volume_offset); + encode_u64_le(buffer, &offset, layout->super.start_offset); + } + + result = uds_write_to_buffered_writer(writer, buffer, offset); + uds_free(buffer); + if (result != UDS_SUCCESS) + return result; + + return uds_flush_buffered_writer(writer); +} + +static int __must_check write_uds_index_config(struct index_layout *layout, + struct configuration *config, + off_t offset) +{ + int result; + struct buffered_writer *writer = NULL; + + result = open_layout_writer(layout, &layout->config, offset, &writer); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "failed to open config region"); + + result = uds_write_config_contents(writer, config, layout->super.version); + if (result != UDS_SUCCESS) { + uds_free_buffered_writer(writer); + return uds_log_error_strerror(result, "failed to write config region"); + } + + result = uds_flush_buffered_writer(writer); + if (result != UDS_SUCCESS) { + uds_free_buffered_writer(writer); + return uds_log_error_strerror(result, "cannot flush config writer"); + } + + uds_free_buffered_writer(writer); + return UDS_SUCCESS; +} + +static int __must_check save_layout(struct index_layout *layout, off_t offset) +{ + int result; + struct buffered_writer *writer = NULL; + struct region_table *table; + + result = make_layout_region_table(layout, &table); + if (result != UDS_SUCCESS) + return result; + + result = open_layout_writer(layout, &layout->header, offset, &writer); + if (result != UDS_SUCCESS) { + uds_free(table); + return result; + } + + result = write_layout_header(layout, table, writer); + uds_free(table); + uds_free_buffered_writer(writer); + + return result; +} + +static int create_index_layout(struct index_layout *layout, struct configuration *config) +{ + int result; + struct save_layout_sizes sizes; + + result = compute_sizes(config, &sizes); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(sizes.save_count, struct index_save_layout, __func__, + &layout->index.saves); + if (result != UDS_SUCCESS) + return result; + + initialize_layout(layout, &sizes); + + result = discard_index_state_data(layout); + if (result != UDS_SUCCESS) + return result; + + result = write_uds_index_config(layout, config, 0); + if (result != UDS_SUCCESS) + return result; + + return save_layout(layout, 0); +} + +static u64 generate_index_save_nonce(u64 volume_nonce, struct index_save_layout *isl) +{ + struct save_nonce_data { + struct index_save_data data; + u64 offset; + } nonce_data; + u8 buffer[sizeof(nonce_data)]; + size_t offset = 0; + + encode_u64_le(buffer, &offset, isl->save_data.timestamp); + encode_u64_le(buffer, &offset, 0); + encode_u32_le(buffer, &offset, isl->save_data.version); + encode_u32_le(buffer, &offset, 0U); + encode_u64_le(buffer, &offset, isl->index_save.start_block); + ASSERT_LOG_ONLY(offset == sizeof(nonce_data), + "%zu bytes encoded of %zu expected", offset, sizeof(nonce_data)); + return generate_secondary_nonce(volume_nonce, buffer, sizeof(buffer)); +} + +static u64 validate_index_save_layout(struct index_save_layout *isl, u64 volume_nonce) +{ + if ((isl->zone_count == 0) || (isl->save_data.timestamp == 0)) + return 0; + + if (isl->save_data.nonce != generate_index_save_nonce(volume_nonce, isl)) + return 0; + + return isl->save_data.timestamp; +} + +static int find_latest_uds_index_save_slot(struct index_layout *layout, + struct index_save_layout **isl_ptr) +{ + struct index_save_layout *latest = NULL; + struct index_save_layout *isl; + unsigned int i; + u64 save_time = 0; + u64 latest_time = 0; + + for (i = 0; i < layout->super.max_saves; i++) { + isl = &layout->index.saves[i]; + save_time = validate_index_save_layout(isl, layout->index.nonce); + if (save_time > latest_time) { + latest = isl; + latest_time = save_time; + } + } + + if (latest == NULL) { + uds_log_error("No valid index save found"); + return UDS_INDEX_NOT_SAVED_CLEANLY; + } + + *isl_ptr = latest; + return UDS_SUCCESS; +} + +int uds_discard_open_chapter(struct index_layout *layout) +{ + int result; + struct index_save_layout *isl; + struct buffered_writer *writer; + + result = find_latest_uds_index_save_slot(layout, &isl); + if (result != UDS_SUCCESS) + return result; + + result = open_region_writer(layout, &isl->open_chapter, &writer); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(writer, NULL, UDS_BLOCK_SIZE); + if (result != UDS_SUCCESS) { + uds_free_buffered_writer(writer); + return result; + } + + result = uds_flush_buffered_writer(writer); + uds_free_buffered_writer(writer); + return result; +} + +int uds_load_index_state(struct index_layout *layout, struct uds_index *index) +{ + int result; + unsigned int zone; + struct index_save_layout *isl; + struct buffered_reader *readers[MAX_ZONES]; + + result = find_latest_uds_index_save_slot(layout, &isl); + if (result != UDS_SUCCESS) + return result; + + index->newest_virtual_chapter = isl->state_data.newest_chapter; + index->oldest_virtual_chapter = isl->state_data.oldest_chapter; + index->last_save = isl->state_data.last_save; + + result = open_region_reader(layout, &isl->open_chapter, &readers[0]); + if (result != UDS_SUCCESS) + return result; + + result = uds_load_open_chapter(index, readers[0]); + uds_free_buffered_reader(readers[0]); + if (result != UDS_SUCCESS) + return result; + + for (zone = 0; zone < isl->zone_count; zone++) { + result = open_region_reader(layout, &isl->volume_index_zones[zone], + &readers[zone]); + if (result != UDS_SUCCESS) { + for (; zone > 0; zone--) + uds_free_buffered_reader(readers[zone - 1]); + + return result; + } + } + + result = uds_load_volume_index(index->volume_index, readers, isl->zone_count); + for (zone = 0; zone < isl->zone_count; zone++) + uds_free_buffered_reader(readers[zone]); + if (result != UDS_SUCCESS) + return result; + + result = open_region_reader(layout, &isl->index_page_map, &readers[0]); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_index_page_map(index->volume->index_page_map, readers[0]); + uds_free_buffered_reader(readers[0]); + + return result; +} + +static struct index_save_layout *select_oldest_index_save_layout(struct index_layout *layout) +{ + struct index_save_layout *oldest = NULL; + struct index_save_layout *isl; + unsigned int i; + u64 save_time = 0; + u64 oldest_time = 0; + + for (i = 0; i < layout->super.max_saves; i++) { + isl = &layout->index.saves[i]; + save_time = validate_index_save_layout(isl, layout->index.nonce); + if (oldest == NULL || save_time < oldest_time) { + oldest = isl; + oldest_time = save_time; + } + } + + return oldest; +} + +static void instantiate_index_save_layout(struct index_save_layout *isl, + struct super_block_data *super, + u64 volume_nonce, unsigned int zone_count) +{ + unsigned int z; + u64 next_block; + u64 free_blocks; + u64 volume_index_blocks; + + isl->zone_count = zone_count; + memset(&isl->save_data, 0, sizeof(isl->save_data)); + isl->save_data.timestamp = ktime_to_ms(current_time_ns(CLOCK_REALTIME)); + isl->save_data.version = 1; + isl->save_data.nonce = generate_index_save_nonce(volume_nonce, isl); + + next_block = isl->index_save.start_block; + isl->header = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_HEADER, + .instance = RL_SOLE_INSTANCE, + }; + + isl->index_page_map = (struct layout_region) { + .start_block = next_block, + .block_count = super->page_map_blocks, + .kind = RL_KIND_INDEX_PAGE_MAP, + .instance = RL_SOLE_INSTANCE, + }; + next_block += super->page_map_blocks; + + free_blocks = (isl->index_save.block_count - 1 - + super->page_map_blocks - + super->open_chapter_blocks); + volume_index_blocks = free_blocks / isl->zone_count; + for (z = 0; z < isl->zone_count; z++) { + isl->volume_index_zones[z] = (struct layout_region) { + .start_block = next_block, + .block_count = volume_index_blocks, + .kind = RL_KIND_VOLUME_INDEX, + .instance = z, + }; + + next_block += volume_index_blocks; + free_blocks -= volume_index_blocks; + } + + isl->open_chapter = (struct layout_region) { + .start_block = next_block, + .block_count = super->open_chapter_blocks, + .kind = RL_KIND_OPEN_CHAPTER, + .instance = RL_SOLE_INSTANCE, + }; + + next_block += super->open_chapter_blocks; + + isl->free_space = (struct layout_region) { + .start_block = next_block, + .block_count = free_blocks, + .kind = RL_KIND_EMPTY, + .instance = RL_SOLE_INSTANCE, + }; +} + +static int setup_uds_index_save_slot(struct index_layout *layout, + unsigned int zone_count, + struct index_save_layout **isl_ptr) +{ + int result; + struct index_save_layout *isl; + + isl = select_oldest_index_save_layout(layout); + result = invalidate_old_save(layout, isl); + if (result != UDS_SUCCESS) + return result; + + instantiate_index_save_layout(isl, &layout->super, layout->index.nonce, + zone_count); + + *isl_ptr = isl; + return UDS_SUCCESS; +} + +static void cancel_uds_index_save(struct index_save_layout *isl) +{ + memset(&isl->save_data, 0, sizeof(isl->save_data)); + memset(&isl->state_data, 0, sizeof(isl->state_data)); + isl->zone_count = 0; +} + +int uds_save_index_state(struct index_layout *layout, struct uds_index *index) +{ + int result; + unsigned int zone; + struct index_save_layout *isl; + struct buffered_writer *writers[MAX_ZONES]; + + result = setup_uds_index_save_slot(layout, index->zone_count, &isl); + if (result != UDS_SUCCESS) + return result; + + isl->state_data = (struct index_state_data301) { + .newest_chapter = index->newest_virtual_chapter, + .oldest_chapter = index->oldest_virtual_chapter, + .last_save = index->last_save, + }; + + result = open_region_writer(layout, &isl->open_chapter, &writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + result = uds_save_open_chapter(index, writers[0]); + uds_free_buffered_writer(writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + for (zone = 0; zone < index->zone_count; zone++) { + result = open_region_writer(layout, &isl->volume_index_zones[zone], + &writers[zone]); + if (result != UDS_SUCCESS) { + for (; zone > 0; zone--) + uds_free_buffered_writer(writers[zone - 1]); + + cancel_uds_index_save(isl); + return result; + } + } + + result = uds_save_volume_index(index->volume_index, writers, index->zone_count); + for (zone = 0; zone < index->zone_count; zone++) + uds_free_buffered_writer(writers[zone]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + result = open_region_writer(layout, &isl->index_page_map, &writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + result = uds_write_index_page_map(index->volume->index_page_map, writers[0]); + uds_free_buffered_writer(writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + return write_index_save_layout(layout, isl); +} + +static int __must_check load_region_table(struct buffered_reader *reader, + struct region_table **table_ptr) +{ + int result; + unsigned int i; + struct region_header header; + struct region_table *table; + u8 buffer[sizeof(struct region_header)]; + size_t offset = 0; + + result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "cannot read region table header"); + + decode_u64_le(buffer, &offset, &header.magic); + decode_u64_le(buffer, &offset, &header.region_blocks); + decode_u16_le(buffer, &offset, &header.type); + decode_u16_le(buffer, &offset, &header.version); + decode_u16_le(buffer, &offset, &header.region_count); + decode_u16_le(buffer, &offset, &header.payload); + + if (header.magic != REGION_MAGIC) + return UDS_NO_INDEX; + + if (header.version != 1) { + return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "unknown region table version %hu", + header.version); + } + + result = uds_allocate_extended(struct region_table, header.region_count, + struct layout_region, + "single file layout region table", &table); + if (result != UDS_SUCCESS) + return result; + + table->header = header; + for (i = 0; i < header.region_count; i++) { + u8 region_buffer[sizeof(struct layout_region)]; + + offset = 0; + result = uds_read_from_buffered_reader(reader, region_buffer, + sizeof(region_buffer)); + if (result != UDS_SUCCESS) { + uds_free(table); + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "cannot read region table layouts"); + } + + decode_u64_le(region_buffer, &offset, &table->regions[i].start_block); + decode_u64_le(region_buffer, &offset, &table->regions[i].block_count); + offset += sizeof(u32); + decode_u16_le(region_buffer, &offset, &table->regions[i].kind); + decode_u16_le(region_buffer, &offset, &table->regions[i].instance); + } + + *table_ptr = table; + return UDS_SUCCESS; +} + +static int __must_check read_super_block_data(struct buffered_reader *reader, + struct index_layout *layout, + size_t saved_size) +{ + int result; + struct super_block_data *super = &layout->super; + u8 *buffer; + size_t offset = 0; + + result = uds_allocate(saved_size, u8, "super block data", &buffer); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, buffer, saved_size); + if (result != UDS_SUCCESS) { + uds_free(buffer); + return uds_log_error_strerror(result, "cannot read region table header"); + } + + memcpy(&super->magic_label, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + memcpy(&super->nonce_info, buffer + offset, NONCE_INFO_SIZE); + offset += NONCE_INFO_SIZE; + decode_u64_le(buffer, &offset, &super->nonce); + decode_u32_le(buffer, &offset, &super->version); + decode_u32_le(buffer, &offset, &super->block_size); + decode_u16_le(buffer, &offset, &super->index_count); + decode_u16_le(buffer, &offset, &super->max_saves); + offset += sizeof(u32); + decode_u64_le(buffer, &offset, &super->open_chapter_blocks); + decode_u64_le(buffer, &offset, &super->page_map_blocks); + + if (is_converted_super_block(super)) { + decode_u64_le(buffer, &offset, &super->volume_offset); + decode_u64_le(buffer, &offset, &super->start_offset); + } else { + super->volume_offset = 0; + super->start_offset = 0; + } + + uds_free(buffer); + + if (memcmp(super->magic_label, LAYOUT_MAGIC, MAGIC_SIZE) != 0) + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "unknown superblock magic label"); + + if ((super->version < SUPER_VERSION_MINIMUM) || + (super->version == 4) || + (super->version == 5) || + (super->version == 6) || + (super->version > SUPER_VERSION_MAXIMUM)) { + return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "unknown superblock version number %u", + super->version); + } + + if (super->volume_offset < super->start_offset) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "inconsistent offsets (start %llu, volume %llu)", + (unsigned long long) super->start_offset, + (unsigned long long) super->volume_offset); + } + + /* Sub-indexes are no longer used but the layout retains this field. */ + if (super->index_count != 1) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "invalid subindex count %u", + super->index_count); + } + + if (generate_primary_nonce(super->nonce_info, sizeof(super->nonce_info)) != super->nonce) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "inconsistent superblock nonce"); + } + + return UDS_SUCCESS; +} + +static int __must_check verify_region(struct layout_region *lr, u64 start_block, + enum region_kind kind, unsigned int instance) +{ + if (lr->start_block != start_block) + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "incorrect layout region offset"); + + if (lr->kind != kind) + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "incorrect layout region kind"); + + if (lr->instance != instance) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "incorrect layout region instance"); + } + + return UDS_SUCCESS; +} + +static int __must_check verify_sub_index(struct index_layout *layout, u64 start_block, + struct region_table *table) +{ + int result; + unsigned int i; + struct sub_index_layout *sil = &layout->index; + u64 next_block = start_block; + + sil->sub_index = table->regions[2]; + result = verify_region(&sil->sub_index, next_block, RL_KIND_INDEX, 0); + if (result != UDS_SUCCESS) + return result; + + define_sub_index_nonce(layout); + + sil->volume = table->regions[3]; + result = verify_region(&sil->volume, next_block, RL_KIND_VOLUME, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += sil->volume.block_count + layout->super.volume_offset; + + for (i = 0; i < layout->super.max_saves; i++) { + sil->saves[i].index_save = table->regions[i + 4]; + result = verify_region(&sil->saves[i].index_save, next_block, + RL_KIND_SAVE, i); + if (result != UDS_SUCCESS) + return result; + + next_block += sil->saves[i].index_save.block_count; + } + + next_block -= layout->super.volume_offset; + if (next_block != start_block + sil->sub_index.block_count) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "sub index region does not span all saves"); + } + + return UDS_SUCCESS; +} + +static int __must_check reconstitute_layout(struct index_layout *layout, + struct region_table *table, u64 first_block) +{ + int result; + u64 next_block = first_block; + + result = uds_allocate(layout->super.max_saves, struct index_save_layout, + __func__, &layout->index.saves); + if (result != UDS_SUCCESS) + return result; + + layout->total_blocks = table->header.region_blocks; + + layout->header = table->regions[0]; + result = verify_region(&layout->header, next_block++, RL_KIND_HEADER, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + layout->config = table->regions[1]; + result = verify_region(&layout->config, next_block++, RL_KIND_CONFIG, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + result = verify_sub_index(layout, next_block, table); + if (result != UDS_SUCCESS) + return result; + + next_block += layout->index.sub_index.block_count; + + layout->seal = table->regions[table->header.region_count - 1]; + result = verify_region(&layout->seal, next_block + layout->super.volume_offset, + RL_KIND_SEAL, RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + if (++next_block != (first_block + layout->total_blocks)) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "layout table does not span total blocks"); + } + + return UDS_SUCCESS; +} + +static int __must_check load_super_block(struct index_layout *layout, size_t block_size, + u64 first_block, struct buffered_reader *reader) +{ + int result; + struct region_table *table = NULL; + struct super_block_data *super = &layout->super; + + result = load_region_table(reader, &table); + if (result != UDS_SUCCESS) + return result; + + if (table->header.type != RH_TYPE_SUPER) { + uds_free(table); + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "not a superblock region table"); + } + + result = read_super_block_data(reader, layout, table->header.payload); + if (result != UDS_SUCCESS) { + uds_free(table); + return uds_log_error_strerror(result, "unknown superblock format"); + } + + if (super->block_size != block_size) { + uds_free(table); + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "superblock saved block_size %u differs from supplied block_size %zu", + super->block_size, block_size); + } + + first_block -= (super->volume_offset - super->start_offset); + result = reconstitute_layout(layout, table, first_block); + uds_free(table); + return result; +} + +static int __must_check read_index_save_data(struct buffered_reader *reader, + struct index_save_layout *isl, + size_t saved_size) +{ + int result; + struct index_state_version file_version; + u8 buffer[sizeof(struct index_save_data) + sizeof(struct index_state_data301)]; + size_t offset = 0; + + if (saved_size != sizeof(buffer)) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save data size %zu", + saved_size); + } + + result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "cannot read index save data"); + + decode_u64_le(buffer, &offset, &isl->save_data.timestamp); + decode_u64_le(buffer, &offset, &isl->save_data.nonce); + decode_u32_le(buffer, &offset, &isl->save_data.version); + offset += sizeof(u32); + + if (isl->save_data.version > 1) { + return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "unknown index save version number %u", + isl->save_data.version); + } + + decode_s32_le(buffer, &offset, &file_version.signature); + decode_s32_le(buffer, &offset, &file_version.version_id); + + if ((file_version.signature != INDEX_STATE_VERSION_301.signature) || + (file_version.version_id != INDEX_STATE_VERSION_301.version_id)) { + return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "index state version %d,%d is unsupported", + file_version.signature, + file_version.version_id); + } + + decode_u64_le(buffer, &offset, &isl->state_data.newest_chapter); + decode_u64_le(buffer, &offset, &isl->state_data.oldest_chapter); + decode_u64_le(buffer, &offset, &isl->state_data.last_save); + /* Skip past some historical fields that are now unused */ + offset += sizeof(u32) + sizeof(u32); + return UDS_SUCCESS; +} + +static int __must_check reconstruct_index_save(struct index_save_layout *isl, + struct region_table *table) +{ + int result; + unsigned int z; + struct layout_region *last_region; + u64 next_block = isl->index_save.start_block; + u64 last_block = next_block + isl->index_save.block_count; + + isl->zone_count = table->header.region_count - 3; + + last_region = &table->regions[table->header.region_count - 1]; + if (last_region->kind == RL_KIND_EMPTY) { + isl->free_space = *last_region; + isl->zone_count--; + } else { + isl->free_space = (struct layout_region) { + .start_block = last_block, + .block_count = 0, + .kind = RL_KIND_EMPTY, + .instance = RL_SOLE_INSTANCE, + }; + } + + isl->header = table->regions[0]; + result = verify_region(&isl->header, next_block++, RL_KIND_HEADER, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + isl->index_page_map = table->regions[1]; + result = verify_region(&isl->index_page_map, next_block, RL_KIND_INDEX_PAGE_MAP, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->index_page_map.block_count; + + for (z = 0; z < isl->zone_count; z++) { + isl->volume_index_zones[z] = table->regions[z + 2]; + result = verify_region(&isl->volume_index_zones[z], next_block, + RL_KIND_VOLUME_INDEX, z); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->volume_index_zones[z].block_count; + } + + isl->open_chapter = table->regions[isl->zone_count + 2]; + result = verify_region(&isl->open_chapter, next_block, RL_KIND_OPEN_CHAPTER, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->open_chapter.block_count; + + result = verify_region(&isl->free_space, next_block, RL_KIND_EMPTY, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->free_space.block_count; + if (next_block != last_block) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "index save layout table incomplete"); + } + + return UDS_SUCCESS; +} + +static int __must_check load_index_save(struct index_save_layout *isl, + struct buffered_reader *reader, + unsigned int instance) +{ + int result; + struct region_table *table = NULL; + + result = load_region_table(reader, &table); + if (result != UDS_SUCCESS) { + return uds_log_error_strerror(result, "cannot read index save %u header", + instance); + } + + if (table->header.region_blocks != isl->index_save.block_count) { + u64 region_blocks = table->header.region_blocks; + + uds_free(table); + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save %u region block count %llu", + instance, + (unsigned long long) region_blocks); + } + + if (table->header.type == RH_TYPE_UNSAVED) { + uds_free(table); + reset_index_save_layout(isl, 0); + return UDS_SUCCESS; + } + + + if (table->header.type != RH_TYPE_SAVE) { + uds_free(table); + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save %u header type %u", + instance, table->header.type); + } + + result = read_index_save_data(reader, isl, table->header.payload); + if (result != UDS_SUCCESS) { + uds_free(table); + return uds_log_error_strerror(result, + "unknown index save %u data format", + instance); + } + + result = reconstruct_index_save(isl, table); + uds_free(table); + if (result != UDS_SUCCESS) { + return uds_log_error_strerror(result, "cannot reconstruct index save %u", + instance); + } + + return UDS_SUCCESS; +} + +static int __must_check load_sub_index_regions(struct index_layout *layout) +{ + int result; + unsigned int j; + struct index_save_layout *isl; + struct buffered_reader *reader; + + for (j = 0; j < layout->super.max_saves; j++) { + isl = &layout->index.saves[j]; + result = open_region_reader(layout, &isl->index_save, &reader); + + if (result != UDS_SUCCESS) { + uds_log_error_strerror(result, + "cannot get reader for index 0 save %u", + j); + return result; + } + + result = load_index_save(isl, reader, j); + uds_free_buffered_reader(reader); + if (result != UDS_SUCCESS) { + /* Another save slot might be valid. */ + reset_index_save_layout(isl, 0); + continue; + } + } + + return UDS_SUCCESS; +} + +static int __must_check verify_uds_index_config(struct index_layout *layout, + struct configuration *config) +{ + int result; + struct buffered_reader *reader = NULL; + u64 offset; + + offset = layout->super.volume_offset - layout->super.start_offset; + result = open_layout_reader(layout, &layout->config, offset, &reader); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "failed to open config reader"); + + result = uds_validate_config_contents(reader, config); + if (result != UDS_SUCCESS) { + uds_free_buffered_reader(reader); + return uds_log_error_strerror(result, "failed to read config region"); + } + + uds_free_buffered_reader(reader); + return UDS_SUCCESS; +} + +static int load_index_layout(struct index_layout *layout, struct configuration *config) +{ + int result; + struct buffered_reader *reader; + + result = uds_make_buffered_reader(layout->factory, + layout->offset / UDS_BLOCK_SIZE, 1, &reader); + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, "unable to read superblock"); + + result = load_super_block(layout, UDS_BLOCK_SIZE, + layout->offset / UDS_BLOCK_SIZE, reader); + uds_free_buffered_reader(reader); + if (result != UDS_SUCCESS) + return result; + + result = verify_uds_index_config(layout, config); + if (result != UDS_SUCCESS) + return result; + + return load_sub_index_regions(layout); +} + +static int create_layout_factory(struct index_layout *layout, + const struct configuration *config) +{ + int result; + size_t writable_size; + struct io_factory *factory = NULL; + + result = uds_make_io_factory(config->bdev, &factory); + if (result != UDS_SUCCESS) + return result; + + writable_size = uds_get_writable_size(factory) & -UDS_BLOCK_SIZE; + if (writable_size < config->size + config->offset) { + uds_put_io_factory(factory); + uds_log_error("index storage (%zu) is smaller than the requested size %zu", + writable_size, config->size + config->offset); + return -ENOSPC; + } + + layout->factory = factory; + layout->factory_size = (config->size > 0) ? config->size : writable_size; + layout->offset = config->offset; + return UDS_SUCCESS; +} + +int uds_make_index_layout(struct configuration *config, bool new_layout, + struct index_layout **layout_ptr) +{ + int result; + struct index_layout *layout = NULL; + struct save_layout_sizes sizes; + + result = compute_sizes(config, &sizes); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(1, struct index_layout, __func__, &layout); + if (result != UDS_SUCCESS) + return result; + + result = create_layout_factory(layout, config); + if (result != UDS_SUCCESS) { + uds_free_index_layout(layout); + return result; + } + + if (layout->factory_size < sizes.total_size) { + uds_log_error("index storage (%zu) is smaller than the required size %llu", + layout->factory_size, + (unsigned long long) sizes.total_size); + uds_free_index_layout(layout); + return -ENOSPC; + } + + if (new_layout) + result = create_index_layout(layout, config); + else + result = load_index_layout(layout, config); + if (result != UDS_SUCCESS) { + uds_free_index_layout(layout); + return result; + } + + *layout_ptr = layout; + return UDS_SUCCESS; +} + +void uds_free_index_layout(struct index_layout *layout) +{ + if (layout == NULL) + return; + + uds_free(layout->index.saves); + if (layout->factory != NULL) + uds_put_io_factory(layout->factory); + + uds_free(layout); +} + +int uds_replace_index_layout_storage(struct index_layout *layout, + struct block_device *bdev) +{ + return uds_replace_storage(layout->factory, bdev); +} + +/* Obtain a dm_bufio_client for the volume region. */ +int uds_open_volume_bufio(struct index_layout *layout, size_t block_size, + unsigned int reserved_buffers, + struct dm_bufio_client **client_ptr) +{ + off_t offset = (layout->index.volume.start_block + + layout->super.volume_offset - + layout->super.start_offset); + + return uds_make_bufio(layout->factory, offset, block_size, reserved_buffers, + client_ptr); +} + +u64 uds_get_volume_nonce(struct index_layout *layout) +{ + return layout->index.nonce; +} diff --git a/drivers/md/dm-vdo/index-layout.h b/drivers/md/dm-vdo/index-layout.h new file mode 100644 index 000000000000..ba5d8b004ebe --- /dev/null +++ b/drivers/md/dm-vdo/index-layout.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_LAYOUT_H +#define UDS_INDEX_LAYOUT_H + +#include "config.h" +#include "io-factory.h" +#include "uds.h" + +/* + * The index layout describes the format of the index on the underlying storage, and is responsible + * for creating those structures when the index is first created. It also validates the index data + * when loading a saved index, and updates it when saving the index. + */ + +struct index_layout; + +int __must_check uds_make_index_layout(struct configuration *config, bool new_layout, + struct index_layout **layout_ptr); + +void uds_free_index_layout(struct index_layout *layout); + +int __must_check uds_replace_index_layout_storage(struct index_layout *layout, + struct block_device *bdev); + +int __must_check uds_load_index_state(struct index_layout *layout, + struct uds_index *index); + +int __must_check uds_save_index_state(struct index_layout *layout, + struct uds_index *index); + +int __must_check uds_discard_open_chapter(struct index_layout *layout); + +u64 __must_check uds_get_volume_nonce(struct index_layout *layout); + +int __must_check uds_open_volume_bufio(struct index_layout *layout, size_t block_size, + unsigned int reserved_buffers, + struct dm_bufio_client **client_ptr); + +#endif /* UDS_INDEX_LAYOUT_H */ diff --git a/drivers/md/dm-vdo/io-factory.c b/drivers/md/dm-vdo/io-factory.c new file mode 100644 index 000000000000..02242df94e37 --- /dev/null +++ b/drivers/md/dm-vdo/io-factory.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "io-factory.h" + +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" + +/* + * The I/O factory object manages access to index storage, which is a contiguous range of blocks on + * a block device. + * + * The factory holds the open device and is responsible for closing it. The factory has methods to + * make helper structures that can be used to access sections of the index. + */ +struct io_factory { + struct block_device *bdev; + atomic_t ref_count; +}; + +/* The buffered reader allows efficient I/O by reading page-sized segments into a buffer. */ +struct buffered_reader { + struct io_factory *factory; + struct dm_bufio_client *client; + struct dm_buffer *buffer; + sector_t limit; + sector_t block_number; + u8 *start; + u8 *end; +}; + +enum { MAX_READ_AHEAD_BLOCKS = 4 }; + +/* + * The buffered writer allows efficient I/O by buffering writes and committing page-sized segments + * to storage. + */ +struct buffered_writer { + struct io_factory *factory; + struct dm_bufio_client *client; + struct dm_buffer *buffer; + sector_t limit; + sector_t block_number; + u8 *start; + u8 *end; + int error; +}; + +static void uds_get_io_factory(struct io_factory *factory) +{ + atomic_inc(&factory->ref_count); +} + +int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_ptr) +{ + int result; + struct io_factory *factory; + + result = uds_allocate(1, struct io_factory, __func__, &factory); + if (result != UDS_SUCCESS) + return result; + + factory->bdev = bdev; + atomic_set_release(&factory->ref_count, 1); + + *factory_ptr = factory; + return UDS_SUCCESS; +} + +int uds_replace_storage(struct io_factory *factory, struct block_device *bdev) +{ + factory->bdev = bdev; + return UDS_SUCCESS; +} + +/* Free an I/O factory once all references have been released. */ +void uds_put_io_factory(struct io_factory *factory) +{ + if (atomic_add_return(-1, &factory->ref_count) <= 0) + uds_free(factory); +} + +size_t uds_get_writable_size(struct io_factory *factory) +{ + return i_size_read(factory->bdev->bd_inode); +} + +/* Create a struct dm_bufio_client for an index region starting at offset. */ +int uds_make_bufio(struct io_factory *factory, off_t block_offset, size_t block_size, + unsigned int reserved_buffers, struct dm_bufio_client **client_ptr) +{ + struct dm_bufio_client *client; + + client = dm_bufio_client_create(factory->bdev, block_size, reserved_buffers, 0, + NULL, NULL, 0); + if (IS_ERR(client)) + return -PTR_ERR(client); + + dm_bufio_set_sector_offset(client, block_offset * SECTORS_PER_BLOCK); + *client_ptr = client; + return UDS_SUCCESS; +} + +static void read_ahead(struct buffered_reader *reader, sector_t block_number) +{ + if (block_number < reader->limit) { + sector_t read_ahead = min((sector_t) MAX_READ_AHEAD_BLOCKS, + reader->limit - block_number); + + dm_bufio_prefetch(reader->client, block_number, read_ahead); + } +} + +void uds_free_buffered_reader(struct buffered_reader *reader) +{ + if (reader == NULL) + return; + + if (reader->buffer != NULL) + dm_bufio_release(reader->buffer); + + dm_bufio_client_destroy(reader->client); + uds_put_io_factory(reader->factory); + uds_free(reader); +} + +/* Create a buffered reader for an index region starting at offset. */ +int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block_count, + struct buffered_reader **reader_ptr) +{ + int result; + struct dm_bufio_client *client = NULL; + struct buffered_reader *reader = NULL; + + result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(1, struct buffered_reader, "buffered reader", &reader); + if (result != UDS_SUCCESS) { + dm_bufio_client_destroy(client); + return result; + } + + *reader = (struct buffered_reader) { + .factory = factory, + .client = client, + .buffer = NULL, + .limit = block_count, + .block_number = 0, + .start = NULL, + .end = NULL, + }; + + read_ahead(reader, 0); + uds_get_io_factory(factory); + *reader_ptr = reader; + return UDS_SUCCESS; +} + +static int position_reader(struct buffered_reader *reader, sector_t block_number, + off_t offset) +{ + struct dm_buffer *buffer = NULL; + void *data; + + if ((reader->end == NULL) || (block_number != reader->block_number)) { + if (block_number >= reader->limit) + return UDS_OUT_OF_RANGE; + + if (reader->buffer != NULL) + dm_bufio_release(uds_forget(reader->buffer)); + + data = dm_bufio_read(reader->client, block_number, &buffer); + if (IS_ERR(data)) + return -PTR_ERR(data); + + reader->buffer = buffer; + reader->start = data; + if (block_number == reader->block_number + 1) + read_ahead(reader, block_number + 1); + } + + reader->block_number = block_number; + reader->end = reader->start + offset; + return UDS_SUCCESS; +} + +static size_t bytes_remaining_in_read_buffer(struct buffered_reader *reader) +{ + return (reader->end == NULL) ? 0 : reader->start + UDS_BLOCK_SIZE - reader->end; +} + +static int reset_reader(struct buffered_reader *reader) +{ + sector_t block_number; + + if (bytes_remaining_in_read_buffer(reader) > 0) + return UDS_SUCCESS; + + block_number = reader->block_number; + if (reader->end != NULL) + block_number++; + + return position_reader(reader, block_number, 0); +} + +int uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data, + size_t length) +{ + int result = UDS_SUCCESS; + size_t chunk_size; + + while (length > 0) { + result = reset_reader(reader); + if (result != UDS_SUCCESS) + return result; + + chunk_size = min(length, bytes_remaining_in_read_buffer(reader)); + memcpy(data, reader->end, chunk_size); + length -= chunk_size; + data += chunk_size; + reader->end += chunk_size; + } + + return UDS_SUCCESS; +} + +/* + * Verify that the next data on the reader matches the required value. If the value matches, the + * matching contents are consumed. If the value does not match, the reader state is unchanged. + */ +int uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value, + size_t length) +{ + int result = UDS_SUCCESS; + size_t chunk_size; + sector_t start_block_number = reader->block_number; + int start_offset = reader->end - reader->start; + + while (length > 0) { + result = reset_reader(reader); + if (result != UDS_SUCCESS) { + result = UDS_CORRUPT_DATA; + break; + } + + chunk_size = min(length, bytes_remaining_in_read_buffer(reader)); + if (memcmp(value, reader->end, chunk_size) != 0) { + result = UDS_CORRUPT_DATA; + break; + } + + length -= chunk_size; + value += chunk_size; + reader->end += chunk_size; + } + + if (result != UDS_SUCCESS) + position_reader(reader, start_block_number, start_offset); + + return result; +} + +/* Create a buffered writer for an index region starting at offset. */ +int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block_count, + struct buffered_writer **writer_ptr) +{ + int result; + struct dm_bufio_client *client = NULL; + struct buffered_writer *writer; + + result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(1, struct buffered_writer, "buffered writer", &writer); + if (result != UDS_SUCCESS) { + dm_bufio_client_destroy(client); + return result; + } + + *writer = (struct buffered_writer) { + .factory = factory, + .client = client, + .buffer = NULL, + .limit = block_count, + .start = NULL, + .end = NULL, + .block_number = 0, + .error = UDS_SUCCESS, + }; + + uds_get_io_factory(factory); + *writer_ptr = writer; + return UDS_SUCCESS; +} + +static size_t get_remaining_write_space(struct buffered_writer *writer) +{ + return writer->start + UDS_BLOCK_SIZE - writer->end; +} + +static int __must_check prepare_next_buffer(struct buffered_writer *writer) +{ + struct dm_buffer *buffer = NULL; + void *data; + + if (writer->block_number >= writer->limit) { + writer->error = UDS_OUT_OF_RANGE; + return UDS_OUT_OF_RANGE; + } + + data = dm_bufio_new(writer->client, writer->block_number, &buffer); + if (IS_ERR(data)) { + writer->error = -PTR_ERR(data); + return writer->error; + } + + writer->buffer = buffer; + writer->start = data; + writer->end = data; + return UDS_SUCCESS; +} + +static int flush_previous_buffer(struct buffered_writer *writer) +{ + size_t available; + + if (writer->buffer == NULL) + return writer->error; + + if (writer->error == UDS_SUCCESS) { + available = get_remaining_write_space(writer); + + if (available > 0) + memset(writer->end, 0, available); + + dm_bufio_mark_buffer_dirty(writer->buffer); + } + + dm_bufio_release(writer->buffer); + writer->buffer = NULL; + writer->start = NULL; + writer->end = NULL; + writer->block_number++; + return writer->error; +} + +void uds_free_buffered_writer(struct buffered_writer *writer) +{ + int result; + + if (writer == NULL) + return; + + flush_previous_buffer(writer); + result = -dm_bufio_write_dirty_buffers(writer->client); + if (result != UDS_SUCCESS) + uds_log_warning_strerror(result, "%s: failed to sync storage", __func__); + + dm_bufio_client_destroy(writer->client); + uds_put_io_factory(writer->factory); + uds_free(writer); +} + +/* + * Append data to the buffer, writing as needed. If no data is provided, zeros are written instead. + * If a write error occurs, it is recorded and returned on every subsequent write attempt. + */ +int uds_write_to_buffered_writer(struct buffered_writer *writer, const u8 *data, + size_t length) +{ + int result = writer->error; + size_t chunk_size; + + while ((length > 0) && (result == UDS_SUCCESS)) { + if (writer->buffer == NULL) { + result = prepare_next_buffer(writer); + continue; + } + + chunk_size = min(length, get_remaining_write_space(writer)); + if (data == NULL) { + memset(writer->end, 0, chunk_size); + } else { + memcpy(writer->end, data, chunk_size); + data += chunk_size; + } + + length -= chunk_size; + writer->end += chunk_size; + + if (get_remaining_write_space(writer) == 0) + result = uds_flush_buffered_writer(writer); + } + + return result; +} + +int uds_flush_buffered_writer(struct buffered_writer *writer) +{ + if (writer->error != UDS_SUCCESS) + return writer->error; + + return flush_previous_buffer(writer); +} diff --git a/drivers/md/dm-vdo/io-factory.h b/drivers/md/dm-vdo/io-factory.h new file mode 100644 index 000000000000..7fb5a0616a79 --- /dev/null +++ b/drivers/md/dm-vdo/io-factory.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_IO_FACTORY_H +#define UDS_IO_FACTORY_H + +#include + +/* + * The I/O factory manages all low-level I/O operations to the underlying storage device. Its main + * clients are the index layout and the volume. The buffered reader and buffered writer interfaces + * are helpers for accessing data in a contiguous range of storage blocks. + */ + +struct buffered_reader; +struct buffered_writer; + +struct io_factory; + +enum { + UDS_BLOCK_SIZE = 4096, + SECTORS_PER_BLOCK = UDS_BLOCK_SIZE >> SECTOR_SHIFT, +}; + +int __must_check uds_make_io_factory(struct block_device *bdev, + struct io_factory **factory_ptr); + +int __must_check uds_replace_storage(struct io_factory *factory, + struct block_device *bdev); + +void uds_put_io_factory(struct io_factory *factory); + +size_t __must_check uds_get_writable_size(struct io_factory *factory); + +int __must_check uds_make_bufio(struct io_factory *factory, off_t block_offset, + size_t block_size, unsigned int reserved_buffers, + struct dm_bufio_client **client_ptr); + +int __must_check uds_make_buffered_reader(struct io_factory *factory, off_t offset, + u64 block_count, + struct buffered_reader **reader_ptr); + +void uds_free_buffered_reader(struct buffered_reader *reader); + +int __must_check uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data, + size_t length); + +int __must_check uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value, + size_t length); + +int __must_check uds_make_buffered_writer(struct io_factory *factory, off_t offset, + u64 block_count, + struct buffered_writer **writer_ptr); + +void uds_free_buffered_writer(struct buffered_writer *buffer); + +int __must_check uds_write_to_buffered_writer(struct buffered_writer *writer, + const u8 *data, size_t length); + +int __must_check uds_flush_buffered_writer(struct buffered_writer *writer); + +#endif /* UDS_IO_FACTORY_H */ diff --git a/drivers/md/dm-vdo/numeric.h b/drivers/md/dm-vdo/numeric.h new file mode 100644 index 000000000000..dc8c400b21d2 --- /dev/null +++ b/drivers/md/dm-vdo/numeric.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_NUMERIC_H +#define UDS_NUMERIC_H + +#include +#include +#include + +/* + * These utilities encode or decode a number from an offset in a larger data buffer and then + * advance the offset pointer to the next field in the buffer. + */ + +static inline void decode_s64_le(const u8 *buffer, size_t *offset, s64 *decoded) +{ + *decoded = get_unaligned_le64(buffer + *offset); + *offset += sizeof(s64); +} + +static inline void encode_s64_le(u8 *data, size_t *offset, s64 to_encode) +{ + put_unaligned_le64(to_encode, data + *offset); + *offset += sizeof(s64); +} + +static inline void decode_u64_le(const u8 *buffer, size_t *offset, u64 *decoded) +{ + *decoded = get_unaligned_le64(buffer + *offset); + *offset += sizeof(u64); +} + +static inline void encode_u64_le(u8 *data, size_t *offset, u64 to_encode) +{ + put_unaligned_le64(to_encode, data + *offset); + *offset += sizeof(u64); +} + +static inline void decode_s32_le(const u8 *buffer, size_t *offset, s32 *decoded) +{ + *decoded = get_unaligned_le32(buffer + *offset); + *offset += sizeof(s32); +} + +static inline void encode_s32_le(u8 *data, size_t *offset, s32 to_encode) +{ + put_unaligned_le32(to_encode, data + *offset); + *offset += sizeof(s32); +} + +static inline void decode_u32_le(const u8 *buffer, size_t *offset, u32 *decoded) +{ + *decoded = get_unaligned_le32(buffer + *offset); + *offset += sizeof(u32); +} + +static inline void encode_u32_le(u8 *data, size_t *offset, u32 to_encode) +{ + put_unaligned_le32(to_encode, data + *offset); + *offset += sizeof(u32); +} + +static inline void decode_u16_le(const u8 *buffer, size_t *offset, u16 *decoded) +{ + *decoded = get_unaligned_le16(buffer + *offset); + *offset += sizeof(u16); +} + +static inline void encode_u16_le(u8 *data, size_t *offset, u16 to_encode) +{ + put_unaligned_le16(to_encode, data + *offset); + *offset += sizeof(u16); +} + +#endif /* UDS_NUMERIC_H */ From 024512e7326a27311de5158d5396eb39bbe030bc Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:52:58 -0500 Subject: [PATCH 0559/1038] dm vdo: implement the delta index The delta index is a space and memory efficient alternative to a hashtable. Instead of storing the entire key for each entry, the entries are sorted by key and only the difference between adjacent keys (the delta) is stored. If the keys are evenly distributed, the size of the deltas follows an exponential distribution, and the deltas can use a Huffman code to take up even less space. This structure allows the index to use many fewer bytes per entry than a traditional hash table, but it is slightly more expensive to look up entries, because a request must read and sum every entry in a list of deltas in order to find a given record. The delta index reduces this lookup cost by splitting its key space into many sub-lists, each starting at a fixed key value, so that each individual list is short. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/delta-index.c | 1990 +++++++++++++++++++++++++++++++ drivers/md/dm-vdo/delta-index.h | 278 +++++ drivers/md/dm-vdo/hash-utils.h | 65 + 3 files changed, 2333 insertions(+) create mode 100644 drivers/md/dm-vdo/delta-index.c create mode 100644 drivers/md/dm-vdo/delta-index.h create mode 100644 drivers/md/dm-vdo/hash-utils.h diff --git a/drivers/md/dm-vdo/delta-index.c b/drivers/md/dm-vdo/delta-index.c new file mode 100644 index 000000000000..0322552cdbd8 --- /dev/null +++ b/drivers/md/dm-vdo/delta-index.c @@ -0,0 +1,1990 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ +#include "delta-index.h" + +#include +#include +#include +#include +#include + +#include "config.h" +#include "cpu.h" +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" +#include "time-utils.h" +#include "uds.h" + +/* + * The entries in a delta index could be stored in a single delta list, but to reduce search times + * and update costs it uses multiple delta lists. These lists are stored in a single chunk of + * memory managed by the delta_zone structure. The delta_zone can move the data around within its + * memory, so the location of each delta list is recorded as a bit offset into the memory. Because + * the volume index can contain over a million delta lists, we want to be efficient with the size + * of the delta list header information. This information is encoded into 16 bytes per list. The + * volume index delta list memory can easily exceed 4 gigabits, so a 64 bit value is needed to + * address the memory. The volume index delta lists average around 6 kilobits, so 16 bits are + * sufficient to store the size of a delta list. + * + * Each delta list is stored as a bit stream. Within the delta list encoding, bits and bytes are + * numbered in little endian order. Within a byte, bit 0 is the least significant bit (0x1), and + * bit 7 is the most significant bit (0x80). Within a bit stream, bit 7 is the most signficant bit + * of byte 0, and bit 8 is the least significant bit of byte 1. Within a byte array, a byte's + * number corresponds to its index in the array. + * + * A standard delta list entry is stored as a fixed length payload (the value) followed by a + * variable length key (the delta). A collision entry is used when two block names have the same + * delta list address. A collision entry always follows a standard entry for the hash with which it + * collides, and is encoded with DELTA == 0 with an additional 256 bits field at the end, + * containing the full block name. An entry with a delta of 0 at the beginning of a delta list + * indicates a normal entry. + * + * The delta in each entry is encoded with a variable-length Huffman code to minimize the memory + * used by small deltas. The Huffman code is specified by three parameters, which can be computed + * from the desired mean delta when the index is full. (See compute_coding constants() for + * details.) + * + * The bit field utilities used to read and write delta entries assume that it is possible to read + * some bytes beyond the end of the bit field, so a delta_zone memory allocation is guarded by two + * invalid delta lists to prevent reading outside the delta_zone memory. The valid delta lists are + * numbered 1 to N, and the guard lists are numbered 0 and N+1. The function to decode the bit + * stream include a step that skips over bits set to 0 until the first 1 bit is found. A corrupted + * delta list could cause this step to run off the end of the delta_zone memory, so as extra + * protection against this happening, the tail guard list is set to all ones. + * + * The delta_index supports two different forms. The mutable form is created by + * uds_initialize_delta_index(), and is used for the volume index and for open chapter indexes. The + * immutable form is created by uds_initialize_delta_index_page(), and is used for closed (and + * cached) chapter index pages. The immutable form does not allocate delta list headers or + * temporary offsets, and thus is somewhat more memory efficient. + */ + +/* + * This is the largest field size supported by get_field() and set_field(). Any field that is + * larger is not guaranteed to fit in a single byte-aligned u32. + */ +enum { + MAX_FIELD_BITS = (sizeof(u32) - 1) * BITS_PER_BYTE + 1, +}; + +/* + * This is the largest field size supported by get_big_field() and set_big_field(). Any field that + * is larger is not guaranteed to fit in a single byte-aligned u64. + */ +enum { + MAX_BIG_FIELD_BITS = (sizeof(u64) - 1) * BITS_PER_BYTE + 1, +}; + +/* + * This is the number of guard bytes needed at the end of the memory byte array when using the bit + * utilities. These utilities call get_big_field() and set_big_field(), which can access up to 7 + * bytes beyond the end of the desired field. The definition is written to make it clear how this + * value is derived. + */ +enum { + POST_FIELD_GUARD_BYTES = sizeof(u64) - 1, +}; + +/* The number of guard bits that are needed in the tail guard list */ +enum { + GUARD_BITS = POST_FIELD_GUARD_BYTES * BITS_PER_BYTE +}; + +/* + * The maximum size of a single delta list in bytes. We count guard bytes in this value because a + * buffer of this size can be used with move_bits(). + */ +enum { + DELTA_LIST_MAX_BYTE_COUNT = + ((U16_MAX + BITS_PER_BYTE) / BITS_PER_BYTE + POST_FIELD_GUARD_BYTES) +}; + +/* The number of extra bytes and bits needed to store a collision entry */ +enum { + COLLISION_BYTES = UDS_RECORD_NAME_SIZE, + COLLISION_BITS = COLLISION_BYTES * BITS_PER_BYTE +}; + +/* + * Immutable delta lists are packed into pages containing a header that encodes the delta list + * information into 19 bits per list (64KB bit offset). + */ + +enum { IMMUTABLE_HEADER_SIZE = 19 }; + +/* + * Constants and structures for the saved delta index. "DI" is for delta_index, and -##### is a + * number to increment when the format of the data changes. + */ + +enum { + MAGIC_SIZE = 8, +}; + +static const char DELTA_INDEX_MAGIC[] = "DI-00002"; + +struct delta_index_header { + char magic[MAGIC_SIZE]; + u32 zone_number; + u32 zone_count; + u32 first_list; + u32 list_count; + u64 record_count; + u64 collision_count; +}; + +/* + * Header data used for immutable delta index pages. This data is followed by the delta list offset + * table. + */ +struct delta_page_header { + /* Externally-defined nonce */ + u64 nonce; + /* The virtual chapter number */ + u64 virtual_chapter_number; + /* Index of the first delta list on the page */ + u16 first_list; + /* Number of delta lists on the page */ + u16 list_count; +} __packed; + +static inline u64 get_delta_list_byte_start(const struct delta_list *delta_list) +{ + return delta_list->start / BITS_PER_BYTE; +} + +static inline u16 get_delta_list_byte_size(const struct delta_list *delta_list) +{ + unsigned int bit_offset = delta_list->start % BITS_PER_BYTE; + + return BITS_TO_BYTES(bit_offset + delta_list->size); +} + +static void rebalance_delta_zone(const struct delta_zone *delta_zone, u32 first, + u32 last) +{ + struct delta_list *delta_list; + u64 new_start; + + if (first == last) { + /* Only one list is moving, and we know there is space. */ + delta_list = &delta_zone->delta_lists[first]; + new_start = delta_zone->new_offsets[first]; + if (delta_list->start != new_start) { + u64 source; + u64 destination; + + source = get_delta_list_byte_start(delta_list); + delta_list->start = new_start; + destination = get_delta_list_byte_start(delta_list); + memmove(delta_zone->memory + destination, + delta_zone->memory + source, + get_delta_list_byte_size(delta_list)); + } + } else { + /* + * There is more than one list. Divide the problem in half, and use recursive calls + * to process each half. Note that after this computation, first <= middle, and + * middle < last. + */ + u32 middle = (first + last) / 2; + + delta_list = &delta_zone->delta_lists[middle]; + new_start = delta_zone->new_offsets[middle]; + + /* + * The direction that our middle list is moving determines which half of the + * problem must be processed first. + */ + if (new_start > delta_list->start) { + rebalance_delta_zone(delta_zone, middle + 1, last); + rebalance_delta_zone(delta_zone, first, middle); + } else { + rebalance_delta_zone(delta_zone, first, middle); + rebalance_delta_zone(delta_zone, middle + 1, last); + } + } +} + +static inline size_t get_zone_memory_size(unsigned int zone_count, size_t memory_size) +{ + /* Round up so that each zone is a multiple of 64K in size. */ + enum { + ALLOC_BOUNDARY = 64 * 1024, + }; + + return (memory_size / zone_count + ALLOC_BOUNDARY - 1) & -ALLOC_BOUNDARY; +} + +void uds_reset_delta_index(const struct delta_index *delta_index) +{ + unsigned int z; + + /* + * Initialize all delta lists to be empty. We keep 2 extra delta list descriptors, one + * before the first real entry and one after so that we don't need to bounds check the + * array access when calculating preceding and following gap sizes. + */ + for (z = 0; z < delta_index->zone_count; z++) { + u64 list_bits; + u64 spacing; + u64 offset; + unsigned int i; + struct delta_zone *zone = &delta_index->delta_zones[z]; + struct delta_list *delta_lists = zone->delta_lists; + + /* Zeroing the delta list headers initializes the head guard list correctly. */ + memset(delta_lists, 0, + (zone->list_count + 2) * sizeof(struct delta_list)); + + /* Set all the bits in the end guard list. */ + list_bits = (u64) zone->size * BITS_PER_BYTE - GUARD_BITS; + delta_lists[zone->list_count + 1].start = list_bits; + delta_lists[zone->list_count + 1].size = GUARD_BITS; + memset(zone->memory + (list_bits / BITS_PER_BYTE), ~0, + POST_FIELD_GUARD_BYTES); + + /* Evenly space out the real delta lists by setting regular offsets. */ + spacing = list_bits / zone->list_count; + offset = spacing / 2; + for (i = 1; i <= zone->list_count; i++) { + delta_lists[i].start = offset; + offset += spacing; + } + + /* Update the statistics. */ + zone->discard_count += zone->record_count; + zone->record_count = 0; + zone->collision_count = 0; + } +} + +/* Compute the Huffman coding parameters for the given mean delta. The Huffman code is specified by + * three parameters: + * + * MINBITS The number of bits in the smallest code + * BASE The number of values coded using a code of length MINBITS + * INCR The number of values coded by using one additional bit + * + * These parameters are related by this equation: + * + * BASE + INCR == 1 << MINBITS + * + * The math for the Huffman code of an exponential distribution says that + * + * INCR = log(2) * MEAN_DELTA + * + * Then use the smallest MINBITS value so that + * + * (1 << MINBITS) > INCR + * + * And then + * + * BASE = (1 << MINBITS) - INCR + * + * Now the index can generate a code such that + * - The first BASE values code using MINBITS bits. + * - The next INCR values code using MINBITS+1 bits. + * - The next INCR values code using MINBITS+2 bits. + * - (and so on). + */ +static void compute_coding_constants(u32 mean_delta, u16 *min_bits, u32 *min_keys, + u32 *incr_keys) +{ + /* + * We want to compute the rounded value of log(2) * mean_delta. Since we cannot always use + * floating point, use a really good integer approximation. + */ + *incr_keys = (836158UL * mean_delta + 603160UL) / 1206321UL; + *min_bits = bits_per(*incr_keys + 1); + *min_keys = (1 << *min_bits) - *incr_keys; +} + +void uds_uninitialize_delta_index(struct delta_index *delta_index) +{ + unsigned int z; + + if (delta_index->delta_zones == NULL) + return; + + for (z = 0; z < delta_index->zone_count; z++) { + uds_free(uds_forget(delta_index->delta_zones[z].new_offsets)); + uds_free(uds_forget(delta_index->delta_zones[z].delta_lists)); + uds_free(uds_forget(delta_index->delta_zones[z].memory)); + } + + uds_free(delta_index->delta_zones); + memset(delta_index, 0, sizeof(struct delta_index)); +} + +static int initialize_delta_zone(struct delta_zone *delta_zone, size_t size, + u32 first_list, u32 list_count, u32 mean_delta, + u32 payload_bits, u8 tag) +{ + int result; + + result = uds_allocate(size, u8, "delta list", &delta_zone->memory); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(list_count + 2, u64, "delta list temp", + &delta_zone->new_offsets); + if (result != UDS_SUCCESS) + return result; + + /* Allocate the delta lists. */ + result = uds_allocate(list_count + 2, struct delta_list, "delta lists", + &delta_zone->delta_lists); + if (result != UDS_SUCCESS) + return result; + + compute_coding_constants(mean_delta, &delta_zone->min_bits, + &delta_zone->min_keys, &delta_zone->incr_keys); + delta_zone->value_bits = payload_bits; + delta_zone->buffered_writer = NULL; + delta_zone->size = size; + delta_zone->rebalance_time = 0; + delta_zone->rebalance_count = 0; + delta_zone->record_count = 0; + delta_zone->collision_count = 0; + delta_zone->discard_count = 0; + delta_zone->overflow_count = 0; + delta_zone->first_list = first_list; + delta_zone->list_count = list_count; + delta_zone->tag = tag; + + return UDS_SUCCESS; +} + +int uds_initialize_delta_index(struct delta_index *delta_index, unsigned int zone_count, + u32 list_count, u32 mean_delta, u32 payload_bits, + size_t memory_size, u8 tag) +{ + int result; + unsigned int z; + size_t zone_memory; + + result = uds_allocate(zone_count, struct delta_zone, "Delta Index Zones", + &delta_index->delta_zones); + if (result != UDS_SUCCESS) + return result; + + delta_index->zone_count = zone_count; + delta_index->list_count = list_count; + delta_index->lists_per_zone = DIV_ROUND_UP(list_count, zone_count); + delta_index->memory_size = 0; + delta_index->mutable = true; + delta_index->tag = tag; + + for (z = 0; z < zone_count; z++) { + u32 lists_in_zone = delta_index->lists_per_zone; + u32 first_list_in_zone = z * lists_in_zone; + + if (z == zone_count - 1) { + /* + * The last zone gets fewer lists if zone_count doesn't evenly divide + * list_count. We'll have an underflow if the assertion below doesn't hold. + */ + if (delta_index->list_count <= first_list_in_zone) { + uds_uninitialize_delta_index(delta_index); + return uds_log_error_strerror(UDS_INVALID_ARGUMENT, + "%u delta lists not enough for %u zones", + list_count, zone_count); + } + lists_in_zone = delta_index->list_count - first_list_in_zone; + } + + zone_memory = get_zone_memory_size(zone_count, memory_size); + result = initialize_delta_zone(&delta_index->delta_zones[z], zone_memory, + first_list_in_zone, lists_in_zone, + mean_delta, payload_bits, tag); + if (result != UDS_SUCCESS) { + uds_uninitialize_delta_index(delta_index); + return result; + } + + delta_index->memory_size += + (sizeof(struct delta_zone) + zone_memory + + (lists_in_zone + 2) * (sizeof(struct delta_list) + sizeof(u64))); + } + + uds_reset_delta_index(delta_index); + return UDS_SUCCESS; +} + +/* Read a bit field from an arbitrary bit boundary. */ +static inline u32 get_field(const u8 *memory, u64 offset, u8 size) +{ + const void *addr = memory + offset / BITS_PER_BYTE; + + return (get_unaligned_le32(addr) >> (offset % BITS_PER_BYTE)) & ((1 << size) - 1); +} + +/* Write a bit field to an arbitrary bit boundary. */ +static inline void set_field(u32 value, u8 *memory, u64 offset, u8 size) +{ + void *addr = memory + offset / BITS_PER_BYTE; + int shift = offset % BITS_PER_BYTE; + u32 data = get_unaligned_le32(addr); + + data &= ~(((1 << size) - 1) << shift); + data |= value << shift; + put_unaligned_le32(data, addr); +} + +/* Get the bit offset to the immutable delta list header. */ +static inline u32 get_immutable_header_offset(u32 list_number) +{ + return sizeof(struct delta_page_header) * BITS_PER_BYTE + + list_number * IMMUTABLE_HEADER_SIZE; +} + +/* Get the bit offset to the start of the immutable delta list bit stream. */ +static inline u32 get_immutable_start(const u8 *memory, u32 list_number) +{ + return get_field(memory, get_immutable_header_offset(list_number), + IMMUTABLE_HEADER_SIZE); +} + +/* Set the bit offset to the start of the immutable delta list bit stream. */ +static inline void set_immutable_start(u8 *memory, u32 list_number, u32 start) +{ + set_field(start, memory, get_immutable_header_offset(list_number), + IMMUTABLE_HEADER_SIZE); +} + +static bool verify_delta_index_page(u64 nonce, u16 list_count, u64 expected_nonce, + u8 *memory, size_t memory_size) +{ + unsigned int i; + + /* + * Verify the nonce. A mismatch can happen here during rebuild if we haven't written the + * entire volume at least once. + */ + if (nonce != expected_nonce) + return false; + + /* Verify that the number of delta lists can fit in the page. */ + if (list_count > ((memory_size - sizeof(struct delta_page_header)) * + BITS_PER_BYTE / IMMUTABLE_HEADER_SIZE)) + return false; + + /* + * Verify that the first delta list is immediately after the last delta + * list header. + */ + if (get_immutable_start(memory, 0) != get_immutable_header_offset(list_count + 1)) + return false; + + /* Verify that the lists are in the correct order. */ + for (i = 0; i < list_count; i++) { + if (get_immutable_start(memory, i) > get_immutable_start(memory, i + 1)) + return false; + } + + /* + * Verify that the last list ends on the page, and that there is room + * for the post-field guard bits. + */ + if (get_immutable_start(memory, list_count) > + (memory_size - POST_FIELD_GUARD_BYTES) * BITS_PER_BYTE) + return false; + + /* Verify that the guard bytes are correctly set to all ones. */ + for (i = 0; i < POST_FIELD_GUARD_BYTES; i++) { + if (memory[memory_size - POST_FIELD_GUARD_BYTES + i] != (u8) ~0) + return false; + } + + /* All verifications passed. */ + return true; +} + +/* Initialize a delta index page to refer to a supplied page. */ +int uds_initialize_delta_index_page(struct delta_index_page *delta_index_page, + u64 expected_nonce, u32 mean_delta, u32 payload_bits, + u8 *memory, size_t memory_size) +{ + u64 nonce; + u64 vcn; + u64 first_list; + u64 list_count; + struct delta_page_header *header = (struct delta_page_header *) memory; + struct delta_zone *delta_zone = &delta_index_page->delta_zone; + const u8 *nonce_addr = (const u8 *) &header->nonce; + const u8 *vcn_addr = (const u8 *) &header->virtual_chapter_number; + const u8 *first_list_addr = (const u8 *) &header->first_list; + const u8 *list_count_addr = (const u8 *) &header->list_count; + + /* First assume that the header is little endian. */ + nonce = get_unaligned_le64(nonce_addr); + vcn = get_unaligned_le64(vcn_addr); + first_list = get_unaligned_le16(first_list_addr); + list_count = get_unaligned_le16(list_count_addr); + if (!verify_delta_index_page(nonce, list_count, expected_nonce, memory, + memory_size)) { + /* If that fails, try big endian. */ + nonce = get_unaligned_be64(nonce_addr); + vcn = get_unaligned_be64(vcn_addr); + first_list = get_unaligned_be16(first_list_addr); + list_count = get_unaligned_be16(list_count_addr); + if (!verify_delta_index_page(nonce, list_count, expected_nonce, memory, + memory_size)) { + /* + * Both attempts failed. Do not log this as an error, because it can happen + * during a rebuild if we haven't written the entire volume at least once. + */ + return UDS_CORRUPT_DATA; + } + } + + delta_index_page->delta_index.delta_zones = delta_zone; + delta_index_page->delta_index.zone_count = 1; + delta_index_page->delta_index.list_count = list_count; + delta_index_page->delta_index.lists_per_zone = list_count; + delta_index_page->delta_index.mutable = false; + delta_index_page->delta_index.tag = 'p'; + delta_index_page->virtual_chapter_number = vcn; + delta_index_page->lowest_list_number = first_list; + delta_index_page->highest_list_number = first_list + list_count - 1; + + compute_coding_constants(mean_delta, &delta_zone->min_bits, + &delta_zone->min_keys, &delta_zone->incr_keys); + delta_zone->value_bits = payload_bits; + delta_zone->memory = memory; + delta_zone->delta_lists = NULL; + delta_zone->new_offsets = NULL; + delta_zone->buffered_writer = NULL; + delta_zone->size = memory_size; + delta_zone->rebalance_time = 0; + delta_zone->rebalance_count = 0; + delta_zone->record_count = 0; + delta_zone->collision_count = 0; + delta_zone->discard_count = 0; + delta_zone->overflow_count = 0; + delta_zone->first_list = 0; + delta_zone->list_count = list_count; + delta_zone->tag = 'p'; + + return UDS_SUCCESS; +} + +/* Read a large bit field from an arbitrary bit boundary. */ +static inline u64 get_big_field(const u8 *memory, u64 offset, u8 size) +{ + const void *addr = memory + offset / BITS_PER_BYTE; + + return (get_unaligned_le64(addr) >> (offset % BITS_PER_BYTE)) & ((1UL << size) - 1); +} + +/* Write a large bit field to an arbitrary bit boundary. */ +static inline void set_big_field(u64 value, u8 *memory, u64 offset, u8 size) +{ + void *addr = memory + offset / BITS_PER_BYTE; + u8 shift = offset % BITS_PER_BYTE; + u64 data = get_unaligned_le64(addr); + + data &= ~(((1UL << size) - 1) << shift); + data |= value << shift; + put_unaligned_le64(data, addr); +} + +/* Set a sequence of bits to all zeros. */ +static inline void set_zero(u8 *memory, u64 offset, u32 size) +{ + if (size > 0) { + u8 *addr = memory + offset / BITS_PER_BYTE; + u8 shift = offset % BITS_PER_BYTE; + u32 count = size + shift > BITS_PER_BYTE ? (u32) BITS_PER_BYTE - shift : size; + + *addr++ &= ~(((1 << count) - 1) << shift); + for (size -= count; size > BITS_PER_BYTE; size -= BITS_PER_BYTE) + *addr++ = 0; + + if (size > 0) + *addr &= 0xFF << size; + } +} + +/* + * Move several bits from a higher to a lower address, moving the lower addressed bits first. The + * size and memory offsets are measured in bits. + */ +static void move_bits_down(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, + u32 size) +{ + const u8 *source; + u8 *destination; + u8 offset; + u8 count; + u64 field; + + /* Start by moving one field that ends on a to int boundary. */ + count = (MAX_BIG_FIELD_BITS - ((to_offset + MAX_BIG_FIELD_BITS) % BITS_PER_TYPE(u32))); + field = get_big_field(from, from_offset, count); + set_big_field(field, to, to_offset, count); + from_offset += count; + to_offset += count; + size -= count; + + /* Now do the main loop to copy 32 bit chunks that are int-aligned at the destination. */ + offset = from_offset % BITS_PER_TYPE(u32); + source = from + (from_offset - offset) / BITS_PER_BYTE; + destination = to + to_offset / BITS_PER_BYTE; + while (size > MAX_BIG_FIELD_BITS) { + put_unaligned_le32(get_unaligned_le64(source) >> offset, destination); + source += sizeof(u32); + destination += sizeof(u32); + from_offset += BITS_PER_TYPE(u32); + to_offset += BITS_PER_TYPE(u32); + size -= BITS_PER_TYPE(u32); + } + + /* Finish up by moving any remaining bits. */ + if (size > 0) { + field = get_big_field(from, from_offset, size); + set_big_field(field, to, to_offset, size); + } +} + +/* + * Move several bits from a lower to a higher address, moving the higher addressed bits first. The + * size and memory offsets are measured in bits. + */ +static void move_bits_up(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, + u32 size) +{ + const u8 *source; + u8 *destination; + u8 offset; + u8 count; + u64 field; + + /* Start by moving one field that begins on a destination int boundary. */ + count = (to_offset + size) % BITS_PER_TYPE(u32); + if (count > 0) { + size -= count; + field = get_big_field(from, from_offset + size, count); + set_big_field(field, to, to_offset + size, count); + } + + /* Now do the main loop to copy 32 bit chunks that are int-aligned at the destination. */ + offset = (from_offset + size) % BITS_PER_TYPE(u32); + source = from + (from_offset + size - offset) / BITS_PER_BYTE; + destination = to + (to_offset + size) / BITS_PER_BYTE; + while (size > MAX_BIG_FIELD_BITS) { + source -= sizeof(u32); + destination -= sizeof(u32); + size -= BITS_PER_TYPE(u32); + put_unaligned_le32(get_unaligned_le64(source) >> offset, destination); + } + + /* Finish up by moving any remaining bits. */ + if (size > 0) { + field = get_big_field(from, from_offset, size); + set_big_field(field, to, to_offset, size); + } +} + +/* + * Move bits from one field to another. When the fields overlap, behave as if we first move all the + * bits from the source to a temporary value, and then move all the bits from the temporary value + * to the destination. The size and memory offsets are measured in bits. + */ +static void move_bits(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, u32 size) +{ + u64 field; + + /* A small move doesn't require special handling. */ + if (size <= MAX_BIG_FIELD_BITS) { + if (size > 0) { + field = get_big_field(from, from_offset, size); + set_big_field(field, to, to_offset, size); + } + + return; + } + + if (from_offset > to_offset) + move_bits_down(from, from_offset, to, to_offset, size); + else + move_bits_up(from, from_offset, to, to_offset, size); +} + +/* + * Pack delta lists from a mutable delta index into an immutable delta index page. A range of delta + * lists (starting with a specified list index) is copied from the mutable delta index into a + * memory page used in the immutable index. The number of lists copied onto the page is returned in + * list_count. + */ +int uds_pack_delta_index_page(const struct delta_index *delta_index, u64 header_nonce, + u8 *memory, size_t memory_size, u64 virtual_chapter_number, + u32 first_list, u32 *list_count) +{ + const struct delta_zone *delta_zone; + struct delta_list *delta_lists; + u32 max_lists; + u32 n_lists = 0; + u32 offset; + u32 i; + int free_bits; + int bits; + struct delta_page_header *header; + + delta_zone = &delta_index->delta_zones[0]; + delta_lists = &delta_zone->delta_lists[first_list + 1]; + max_lists = delta_index->list_count - first_list; + + /* + * Compute how many lists will fit on the page. Subtract the size of the fixed header, one + * delta list offset, and the guard bytes from the page size to determine how much space is + * available for delta lists. + */ + free_bits = memory_size * BITS_PER_BYTE; + free_bits -= get_immutable_header_offset(1); + free_bits -= GUARD_BITS; + if (free_bits < IMMUTABLE_HEADER_SIZE) { + /* This page is too small to store any delta lists. */ + return uds_log_error_strerror(UDS_OVERFLOW, + "Chapter Index Page of %zu bytes is too small", + memory_size); + } + + while (n_lists < max_lists) { + /* Each list requires a delta list offset and the list data. */ + bits = IMMUTABLE_HEADER_SIZE + delta_lists[n_lists].size; + if (bits > free_bits) + break; + + n_lists++; + free_bits -= bits; + } + + *list_count = n_lists; + + header = (struct delta_page_header *) memory; + put_unaligned_le64(header_nonce, (u8 *) &header->nonce); + put_unaligned_le64(virtual_chapter_number, + (u8 *) &header->virtual_chapter_number); + put_unaligned_le16(first_list, (u8 *) &header->first_list); + put_unaligned_le16(n_lists, (u8 *) &header->list_count); + + /* Construct the delta list offset table. */ + offset = get_immutable_header_offset(n_lists + 1); + set_immutable_start(memory, 0, offset); + for (i = 0; i < n_lists; i++) { + offset += delta_lists[i].size; + set_immutable_start(memory, i + 1, offset); + } + + /* Copy the delta list data onto the memory page. */ + for (i = 0; i < n_lists; i++) { + move_bits(delta_zone->memory, delta_lists[i].start, memory, + get_immutable_start(memory, i), delta_lists[i].size); + } + + /* Set all the bits in the guard bytes. */ + memset(memory + memory_size - POST_FIELD_GUARD_BYTES, ~0, + POST_FIELD_GUARD_BYTES); + return UDS_SUCCESS; +} + +/* Compute the new offsets of the delta lists. */ +static void compute_new_list_offsets(struct delta_zone *delta_zone, u32 growing_index, + size_t growing_size, size_t used_space) +{ + size_t spacing; + u32 i; + struct delta_list *delta_lists = delta_zone->delta_lists; + u32 tail_guard_index = delta_zone->list_count + 1; + + spacing = (delta_zone->size - used_space) / delta_zone->list_count; + delta_zone->new_offsets[0] = 0; + for (i = 0; i <= delta_zone->list_count; i++) { + delta_zone->new_offsets[i + 1] = + (delta_zone->new_offsets[i] + + get_delta_list_byte_size(&delta_lists[i]) + spacing); + delta_zone->new_offsets[i] *= BITS_PER_BYTE; + delta_zone->new_offsets[i] += delta_lists[i].start % BITS_PER_BYTE; + if (i == 0) + delta_zone->new_offsets[i + 1] -= spacing / 2; + if (i + 1 == growing_index) + delta_zone->new_offsets[i + 1] += growing_size; + } + + delta_zone->new_offsets[tail_guard_index] = + (delta_zone->size * BITS_PER_BYTE - delta_lists[tail_guard_index].size); +} + +static void rebalance_lists(struct delta_zone *delta_zone) +{ + struct delta_list *delta_lists; + u32 i; + size_t used_space = 0; + + /* Extend and balance memory to receive the delta lists */ + delta_lists = delta_zone->delta_lists; + for (i = 0; i <= delta_zone->list_count + 1; i++) + used_space += get_delta_list_byte_size(&delta_lists[i]); + + compute_new_list_offsets(delta_zone, 0, 0, used_space); + for (i = 1; i <= delta_zone->list_count + 1; i++) + delta_lists[i].start = delta_zone->new_offsets[i]; +} + +/* Start restoring a delta index from multiple input streams. */ +int uds_start_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + unsigned int zone_count = reader_count; + u64 record_count = 0; + u64 collision_count = 0; + u32 first_list[MAX_ZONES]; + u32 list_count[MAX_ZONES]; + unsigned int z; + u32 list_next = 0; + const struct delta_zone *delta_zone; + + /* Read and validate each header. */ + for (z = 0; z < zone_count; z++) { + struct delta_index_header header; + u8 buffer[sizeof(struct delta_index_header)]; + size_t offset = 0; + + result = uds_read_from_buffered_reader(buffered_readers[z], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read delta index header"); + } + + memcpy(&header.magic, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + decode_u32_le(buffer, &offset, &header.zone_number); + decode_u32_le(buffer, &offset, &header.zone_count); + decode_u32_le(buffer, &offset, &header.first_list); + decode_u32_le(buffer, &offset, &header.list_count); + decode_u64_le(buffer, &offset, &header.record_count); + decode_u64_le(buffer, &offset, &header.collision_count); + + result = ASSERT(offset == sizeof(struct delta_index_header), + "%zu bytes decoded of %zu expected", offset, + sizeof(struct delta_index_header)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read delta index header"); + } + + if (memcmp(header.magic, DELTA_INDEX_MAGIC, MAGIC_SIZE) != 0) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index file has bad magic number"); + } + + if (zone_count != header.zone_count) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index files contain mismatched zone counts (%u,%u)", + zone_count, header.zone_count); + } + + if (header.zone_number != z) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index zone %u found in slot %u", + header.zone_number, z); + } + + first_list[z] = header.first_list; + list_count[z] = header.list_count; + record_count += header.record_count; + collision_count += header.collision_count; + + if (first_list[z] != list_next) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index file for zone %u starts with list %u instead of list %u", + z, first_list[z], list_next); + } + + list_next += list_count[z]; + } + + if (list_next != delta_index->list_count) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index files contain %u delta lists instead of %u delta lists", + list_next, delta_index->list_count); + } + + if (collision_count > record_count) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index files contain %llu collisions and %llu records", + (unsigned long long) collision_count, + (unsigned long long) record_count); + } + + uds_reset_delta_index(delta_index); + delta_index->delta_zones[0].record_count = record_count; + delta_index->delta_zones[0].collision_count = collision_count; + + /* Read the delta lists and distribute them to the proper zones. */ + for (z = 0; z < zone_count; z++) { + u32 i; + + delta_index->load_lists[z] = 0; + for (i = 0; i < list_count[z]; i++) { + u16 delta_list_size; + u32 list_number; + unsigned int zone_number; + u8 size_data[sizeof(u16)]; + + result = uds_read_from_buffered_reader(buffered_readers[z], + size_data, + sizeof(size_data)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read delta index size"); + } + + delta_list_size = get_unaligned_le16(size_data); + if (delta_list_size > 0) + delta_index->load_lists[z] += 1; + + list_number = first_list[z] + i; + zone_number = list_number / delta_index->lists_per_zone; + delta_zone = &delta_index->delta_zones[zone_number]; + list_number -= delta_zone->first_list; + delta_zone->delta_lists[list_number + 1].size = delta_list_size; + } + } + + /* Prepare each zone to start receiving the delta list data. */ + for (z = 0; z < delta_index->zone_count; z++) + rebalance_lists(&delta_index->delta_zones[z]); + + return UDS_SUCCESS; +} + +static int restore_delta_list_to_zone(struct delta_zone *delta_zone, + const struct delta_list_save_info *save_info, + const u8 *data) +{ + struct delta_list *delta_list; + u16 bit_count; + u16 byte_count; + u32 list_number = save_info->index - delta_zone->first_list; + + if (list_number >= delta_zone->list_count) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "invalid delta list number %u not in range [%u,%u)", + save_info->index, delta_zone->first_list, + delta_zone->first_list + delta_zone->list_count); + } + + delta_list = &delta_zone->delta_lists[list_number + 1]; + if (delta_list->size == 0) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "unexpected delta list number %u", + save_info->index); + } + + bit_count = delta_list->size + save_info->bit_offset; + byte_count = BITS_TO_BYTES(bit_count); + if (save_info->byte_count != byte_count) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "unexpected delta list size %u != %u", + save_info->byte_count, byte_count); + } + + move_bits(data, save_info->bit_offset, delta_zone->memory, delta_list->start, + delta_list->size); + return UDS_SUCCESS; +} + +static int restore_delta_list_data(struct delta_index *delta_index, + unsigned int load_zone, + struct buffered_reader *buffered_reader, u8 *data) +{ + int result; + struct delta_list_save_info save_info; + u8 buffer[sizeof(struct delta_list_save_info)]; + unsigned int new_zone; + + result = uds_read_from_buffered_reader(buffered_reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return uds_log_warning_strerror(result, + "failed to read delta list data"); + + save_info = (struct delta_list_save_info) { + .tag = buffer[0], + .bit_offset = buffer[1], + .byte_count = get_unaligned_le16(&buffer[2]), + .index = get_unaligned_le32(&buffer[4]), + }; + + if ((save_info.bit_offset >= BITS_PER_BYTE) || + (save_info.byte_count > DELTA_LIST_MAX_BYTE_COUNT)) + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "corrupt delta list data"); + + /* Make sure the data is intended for this delta index. */ + if (save_info.tag != delta_index->tag) + return UDS_CORRUPT_DATA; + + if (save_info.index >= delta_index->list_count) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "invalid delta list number %u of %u", + save_info.index, + delta_index->list_count); + } + + result = uds_read_from_buffered_reader(buffered_reader, data, + save_info.byte_count); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read delta list data"); + } + + delta_index->load_lists[load_zone] -= 1; + new_zone = save_info.index / delta_index->lists_per_zone; + return restore_delta_list_to_zone(&delta_index->delta_zones[new_zone], + &save_info, data); +} + +/* Restore delta lists from saved data. */ +int uds_finish_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + int saved_result = UDS_SUCCESS; + unsigned int z; + u8 *data; + + result = uds_allocate(DELTA_LIST_MAX_BYTE_COUNT, u8, __func__, &data); + if (result != UDS_SUCCESS) + return result; + + for (z = 0; z < reader_count; z++) { + while (delta_index->load_lists[z] > 0) { + result = restore_delta_list_data(delta_index, z, + buffered_readers[z], data); + if (result != UDS_SUCCESS) { + saved_result = result; + break; + } + } + } + + uds_free(data); + return saved_result; +} + +int uds_check_guard_delta_lists(struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + unsigned int z; + u8 buffer[sizeof(struct delta_list_save_info)]; + + for (z = 0; z < reader_count; z++) { + result = uds_read_from_buffered_reader(buffered_readers[z], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) + return result; + + if (buffer[0] != 'z') + return UDS_CORRUPT_DATA; + } + + return UDS_SUCCESS; +} + +static int flush_delta_list(struct delta_zone *zone, u32 flush_index) +{ + struct delta_list *delta_list; + u8 buffer[sizeof(struct delta_list_save_info)]; + int result; + + delta_list = &zone->delta_lists[flush_index + 1]; + + buffer[0] = zone->tag; + buffer[1] = delta_list->start % BITS_PER_BYTE; + put_unaligned_le16(get_delta_list_byte_size(delta_list), &buffer[2]); + put_unaligned_le32(zone->first_list + flush_index, &buffer[4]); + + result = uds_write_to_buffered_writer(zone->buffered_writer, buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + uds_log_warning_strerror(result, "failed to write delta list memory"); + return result; + } + + result = uds_write_to_buffered_writer(zone->buffered_writer, + zone->memory + get_delta_list_byte_start(delta_list), + get_delta_list_byte_size(delta_list)); + if (result != UDS_SUCCESS) + uds_log_warning_strerror(result, "failed to write delta list memory"); + + return result; +} + +/* Start saving a delta index zone to a buffered output stream. */ +int uds_start_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number, + struct buffered_writer *buffered_writer) +{ + int result; + u32 i; + struct delta_zone *delta_zone; + u8 buffer[sizeof(struct delta_index_header)]; + size_t offset = 0; + + delta_zone = &delta_index->delta_zones[zone_number]; + memcpy(buffer, DELTA_INDEX_MAGIC, MAGIC_SIZE); + offset += MAGIC_SIZE; + encode_u32_le(buffer, &offset, zone_number); + encode_u32_le(buffer, &offset, delta_index->zone_count); + encode_u32_le(buffer, &offset, delta_zone->first_list); + encode_u32_le(buffer, &offset, delta_zone->list_count); + encode_u64_le(buffer, &offset, delta_zone->record_count); + encode_u64_le(buffer, &offset, delta_zone->collision_count); + + result = ASSERT(offset == sizeof(struct delta_index_header), + "%zu bytes encoded of %zu expected", offset, + sizeof(struct delta_index_header)); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); + if (result != UDS_SUCCESS) + return uds_log_warning_strerror(result, + "failed to write delta index header"); + + for (i = 0; i < delta_zone->list_count; i++) { + u8 data[sizeof(u16)]; + struct delta_list *delta_list; + + delta_list = &delta_zone->delta_lists[i + 1]; + put_unaligned_le16(delta_list->size, data); + result = uds_write_to_buffered_writer(buffered_writer, data, + sizeof(data)); + if (result != UDS_SUCCESS) + return uds_log_warning_strerror(result, + "failed to write delta list size"); + } + + delta_zone->buffered_writer = buffered_writer; + return UDS_SUCCESS; +} + +int uds_finish_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number) +{ + int result; + int first_error = UDS_SUCCESS; + u32 i; + struct delta_zone *delta_zone; + struct delta_list *delta_list; + + delta_zone = &delta_index->delta_zones[zone_number]; + for (i = 0; i < delta_zone->list_count; i++) { + delta_list = &delta_zone->delta_lists[i + 1]; + if (delta_list->size > 0) { + result = flush_delta_list(delta_zone, i); + if ((result != UDS_SUCCESS) && (first_error == UDS_SUCCESS)) + first_error = result; + } + } + + delta_zone->buffered_writer = NULL; + return first_error; +} + +int uds_write_guard_delta_list(struct buffered_writer *buffered_writer) +{ + int result; + u8 buffer[sizeof(struct delta_list_save_info)]; + + memset(buffer, 0, sizeof(struct delta_list_save_info)); + buffer[0] = 'z'; + + result = uds_write_to_buffered_writer(buffered_writer, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + uds_log_warning_strerror(result, "failed to write guard delta list"); + + return UDS_SUCCESS; +} + +size_t uds_compute_delta_index_save_bytes(u32 list_count, size_t memory_size) +{ + /* One zone will use at least as much memory as other zone counts. */ + return (sizeof(struct delta_index_header) + + list_count * (sizeof(struct delta_list_save_info) + 1) + + get_zone_memory_size(1, memory_size)); +} + +static int assert_not_at_end(const struct delta_index_entry *delta_entry) +{ + int result = ASSERT(!delta_entry->at_end, + "operation is invalid because the list entry is at the end of the delta list"); + if (result != UDS_SUCCESS) + result = UDS_BAD_STATE; + + return result; +} + +/* + * Prepare to search for an entry in the specified delta list. + * + * This is always the first function to be called when dealing with delta index entries. It is + * always followed by calls to uds_next_delta_index_entry() to iterate through a delta list. The + * fields of the delta_index_entry argument will be set up for iteration, but will not contain an + * entry from the list. + */ +int uds_start_delta_index_search(const struct delta_index *delta_index, u32 list_number, + u32 key, struct delta_index_entry *delta_entry) +{ + int result; + unsigned int zone_number; + struct delta_zone *delta_zone; + struct delta_list *delta_list; + + result = ASSERT((list_number < delta_index->list_count), + "Delta list number (%u) is out of range (%u)", list_number, + delta_index->list_count); + if (result != UDS_SUCCESS) + return UDS_CORRUPT_DATA; + + zone_number = list_number / delta_index->lists_per_zone; + delta_zone = &delta_index->delta_zones[zone_number]; + list_number -= delta_zone->first_list; + result = ASSERT((list_number < delta_zone->list_count), + "Delta list number (%u) is out of range (%u) for zone (%u)", + list_number, delta_zone->list_count, zone_number); + if (result != UDS_SUCCESS) + return UDS_CORRUPT_DATA; + + if (delta_index->mutable) { + delta_list = &delta_zone->delta_lists[list_number + 1]; + } else { + u32 end_offset; + + /* + * Translate the immutable delta list header into a temporary + * full delta list header. + */ + delta_list = &delta_entry->temp_delta_list; + delta_list->start = get_immutable_start(delta_zone->memory, list_number); + end_offset = get_immutable_start(delta_zone->memory, list_number + 1); + delta_list->size = end_offset - delta_list->start; + delta_list->save_key = 0; + delta_list->save_offset = 0; + } + + if (key > delta_list->save_key) { + delta_entry->key = delta_list->save_key; + delta_entry->offset = delta_list->save_offset; + } else { + delta_entry->key = 0; + delta_entry->offset = 0; + if (key == 0) { + /* + * This usually means we're about to walk the entire delta list, so get all + * of it into the CPU cache. + */ + uds_prefetch_range(&delta_zone->memory[delta_list->start / BITS_PER_BYTE], + delta_list->size / BITS_PER_BYTE, false); + } + } + + delta_entry->at_end = false; + delta_entry->delta_zone = delta_zone; + delta_entry->delta_list = delta_list; + delta_entry->entry_bits = 0; + delta_entry->is_collision = false; + delta_entry->list_number = list_number; + delta_entry->list_overflow = false; + delta_entry->value_bits = delta_zone->value_bits; + return UDS_SUCCESS; +} + +static inline u64 get_delta_entry_offset(const struct delta_index_entry *delta_entry) +{ + return delta_entry->delta_list->start + delta_entry->offset; +} + +/* + * Decode a delta index entry delta value. The delta_index_entry basically describes the previous + * list entry, and has had its offset field changed to point to the subsequent entry. We decode the + * bit stream and update the delta_list_entry to describe the entry. + */ +static inline void decode_delta(struct delta_index_entry *delta_entry) +{ + int key_bits; + u32 delta; + const struct delta_zone *delta_zone = delta_entry->delta_zone; + const u8 *memory = delta_zone->memory; + u64 delta_offset = get_delta_entry_offset(delta_entry) + delta_entry->value_bits; + const u8 *addr = memory + delta_offset / BITS_PER_BYTE; + int offset = delta_offset % BITS_PER_BYTE; + u32 data = get_unaligned_le32(addr) >> offset; + + addr += sizeof(u32); + key_bits = delta_zone->min_bits; + delta = data & ((1 << key_bits) - 1); + if (delta >= delta_zone->min_keys) { + data >>= key_bits; + if (data == 0) { + key_bits = sizeof(u32) * BITS_PER_BYTE - offset; + while ((data = get_unaligned_le32(addr)) == 0) { + addr += sizeof(u32); + key_bits += sizeof(u32) * BITS_PER_BYTE; + } + } + key_bits += ffs(data); + delta += ((key_bits - delta_zone->min_bits - 1) * delta_zone->incr_keys); + } + delta_entry->delta = delta; + delta_entry->key += delta; + + /* Check for a collision, a delta of zero after the start. */ + if (unlikely((delta == 0) && (delta_entry->offset > 0))) { + delta_entry->is_collision = true; + delta_entry->entry_bits = delta_entry->value_bits + key_bits + COLLISION_BITS; + } else { + delta_entry->is_collision = false; + delta_entry->entry_bits = delta_entry->value_bits + key_bits; + } +} + +noinline int uds_next_delta_index_entry(struct delta_index_entry *delta_entry) +{ + int result; + const struct delta_list *delta_list; + u32 next_offset; + u16 size; + + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + delta_list = delta_entry->delta_list; + delta_entry->offset += delta_entry->entry_bits; + size = delta_list->size; + if (unlikely(delta_entry->offset >= size)) { + delta_entry->at_end = true; + delta_entry->delta = 0; + delta_entry->is_collision = false; + result = ASSERT((delta_entry->offset == size), + "next offset past end of delta list"); + if (result != UDS_SUCCESS) + result = UDS_CORRUPT_DATA; + + return result; + } + + decode_delta(delta_entry); + + next_offset = delta_entry->offset + delta_entry->entry_bits; + if (next_offset > size) { + /* + * This is not an assertion because uds_validate_chapter_index_page() wants to + * handle this error. + */ + uds_log_warning("Decoded past the end of the delta list"); + return UDS_CORRUPT_DATA; + } + + return UDS_SUCCESS; +} + +int uds_remember_delta_index_offset(const struct delta_index_entry *delta_entry) +{ + int result; + struct delta_list *delta_list = delta_entry->delta_list; + + result = ASSERT(!delta_entry->is_collision, "entry is not a collision"); + if (result != UDS_SUCCESS) + return result; + + delta_list->save_key = delta_entry->key - delta_entry->delta; + delta_list->save_offset = delta_entry->offset; + return UDS_SUCCESS; +} + +static void set_delta(struct delta_index_entry *delta_entry, u32 delta) +{ + const struct delta_zone *delta_zone = delta_entry->delta_zone; + u32 key_bits = (delta_zone->min_bits + + ((delta_zone->incr_keys - + delta_zone->min_keys + delta) / + delta_zone->incr_keys)); + + delta_entry->delta = delta; + delta_entry->entry_bits = delta_entry->value_bits + key_bits; +} + +static void get_collision_name(const struct delta_index_entry *entry, u8 *name) +{ + u64 offset = get_delta_entry_offset(entry) + entry->entry_bits - COLLISION_BITS; + const u8 *addr = entry->delta_zone->memory + offset / BITS_PER_BYTE; + int size = COLLISION_BYTES; + int shift = offset % BITS_PER_BYTE; + + while (--size >= 0) + *name++ = get_unaligned_le16(addr++) >> shift; +} + +static void set_collision_name(const struct delta_index_entry *entry, const u8 *name) +{ + u64 offset = get_delta_entry_offset(entry) + entry->entry_bits - COLLISION_BITS; + u8 *addr = entry->delta_zone->memory + offset / BITS_PER_BYTE; + int size = COLLISION_BYTES; + int shift = offset % BITS_PER_BYTE; + u16 mask = ~((u16) 0xFF << shift); + u16 data; + + while (--size >= 0) { + data = (get_unaligned_le16(addr) & mask) | (*name++ << shift); + put_unaligned_le16(data, addr++); + } +} + +int uds_get_delta_index_entry(const struct delta_index *delta_index, u32 list_number, + u32 key, const u8 *name, + struct delta_index_entry *delta_entry) +{ + int result; + + result = uds_start_delta_index_search(delta_index, list_number, key, + delta_entry); + if (result != UDS_SUCCESS) + return result; + + do { + result = uds_next_delta_index_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + } while (!delta_entry->at_end && (key > delta_entry->key)); + + result = uds_remember_delta_index_offset(delta_entry); + if (result != UDS_SUCCESS) + return result; + + if (!delta_entry->at_end && (key == delta_entry->key)) { + struct delta_index_entry collision_entry = *delta_entry; + + for (;;) { + u8 full_name[COLLISION_BYTES]; + + result = uds_next_delta_index_entry(&collision_entry); + if (result != UDS_SUCCESS) + return result; + + if (collision_entry.at_end || !collision_entry.is_collision) + break; + + get_collision_name(&collision_entry, full_name); + if (memcmp(full_name, name, COLLISION_BYTES) == 0) { + *delta_entry = collision_entry; + break; + } + } + } + + return UDS_SUCCESS; +} + +int uds_get_delta_entry_collision(const struct delta_index_entry *delta_entry, u8 *name) +{ + int result; + + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(delta_entry->is_collision, + "Cannot get full block name from a non-collision delta index entry"); + if (result != UDS_SUCCESS) + return UDS_BAD_STATE; + + get_collision_name(delta_entry, name); + return UDS_SUCCESS; +} + +u32 uds_get_delta_entry_value(const struct delta_index_entry *delta_entry) +{ + return get_field(delta_entry->delta_zone->memory, + get_delta_entry_offset(delta_entry), delta_entry->value_bits); +} + +static int assert_mutable_entry(const struct delta_index_entry *delta_entry) +{ + int result = ASSERT((delta_entry->delta_list != &delta_entry->temp_delta_list), + "delta index is mutable"); + if (result != UDS_SUCCESS) + result = UDS_BAD_STATE; + + return result; +} + +int uds_set_delta_entry_value(const struct delta_index_entry *delta_entry, u32 value) +{ + int result; + u32 value_mask = (1 << delta_entry->value_bits) - 1; + + result = assert_mutable_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT((value & value_mask) == value, + "Value (%u) being set in a delta index is too large (must fit in %u bits)", + value, delta_entry->value_bits); + if (result != UDS_SUCCESS) + return UDS_INVALID_ARGUMENT; + + set_field(value, delta_entry->delta_zone->memory, + get_delta_entry_offset(delta_entry), delta_entry->value_bits); + return UDS_SUCCESS; +} + +/* + * Extend the memory used by the delta lists by adding growing_size bytes before the list indicated + * by growing_index, then rebalancing the lists in the new chunk. + */ +static int extend_delta_zone(struct delta_zone *delta_zone, u32 growing_index, + size_t growing_size) +{ + ktime_t start_time; + ktime_t end_time; + struct delta_list *delta_lists; + u32 i; + size_t used_space; + + + /* Calculate the amount of space that is or will be in use. */ + start_time = current_time_ns(CLOCK_MONOTONIC); + delta_lists = delta_zone->delta_lists; + used_space = growing_size; + for (i = 0; i <= delta_zone->list_count + 1; i++) + used_space += get_delta_list_byte_size(&delta_lists[i]); + + if (delta_zone->size < used_space) + return UDS_OVERFLOW; + + /* Compute the new offsets of the delta lists. */ + compute_new_list_offsets(delta_zone, growing_index, growing_size, used_space); + + /* + * When we rebalance the delta list, we will include the end guard list in the rebalancing. + * It contains the end guard data, which must be copied. + */ + rebalance_delta_zone(delta_zone, 1, delta_zone->list_count + 1); + end_time = current_time_ns(CLOCK_MONOTONIC); + delta_zone->rebalance_count++; + delta_zone->rebalance_time += ktime_sub(end_time, start_time); + return UDS_SUCCESS; +} + +static int insert_bits(struct delta_index_entry *delta_entry, u16 size) +{ + u64 free_before; + u64 free_after; + u64 source; + u64 destination; + u32 count; + bool before_flag; + u8 *memory; + struct delta_zone *delta_zone = delta_entry->delta_zone; + struct delta_list *delta_list = delta_entry->delta_list; + /* Compute bits in use before and after the inserted bits. */ + u32 total_size = delta_list->size; + u32 before_size = delta_entry->offset; + u32 after_size = total_size - delta_entry->offset; + + if (total_size + size > U16_MAX) { + delta_entry->list_overflow = true; + delta_zone->overflow_count++; + return UDS_OVERFLOW; + } + + /* Compute bits available before and after the delta list. */ + free_before = (delta_list[0].start - (delta_list[-1].start + delta_list[-1].size)); + free_after = (delta_list[1].start - (delta_list[0].start + delta_list[0].size)); + + if ((size <= free_before) && (size <= free_after)) { + /* + * We have enough space to use either before or after the list. Select the smaller + * amount of data. If it is exactly the same, try to take from the larger amount of + * free space. + */ + if (before_size < after_size) + before_flag = true; + else if (after_size < before_size) + before_flag = false; + else + before_flag = free_before > free_after; + } else if (size <= free_before) { + /* There is space before but not after. */ + before_flag = true; + } else if (size <= free_after) { + /* There is space after but not before. */ + before_flag = false; + } else { + /* + * Neither of the surrounding spaces is large enough for this request. Extend + * and/or rebalance the delta list memory choosing to move the least amount of + * data. + */ + int result; + u32 growing_index = delta_entry->list_number + 1; + + before_flag = before_size < after_size; + if (!before_flag) + growing_index++; + result = extend_delta_zone(delta_zone, growing_index, + BITS_TO_BYTES(size)); + if (result != UDS_SUCCESS) + return result; + } + + delta_list->size += size; + if (before_flag) { + source = delta_list->start; + destination = source - size; + delta_list->start -= size; + count = before_size; + } else { + source = delta_list->start + delta_entry->offset; + destination = source + size; + count = after_size; + } + + memory = delta_zone->memory; + move_bits(memory, source, memory, destination, count); + return UDS_SUCCESS; +} + +static void encode_delta(const struct delta_index_entry *delta_entry) +{ + u32 temp; + u32 t1; + u32 t2; + u64 offset; + const struct delta_zone *delta_zone = delta_entry->delta_zone; + u8 *memory = delta_zone->memory; + + offset = get_delta_entry_offset(delta_entry) + delta_entry->value_bits; + if (delta_entry->delta < delta_zone->min_keys) { + set_field(delta_entry->delta, memory, offset, delta_zone->min_bits); + return; + } + + temp = delta_entry->delta - delta_zone->min_keys; + t1 = (temp % delta_zone->incr_keys) + delta_zone->min_keys; + t2 = temp / delta_zone->incr_keys; + set_field(t1, memory, offset, delta_zone->min_bits); + set_zero(memory, offset + delta_zone->min_bits, t2); + set_field(1, memory, offset + delta_zone->min_bits + t2, 1); +} + +static void encode_entry(const struct delta_index_entry *delta_entry, u32 value, + const u8 *name) +{ + u8 *memory = delta_entry->delta_zone->memory; + u64 offset = get_delta_entry_offset(delta_entry); + + set_field(value, memory, offset, delta_entry->value_bits); + encode_delta(delta_entry); + if (name != NULL) + set_collision_name(delta_entry, name); +} + +/* + * Create a new entry in the delta index. If the entry is a collision, the full 256 bit name must + * be provided. + */ +int uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, u32 value, + const u8 *name) +{ + int result; + struct delta_zone *delta_zone; + + result = assert_mutable_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + + if (delta_entry->is_collision) { + /* + * The caller wants us to insert a collision entry onto a collision entry. This + * happens when we find a collision and attempt to add the name again to the index. + * This is normally a fatal error unless we are replaying a closed chapter while we + * are rebuilding a volume index. + */ + return UDS_DUPLICATE_NAME; + } + + if (delta_entry->offset < delta_entry->delta_list->save_offset) { + /* + * The saved entry offset is after the new entry and will no longer be valid, so + * replace it with the insertion point. + */ + result = uds_remember_delta_index_offset(delta_entry); + if (result != UDS_SUCCESS) + return result; + } + + if (name != NULL) { + /* Insert a collision entry which is placed after this entry. */ + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT((key == delta_entry->key), + "incorrect key for collision entry"); + if (result != UDS_SUCCESS) + return result; + + delta_entry->offset += delta_entry->entry_bits; + set_delta(delta_entry, 0); + delta_entry->is_collision = true; + delta_entry->entry_bits += COLLISION_BITS; + result = insert_bits(delta_entry, delta_entry->entry_bits); + } else if (delta_entry->at_end) { + /* Insert a new entry at the end of the delta list. */ + result = ASSERT((key >= delta_entry->key), "key past end of list"); + if (result != UDS_SUCCESS) + return result; + + set_delta(delta_entry, key - delta_entry->key); + delta_entry->key = key; + delta_entry->at_end = false; + result = insert_bits(delta_entry, delta_entry->entry_bits); + } else { + u16 old_entry_size; + u16 additional_size; + struct delta_index_entry next_entry; + u32 next_value; + + /* + * Insert a new entry which requires the delta in the following entry to be + * updated. + */ + result = ASSERT((key < delta_entry->key), + "key precedes following entry"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT((key >= delta_entry->key - delta_entry->delta), + "key effects following entry's delta"); + if (result != UDS_SUCCESS) + return result; + + old_entry_size = delta_entry->entry_bits; + next_entry = *delta_entry; + next_value = uds_get_delta_entry_value(&next_entry); + set_delta(delta_entry, key - (delta_entry->key - delta_entry->delta)); + delta_entry->key = key; + set_delta(&next_entry, next_entry.key - key); + next_entry.offset += delta_entry->entry_bits; + /* The two new entries are always bigger than the single entry being replaced. */ + additional_size = (delta_entry->entry_bits + + next_entry.entry_bits - old_entry_size); + result = insert_bits(delta_entry, additional_size); + if (result != UDS_SUCCESS) + return result; + + encode_entry(&next_entry, next_value, NULL); + } + + if (result != UDS_SUCCESS) + return result; + + encode_entry(delta_entry, value, name); + delta_zone = delta_entry->delta_zone; + delta_zone->record_count++; + delta_zone->collision_count += delta_entry->is_collision ? 1 : 0; + return UDS_SUCCESS; +} + +static void delete_bits(const struct delta_index_entry *delta_entry, int size) +{ + u64 source; + u64 destination; + u32 count; + bool before_flag; + struct delta_list *delta_list = delta_entry->delta_list; + u8 *memory = delta_entry->delta_zone->memory; + /* Compute bits retained before and after the deleted bits. */ + u32 total_size = delta_list->size; + u32 before_size = delta_entry->offset; + u32 after_size = total_size - delta_entry->offset - size; + + /* + * Determine whether to add to the available space either before or after the delta list. + * We prefer to move the least amount of data. If it is exactly the same, try to add to the + * smaller amount of free space. + */ + if (before_size < after_size) { + before_flag = true; + } else if (after_size < before_size) { + before_flag = false; + } else { + u64 free_before = + (delta_list[0].start - (delta_list[-1].start + delta_list[-1].size)); + u64 free_after = + (delta_list[1].start - (delta_list[0].start + delta_list[0].size)); + + before_flag = (free_before < free_after); + } + + delta_list->size -= size; + if (before_flag) { + source = delta_list->start; + destination = source + size; + delta_list->start += size; + count = before_size; + } else { + destination = delta_list->start + delta_entry->offset; + source = destination + size; + count = after_size; + } + + move_bits(memory, source, memory, destination, count); +} + +int uds_remove_delta_index_entry(struct delta_index_entry *delta_entry) +{ + int result; + struct delta_index_entry next_entry; + struct delta_zone *delta_zone; + struct delta_list *delta_list; + + result = assert_mutable_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + + next_entry = *delta_entry; + result = uds_next_delta_index_entry(&next_entry); + if (result != UDS_SUCCESS) + return result; + + delta_zone = delta_entry->delta_zone; + + if (delta_entry->is_collision) { + /* This is a collision entry, so just remove it. */ + delete_bits(delta_entry, delta_entry->entry_bits); + next_entry.offset = delta_entry->offset; + delta_zone->collision_count -= 1; + } else if (next_entry.at_end) { + /* This entry is at the end of the list, so just remove it. */ + delete_bits(delta_entry, delta_entry->entry_bits); + next_entry.key -= delta_entry->delta; + next_entry.offset = delta_entry->offset; + } else { + /* The delta in the next entry needs to be updated. */ + u32 next_value = uds_get_delta_entry_value(&next_entry); + u16 old_size = delta_entry->entry_bits + next_entry.entry_bits; + + if (next_entry.is_collision) { + next_entry.is_collision = false; + delta_zone->collision_count -= 1; + } + + set_delta(&next_entry, delta_entry->delta + next_entry.delta); + next_entry.offset = delta_entry->offset; + /* The one new entry is always smaller than the two entries being replaced. */ + delete_bits(delta_entry, old_size - next_entry.entry_bits); + encode_entry(&next_entry, next_value, NULL); + } + + delta_zone->record_count--; + delta_zone->discard_count++; + *delta_entry = next_entry; + + delta_list = delta_entry->delta_list; + if (delta_entry->offset < delta_list->save_offset) { + /* The saved entry offset is no longer valid. */ + delta_list->save_key = 0; + delta_list->save_offset = 0; + } + + return UDS_SUCCESS; +} + +void uds_get_delta_index_stats(const struct delta_index *delta_index, + struct delta_index_stats *stats) +{ + unsigned int z; + const struct delta_zone *delta_zone; + + memset(stats, 0, sizeof(struct delta_index_stats)); + for (z = 0; z < delta_index->zone_count; z++) { + delta_zone = &delta_index->delta_zones[z]; + stats->rebalance_time += delta_zone->rebalance_time; + stats->rebalance_count += delta_zone->rebalance_count; + stats->record_count += delta_zone->record_count; + stats->collision_count += delta_zone->collision_count; + stats->discard_count += delta_zone->discard_count; + stats->overflow_count += delta_zone->overflow_count; + stats->list_count += delta_zone->list_count; + } +} + +size_t uds_compute_delta_index_size(u32 entry_count, u32 mean_delta, u32 payload_bits) +{ + u16 min_bits; + u32 incr_keys; + u32 min_keys; + + compute_coding_constants(mean_delta, &min_bits, &min_keys, &incr_keys); + /* On average, each delta is encoded into about min_bits + 1.5 bits. */ + return entry_count * (payload_bits + min_bits + 1) + entry_count / 2; +} + +u32 uds_get_delta_index_page_count(u32 entry_count, u32 list_count, u32 mean_delta, + u32 payload_bits, size_t bytes_per_page) +{ + unsigned int bits_per_delta_list; + unsigned int bits_per_page; + size_t bits_per_index; + + /* Compute the expected number of bits needed for all the entries. */ + bits_per_index = uds_compute_delta_index_size(entry_count, mean_delta, + payload_bits); + bits_per_delta_list = bits_per_index / list_count; + + /* Add in the immutable delta list headers. */ + bits_per_index += list_count * IMMUTABLE_HEADER_SIZE; + /* Compute the number of usable bits on an immutable index page. */ + bits_per_page = ((bytes_per_page - sizeof(struct delta_page_header)) * BITS_PER_BYTE); + /* + * Reduce the bits per page by one immutable delta list header and one delta list to + * account for internal fragmentation. + */ + bits_per_page -= IMMUTABLE_HEADER_SIZE + bits_per_delta_list; + /* Now compute the number of pages needed. */ + return DIV_ROUND_UP(bits_per_index, bits_per_page); +} + +void uds_log_delta_index_entry(struct delta_index_entry *delta_entry) +{ + uds_log_ratelimit(uds_log_info, + "List 0x%X Key 0x%X Offset 0x%X%s%s List_size 0x%X%s", + delta_entry->list_number, delta_entry->key, + delta_entry->offset, delta_entry->at_end ? " end" : "", + delta_entry->is_collision ? " collision" : "", + delta_entry->delta_list->size, + delta_entry->list_overflow ? " overflow" : ""); + delta_entry->list_overflow = false; +} diff --git a/drivers/md/dm-vdo/delta-index.h b/drivers/md/dm-vdo/delta-index.h new file mode 100644 index 000000000000..046290b7ebe7 --- /dev/null +++ b/drivers/md/dm-vdo/delta-index.h @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_DELTA_INDEX_H +#define UDS_DELTA_INDEX_H + +#include + +#include "config.h" +#include "io-factory.h" +#include "numeric.h" +#include "time-utils.h" + +/* + * A delta index is a key-value store, where each entry maps an address (the key) to a payload (the + * value). The entries are sorted by address, and only the delta between successive addresses is + * stored in the entry. The addresses are assumed to be uniformly distributed, and the deltas are + * therefore exponentially distributed. + * + * A delta_index can either be mutable or immutable depending on its expected use. The immutable + * form of a delta index is used for the indexes of closed chapters committed to the volume. The + * mutable form of a delta index is used by the volume index, and also by the chapter index in an + * open chapter. Like the index as a whole, each mutable delta index is divided into a number of + * independent zones. + */ + +struct delta_list { + /* The offset of the delta list start, in bits */ + u64 start; + /* The number of bits in the delta list */ + u16 size; + /* Where the last search "found" the key, in bits */ + u16 save_offset; + /* The key for the record just before save_offset */ + u32 save_key; +}; + +struct delta_zone { + /* The delta list memory */ + u8 *memory; + /* The delta list headers */ + struct delta_list *delta_lists; + /* Temporary starts of delta lists */ + u64 *new_offsets; + /* Buffered writer for saving an index */ + struct buffered_writer *buffered_writer; + /* The size of delta list memory */ + size_t size; + /* Nanoseconds spent rebalancing */ + ktime_t rebalance_time; + /* Number of memory rebalances */ + u32 rebalance_count; + /* The number of bits in a stored value */ + u8 value_bits; + /* The number of bits in the minimal key code */ + u16 min_bits; + /* The number of keys used in a minimal code */ + u32 min_keys; + /* The number of keys used for another code bit */ + u32 incr_keys; + /* The number of records in the index */ + u64 record_count; + /* The number of collision records */ + u64 collision_count; + /* The number of records removed */ + u64 discard_count; + /* The number of UDS_OVERFLOW errors detected */ + u64 overflow_count; + /* The index of the first delta list */ + u32 first_list; + /* The number of delta lists */ + u32 list_count; + /* Tag belonging to this delta index */ + u8 tag; +} __aligned(L1_CACHE_BYTES); + +struct delta_list_save_info { + /* Tag identifying which delta index this list is in */ + u8 tag; + /* Bit offset of the start of the list data */ + u8 bit_offset; + /* Number of bytes of list data */ + u16 byte_count; + /* The delta list number within the delta index */ + u32 index; +} __packed; + +struct delta_index { + /* The zones */ + struct delta_zone *delta_zones; + /* The number of zones */ + unsigned int zone_count; + /* The number of delta lists */ + u32 list_count; + /* Maximum lists per zone */ + u32 lists_per_zone; + /* Total memory allocated to this index */ + size_t memory_size; + /* The number of non-empty lists at load time per zone */ + u32 load_lists[MAX_ZONES]; + /* True if this index is mutable */ + bool mutable; + /* Tag belonging to this delta index */ + u8 tag; +}; + +/* + * A delta_index_page describes a single page of a chapter index. The delta_index field allows the + * page to be treated as an immutable delta_index. We use the delta_zone field to treat the chapter + * index page as a single zone index, and without the need to do an additional memory allocation. + */ +struct delta_index_page { + struct delta_index delta_index; + /* These values are loaded from the DeltaPageHeader */ + u32 lowest_list_number; + u32 highest_list_number; + u64 virtual_chapter_number; + /* This structure describes the single zone of a delta index page. */ + struct delta_zone delta_zone; +}; + +/* + * Notes on the delta_index_entries: + * + * The fields documented as "public" can be read by any code that uses a delta_index. The fields + * documented as "private" carry information between delta_index method calls and should not be + * used outside the delta_index module. + * + * (1) The delta_index_entry is used like an iterator when searching a delta list. + * + * (2) It is also the result of a successful search and can be used to refer to the element found + * by the search. + * + * (3) It is also the result of an unsuccessful search and can be used to refer to the insertion + * point for a new record. + * + * (4) If at_end is true, the delta_list entry can only be used as the insertion point for a new + * record at the end of the list. + * + * (5) If at_end is false and is_collision is true, the delta_list entry fields refer to a + * collision entry in the list, and the delta_list entry can be used a a reference to this + * entry. + * + * (6) If at_end is false and is_collision is false, the delta_list entry fields refer to a + * non-collision entry in the list. Such delta_list entries can be used as a reference to a + * found entry, or an insertion point for a non-collision entry before this entry, or an + * insertion point for a collision entry that collides with this entry. + */ +struct delta_index_entry { + /* Public fields */ + /* The key for this entry */ + u32 key; + /* We are after the last list entry */ + bool at_end; + /* This record is a collision */ + bool is_collision; + + /* Private fields */ + /* This delta list overflowed */ + bool list_overflow; + /* The number of bits used for the value */ + u8 value_bits; + /* The number of bits used for the entire entry */ + u16 entry_bits; + /* The delta index zone */ + struct delta_zone *delta_zone; + /* The delta list containing the entry */ + struct delta_list *delta_list; + /* The delta list number */ + u32 list_number; + /* Bit offset of this entry within the list */ + u16 offset; + /* The delta between this and previous entry */ + u32 delta; + /* Temporary delta list for immutable indices */ + struct delta_list temp_delta_list; +}; + +struct delta_index_stats { + /* Number of bytes allocated */ + size_t memory_allocated; + /* Nanoseconds spent rebalancing */ + ktime_t rebalance_time; + /* Number of memory rebalances */ + u32 rebalance_count; + /* The number of records in the index */ + u64 record_count; + /* The number of collision records */ + u64 collision_count; + /* The number of records removed */ + u64 discard_count; + /* The number of UDS_OVERFLOW errors detected */ + u64 overflow_count; + /* The number of delta lists */ + u32 list_count; +}; + +int __must_check uds_initialize_delta_index(struct delta_index *delta_index, + unsigned int zone_count, u32 list_count, + u32 mean_delta, u32 payload_bits, + size_t memory_size, u8 tag); + +int __must_check uds_initialize_delta_index_page(struct delta_index_page *delta_index_page, + u64 expected_nonce, u32 mean_delta, + u32 payload_bits, u8 *memory, + size_t memory_size); + +void uds_uninitialize_delta_index(struct delta_index *delta_index); + +void uds_reset_delta_index(const struct delta_index *delta_index); + +int __must_check uds_pack_delta_index_page(const struct delta_index *delta_index, + u64 header_nonce, u8 *memory, + size_t memory_size, + u64 virtual_chapter_number, u32 first_list, + u32 *list_count); + +int __must_check uds_start_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count); + +int __must_check uds_finish_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count); + +int __must_check uds_check_guard_delta_lists(struct buffered_reader **buffered_readers, + unsigned int reader_count); + +int __must_check uds_start_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number, + struct buffered_writer *buffered_writer); + +int __must_check uds_finish_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number); + +int __must_check uds_write_guard_delta_list(struct buffered_writer *buffered_writer); + +size_t __must_check uds_compute_delta_index_save_bytes(u32 list_count, + size_t memory_size); + +int __must_check uds_start_delta_index_search(const struct delta_index *delta_index, + u32 list_number, u32 key, + struct delta_index_entry *iterator); + +int __must_check uds_next_delta_index_entry(struct delta_index_entry *delta_entry); + +int __must_check uds_remember_delta_index_offset(const struct delta_index_entry *delta_entry); + +int __must_check uds_get_delta_index_entry(const struct delta_index *delta_index, + u32 list_number, u32 key, const u8 *name, + struct delta_index_entry *delta_entry); + +int __must_check uds_get_delta_entry_collision(const struct delta_index_entry *delta_entry, + u8 *name); + +u32 __must_check uds_get_delta_entry_value(const struct delta_index_entry *delta_entry); + +int __must_check uds_set_delta_entry_value(const struct delta_index_entry *delta_entry, u32 value); + +int __must_check uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, + u32 value, const u8 *name); + +int __must_check uds_remove_delta_index_entry(struct delta_index_entry *delta_entry); + +void uds_get_delta_index_stats(const struct delta_index *delta_index, + struct delta_index_stats *stats); + +size_t __must_check uds_compute_delta_index_size(u32 entry_count, u32 mean_delta, + u32 payload_bits); + +u32 uds_get_delta_index_page_count(u32 entry_count, u32 list_count, u32 mean_delta, + u32 payload_bits, size_t bytes_per_page); + +void uds_log_delta_index_entry(struct delta_index_entry *delta_entry); + +#endif /* UDS_DELTA_INDEX_H */ diff --git a/drivers/md/dm-vdo/hash-utils.h b/drivers/md/dm-vdo/hash-utils.h new file mode 100644 index 000000000000..9ee729c14f2e --- /dev/null +++ b/drivers/md/dm-vdo/hash-utils.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_HASH_UTILS_H +#define UDS_HASH_UTILS_H + +#include "geometry.h" +#include "numeric.h" +#include "uds.h" + +/* Utilities for extracting portions of a request name for various uses. */ + +/* How various portions of a record name are apportioned. */ +enum { + VOLUME_INDEX_BYTES_OFFSET = 0, + VOLUME_INDEX_BYTES_COUNT = 8, + CHAPTER_INDEX_BYTES_OFFSET = 8, + CHAPTER_INDEX_BYTES_COUNT = 6, + SAMPLE_BYTES_OFFSET = 14, + SAMPLE_BYTES_COUNT = 2, +}; + +static inline u64 uds_extract_chapter_index_bytes(const struct uds_record_name *name) +{ + const u8 *chapter_bits = &name->name[CHAPTER_INDEX_BYTES_OFFSET]; + u64 bytes = (u64) get_unaligned_be16(chapter_bits) << 32; + + bytes |= get_unaligned_be32(chapter_bits + 2); + return bytes; +} + +static inline u64 uds_extract_volume_index_bytes(const struct uds_record_name *name) +{ + return get_unaligned_be64(&name->name[VOLUME_INDEX_BYTES_OFFSET]); +} + +static inline u32 uds_extract_sampling_bytes(const struct uds_record_name *name) +{ + return get_unaligned_be16(&name->name[SAMPLE_BYTES_OFFSET]); +} + +/* Compute the chapter delta list for a given name. */ +static inline u32 uds_hash_to_chapter_delta_list(const struct uds_record_name *name, + const struct geometry *geometry) +{ + return ((uds_extract_chapter_index_bytes(name) >> geometry->chapter_address_bits) & + ((1 << geometry->chapter_delta_list_bits) - 1)); +} + +/* Compute the chapter delta address for a given name. */ +static inline u32 uds_hash_to_chapter_delta_address(const struct uds_record_name *name, + const struct geometry *geometry) +{ + return uds_extract_chapter_index_bytes(name) & ((1 << geometry->chapter_address_bits) - 1); +} + +static inline unsigned int uds_name_to_hash_slot(const struct uds_record_name *name, + unsigned int slot_count) +{ + return (unsigned int) (uds_extract_chapter_index_bytes(name) % slot_count); +} + +#endif /* UDS_HASH_UTILS_H */ From a4eb7e2555171a95135678ea111a60f0d2a154e6 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:53:47 -0500 Subject: [PATCH 0560/1038] dm vdo: implement the volume index The volume index is a large delta index that maps each record name to the chapter which contains the newest record for that name. The volume index can contain several million records and is stored entirely in memory while the index is operating, accounting for the majority of the deduplication index's memory budget. The volume index is composed of two subindexes in order to handle sparse hook names separately from regular names. If sparse indexing is not enabled, the sparse hook portion of the volume index is not used or instantiated. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/volume-index.c | 1290 ++++++++++++++++++++++++++++++ drivers/md/dm-vdo/volume-index.h | 192 +++++ 2 files changed, 1482 insertions(+) create mode 100644 drivers/md/dm-vdo/volume-index.c create mode 100644 drivers/md/dm-vdo/volume-index.h diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c new file mode 100644 index 000000000000..11dceef0ab60 --- /dev/null +++ b/drivers/md/dm-vdo/volume-index.c @@ -0,0 +1,1290 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ +#include "volume-index.h" + +#include +#include +#include +#include +#include + +#include "config.h" +#include "errors.h" +#include "geometry.h" +#include "hash-utils.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "uds.h" +#include "uds-threads.h" + +/* + * The volume index is a combination of two separate subindexes, one containing sparse hook entries + * (retained for all chapters), and one containing the remaining entries (retained only for the + * dense chapters). If there are no sparse chapters, only the non-hook sub index is used, and it + * will contain all records for all chapters. + * + * The volume index is also divided into zones, with one thread operating on each zone. Each + * incoming request is dispatched to the appropriate thread, and then to the appropriate subindex. + * Each delta list is handled by a single zone. To ensure that the distribution of delta lists to + * zones doesn't underflow (leaving some zone with no delta lists), the minimum number of delta + * lists must be the square of the maximum zone count for both subindexes. + * + * Each subindex zone is a delta index where the payload is a chapter number. The volume index can + * compute the delta list number, address, and zone number from the record name in order to + * dispatch record handling to the correct structures. + * + * Most operations that use all the zones take place either before request processing is allowed, + * or after all requests have been flushed in order to shut down. The only multi-threaded operation + * supported during normal operation is the uds_lookup_volume_index_name() method, used to determine + * whether a new chapter should be loaded into the sparse index cache. This operation only uses the + * sparse hook subindex, and the zone mutexes are used to make this operation safe. + * + * There are three ways of expressing chapter numbers in the volume index: virtual, index, and + * rolling. The interface to the volume index uses virtual chapter numbers, which are 64 bits long. + * Internally the subindex stores only the minimal number of bits necessary by masking away the + * high-order bits. When the index needs to deal with ordering of index chapter numbers, as when + * flushing entries from older chapters, it rolls the index chapter number around so that the + * smallest one in use is mapped to 0. See convert_index_to_virtual() or flush_invalid_entries() + * for an example of this technique. + * + * For efficiency, when older chapter numbers become invalid, the index does not immediately remove + * the invalidated entries. Instead it lazily removes them from a given delta list the next time it + * walks that list during normal operation. Because of this, the index size must be increased + * somewhat to accommodate all the invalid entries that have not yet been removed. For the standard + * index sizes, this requires about 4 chapters of old entries per 1024 chapters of valid entries in + * the index. + */ + +struct sub_index_parameters { + /* The number of bits in address mask */ + u8 address_bits; + /* The number of bits in chapter number */ + u8 chapter_bits; + /* The mean delta */ + u32 mean_delta; + /* The number of delta lists */ + u64 list_count; + /* The number of chapters used */ + u32 chapter_count; + /* The number of bits per chapter */ + size_t chapter_size_in_bits; + /* The number of bytes of delta list memory */ + size_t memory_size; + /* The number of bytes the index should keep free at all times */ + size_t target_free_bytes; +}; + +struct split_config { + /* The hook subindex configuration */ + struct configuration hook_config; + struct geometry hook_geometry; + + /* The non-hook subindex configuration */ + struct configuration non_hook_config; + struct geometry non_hook_geometry; +}; + +struct chapter_range { + u32 chapter_start; + u32 chapter_count; +}; + +enum { MAGIC_SIZE = 8 }; +static const char MAGIC_START_5[] = "MI5-0005"; + +struct sub_index_data { + char magic[MAGIC_SIZE]; /* MAGIC_START_5 */ + u64 volume_nonce; + u64 virtual_chapter_low; + u64 virtual_chapter_high; + u32 first_list; + u32 list_count; +}; + +static const char MAGIC_START_6[] = "MI6-0001"; + +struct volume_index_data { + char magic[MAGIC_SIZE]; /* MAGIC_START_6 */ + u32 sparse_sample_rate; +}; + +static inline u32 extract_address(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + return uds_extract_volume_index_bytes(name) & sub_index->address_mask; +} + +static inline u32 extract_dlist_num(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + u64 bits = uds_extract_volume_index_bytes(name); + + return (bits >> sub_index->address_bits) % sub_index->list_count; +} + +static inline const struct volume_sub_index_zone * +get_zone_for_record(const struct volume_index_record *record) +{ + return &record->sub_index->zones[record->zone_number]; +} + +static inline u64 convert_index_to_virtual(const struct volume_index_record *record, + u32 index_chapter) +{ + const struct volume_sub_index_zone *volume_index_zone = get_zone_for_record(record); + u32 rolling_chapter = ((index_chapter - volume_index_zone->virtual_chapter_low) & + record->sub_index->chapter_mask); + + return volume_index_zone->virtual_chapter_low + rolling_chapter; +} + +static inline u32 convert_virtual_to_index(const struct volume_sub_index *sub_index, + u64 virtual_chapter) +{ + return virtual_chapter & sub_index->chapter_mask; +} + +static inline bool is_virtual_chapter_indexed(const struct volume_index_record *record, + u64 virtual_chapter) +{ + const struct volume_sub_index_zone *volume_index_zone = get_zone_for_record(record); + + return ((virtual_chapter >= volume_index_zone->virtual_chapter_low) && + (virtual_chapter <= volume_index_zone->virtual_chapter_high)); +} + +static inline bool has_sparse(const struct volume_index *volume_index) +{ + return volume_index->sparse_sample_rate > 0; +} + +bool uds_is_volume_index_sample(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + if (!has_sparse(volume_index)) + return false; + + return (uds_extract_sampling_bytes(name) % volume_index->sparse_sample_rate) == 0; +} + +static inline const struct volume_sub_index * +get_volume_sub_index(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + return (uds_is_volume_index_sample(volume_index, name) ? + &volume_index->vi_hook : + &volume_index->vi_non_hook); +} + +static unsigned int get_volume_sub_index_zone(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + return extract_dlist_num(sub_index, name) / sub_index->delta_index.lists_per_zone; +} + +unsigned int uds_get_volume_index_zone(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + return get_volume_sub_index_zone(get_volume_sub_index(volume_index, name), name); +} + +static int compute_volume_sub_index_parameters(const struct configuration *config, + struct sub_index_parameters *params) +{ + enum { DELTA_LIST_SIZE = 256 }; + u64 entries_in_volume_index, address_span; + u32 chapters_in_volume_index, invalid_chapters; + u32 rounded_chapters; + u64 delta_list_records; + u32 address_count; + u64 index_size_in_bits; + size_t expected_index_size; + u64 min_delta_lists = MAX_ZONES * MAX_ZONES; + struct geometry *geometry = config->geometry; + u64 records_per_chapter = geometry->records_per_chapter; + + params->chapter_count = geometry->chapters_per_volume; + /* + * Make sure that the number of delta list records in the volume index does not change when + * the volume is reduced by one chapter. This preserves the mapping from name to volume + * index delta list. + */ + rounded_chapters = params->chapter_count; + if (uds_is_reduced_geometry(geometry)) + rounded_chapters += 1; + delta_list_records = records_per_chapter * rounded_chapters; + address_count = config->volume_index_mean_delta * DELTA_LIST_SIZE; + params->list_count = max(delta_list_records / DELTA_LIST_SIZE, min_delta_lists); + params->address_bits = bits_per(address_count - 1); + params->chapter_bits = bits_per(rounded_chapters - 1); + if ((u32) params->list_count != params->list_count) { + return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot initialize volume index with %llu delta lists", + (unsigned long long) params->list_count); + } + + if (params->address_bits > 31) { + return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot initialize volume index with %u address bits", + params->address_bits); + } + + /* + * The probability that a given delta list is not touched during the writing of an entire + * chapter is: + * + * double p_not_touched = pow((double) (params->list_count - 1) / params->list_count, + * records_per_chapter); + * + * For the standard index sizes, about 78% of the delta lists are not touched, and + * therefore contain old index entries that have not been eliminated by the lazy LRU + * processing. Then the number of old index entries that accumulate over the entire index, + * in terms of full chapters worth of entries, is: + * + * double invalid_chapters = p_not_touched / (1.0 - p_not_touched); + * + * For the standard index sizes, the index needs about 3.5 chapters of space for the old + * entries in a 1024 chapter index, so round this up to use 4 chapters per 1024 chapters in + * the index. + */ + invalid_chapters = max(rounded_chapters / 256, 2U); + chapters_in_volume_index = rounded_chapters + invalid_chapters; + entries_in_volume_index = records_per_chapter * chapters_in_volume_index; + + address_span = params->list_count << params->address_bits; + params->mean_delta = address_span / entries_in_volume_index; + + /* + * Compute the expected size of a full index, then set the total memory to be 6% larger + * than that expected size. This number should be large enough that there are not many + * rebalances when the index is full. + */ + params->chapter_size_in_bits = uds_compute_delta_index_size(records_per_chapter, + params->mean_delta, + params->chapter_bits); + index_size_in_bits = params->chapter_size_in_bits * chapters_in_volume_index; + expected_index_size = index_size_in_bits / BITS_PER_BYTE; + params->memory_size = expected_index_size * 106 / 100; + + params->target_free_bytes = expected_index_size / 20; + return UDS_SUCCESS; +} + +static void uninitialize_volume_sub_index(struct volume_sub_index *sub_index) +{ + uds_free(uds_forget(sub_index->flush_chapters)); + uds_free(uds_forget(sub_index->zones)); + uds_uninitialize_delta_index(&sub_index->delta_index); +} + +void uds_free_volume_index(struct volume_index *volume_index) +{ + if (volume_index == NULL) + return; + + if (volume_index->zones != NULL) { + unsigned int zone; + + for (zone = 0; zone < volume_index->zone_count; zone++) + uds_destroy_mutex(&volume_index->zones[zone].hook_mutex); + uds_free(uds_forget(volume_index->zones)); + } + + uninitialize_volume_sub_index(&volume_index->vi_non_hook); + uninitialize_volume_sub_index(&volume_index->vi_hook); + uds_free(volume_index); +} + + +static int compute_volume_sub_index_save_bytes(const struct configuration *config, + size_t *bytes) +{ + struct sub_index_parameters params = { .address_bits = 0 }; + int result; + + result = compute_volume_sub_index_parameters(config, ¶ms); + if (result != UDS_SUCCESS) + return result; + + *bytes = (sizeof(struct sub_index_data) + params.list_count * sizeof(u64) + + uds_compute_delta_index_save_bytes(params.list_count, + params.memory_size)); + return UDS_SUCCESS; +} + +/* This function is only useful if the configuration includes sparse chapters. */ +static void split_configuration(const struct configuration *config, + struct split_config *split) +{ + u64 sample_rate, sample_records; + u64 dense_chapters, sparse_chapters; + + /* Start with copies of the base configuration. */ + split->hook_config = *config; + split->hook_geometry = *config->geometry; + split->hook_config.geometry = &split->hook_geometry; + split->non_hook_config = *config; + split->non_hook_geometry = *config->geometry; + split->non_hook_config.geometry = &split->non_hook_geometry; + + sample_rate = config->sparse_sample_rate; + sparse_chapters = config->geometry->sparse_chapters_per_volume; + dense_chapters = config->geometry->chapters_per_volume - sparse_chapters; + sample_records = config->geometry->records_per_chapter / sample_rate; + + /* Adjust the number of records indexed for each chapter. */ + split->hook_geometry.records_per_chapter = sample_records; + split->non_hook_geometry.records_per_chapter -= sample_records; + + /* Adjust the number of chapters indexed. */ + split->hook_geometry.sparse_chapters_per_volume = 0; + split->non_hook_geometry.sparse_chapters_per_volume = 0; + split->non_hook_geometry.chapters_per_volume = dense_chapters; +} + +static int compute_volume_index_save_bytes(const struct configuration *config, + size_t *bytes) +{ + size_t hook_bytes, non_hook_bytes; + struct split_config split; + int result; + + if (!uds_is_sparse_geometry(config->geometry)) + return compute_volume_sub_index_save_bytes(config, bytes); + + split_configuration(config, &split); + result = compute_volume_sub_index_save_bytes(&split.hook_config, &hook_bytes); + if (result != UDS_SUCCESS) + return result; + + result = compute_volume_sub_index_save_bytes(&split.non_hook_config, + &non_hook_bytes); + if (result != UDS_SUCCESS) + return result; + + *bytes = sizeof(struct volume_index_data) + hook_bytes + non_hook_bytes; + return UDS_SUCCESS; +} + +int uds_compute_volume_index_save_blocks(const struct configuration *config, + size_t block_size, u64 *block_count) +{ + size_t bytes; + int result; + + result = compute_volume_index_save_bytes(config, &bytes); + if (result != UDS_SUCCESS) + return result; + + bytes += sizeof(struct delta_list_save_info); + *block_count = DIV_ROUND_UP(bytes, block_size) + MAX_ZONES; + return UDS_SUCCESS; +} + +/* Flush invalid entries while walking the delta list. */ +static inline int flush_invalid_entries(struct volume_index_record *record, + struct chapter_range *flush_range, + u32 *next_chapter_to_invalidate) +{ + int result; + + result = uds_next_delta_index_entry(&record->delta_entry); + if (result != UDS_SUCCESS) + return result; + + while (!record->delta_entry.at_end) { + u32 index_chapter = uds_get_delta_entry_value(&record->delta_entry); + u32 relative_chapter = ((index_chapter - flush_range->chapter_start) & + record->sub_index->chapter_mask); + + if (likely(relative_chapter >= flush_range->chapter_count)) { + if (relative_chapter < *next_chapter_to_invalidate) + *next_chapter_to_invalidate = relative_chapter; + break; + } + + result = uds_remove_delta_index_entry(&record->delta_entry); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +/* Find the matching record, or the list offset where the record would go. */ +static int get_volume_index_entry(struct volume_index_record *record, u32 list_number, + u32 key, struct chapter_range *flush_range) +{ + struct volume_index_record other_record; + const struct volume_sub_index *sub_index = record->sub_index; + u32 next_chapter_to_invalidate = sub_index->chapter_mask; + int result; + + result = uds_start_delta_index_search(&sub_index->delta_index, list_number, 0, + &record->delta_entry); + if (result != UDS_SUCCESS) + return result; + + do { + result = flush_invalid_entries(record, flush_range, + &next_chapter_to_invalidate); + if (result != UDS_SUCCESS) + return result; + } while (!record->delta_entry.at_end && (key > record->delta_entry.key)); + + result = uds_remember_delta_index_offset(&record->delta_entry); + if (result != UDS_SUCCESS) + return result; + + /* Check any collision records for a more precise match. */ + other_record = *record; + if (!other_record.delta_entry.at_end && (key == other_record.delta_entry.key)) { + for (;;) { + u8 collision_name[UDS_RECORD_NAME_SIZE]; + + result = flush_invalid_entries(&other_record, flush_range, + &next_chapter_to_invalidate); + if (result != UDS_SUCCESS) + return result; + + if (other_record.delta_entry.at_end || + !other_record.delta_entry.is_collision) + break; + + result = uds_get_delta_entry_collision(&other_record.delta_entry, + collision_name); + if (result != UDS_SUCCESS) + return result; + + if (memcmp(collision_name, record->name, UDS_RECORD_NAME_SIZE) == 0) { + *record = other_record; + break; + } + } + } + while (!other_record.delta_entry.at_end) { + result = flush_invalid_entries(&other_record, flush_range, + &next_chapter_to_invalidate); + if (result != UDS_SUCCESS) + return result; + } + next_chapter_to_invalidate += flush_range->chapter_start; + next_chapter_to_invalidate &= sub_index->chapter_mask; + flush_range->chapter_start = next_chapter_to_invalidate; + flush_range->chapter_count = 0; + return UDS_SUCCESS; +} + +static int get_volume_sub_index_record(struct volume_sub_index *sub_index, + const struct uds_record_name *name, + struct volume_index_record *record) +{ + int result; + const struct volume_sub_index_zone *volume_index_zone; + u32 address = extract_address(sub_index, name); + u32 delta_list_number = extract_dlist_num(sub_index, name); + u64 flush_chapter = sub_index->flush_chapters[delta_list_number]; + + record->sub_index = sub_index; + record->mutex = NULL; + record->name = name; + record->zone_number = delta_list_number / sub_index->delta_index.lists_per_zone; + volume_index_zone = get_zone_for_record(record); + + if (flush_chapter < volume_index_zone->virtual_chapter_low) { + struct chapter_range range; + u64 flush_count = volume_index_zone->virtual_chapter_low - flush_chapter; + + range.chapter_start = convert_virtual_to_index(sub_index, flush_chapter); + range.chapter_count = (flush_count > sub_index->chapter_mask ? + sub_index->chapter_mask + 1 : + flush_count); + result = get_volume_index_entry(record, delta_list_number, address, + &range); + flush_chapter = convert_index_to_virtual(record, range.chapter_start); + if (flush_chapter > volume_index_zone->virtual_chapter_high) + flush_chapter = volume_index_zone->virtual_chapter_high; + sub_index->flush_chapters[delta_list_number] = flush_chapter; + } else { + result = uds_get_delta_index_entry(&sub_index->delta_index, + delta_list_number, address, + name->name, &record->delta_entry); + } + + if (result != UDS_SUCCESS) + return result; + + record->is_found = + (!record->delta_entry.at_end && (record->delta_entry.key == address)); + if (record->is_found) { + u32 index_chapter = uds_get_delta_entry_value(&record->delta_entry); + + record->virtual_chapter = convert_index_to_virtual(record, index_chapter); + } + + record->is_collision = record->delta_entry.is_collision; + return UDS_SUCCESS; +} + +int uds_get_volume_index_record(struct volume_index *volume_index, + const struct uds_record_name *name, + struct volume_index_record *record) +{ + int result; + + if (uds_is_volume_index_sample(volume_index, name)) { + /* + * Other threads cannot be allowed to call uds_lookup_volume_index_name() while + * this thread is finding the volume index record. Due to the lazy LRU flushing of + * the volume index, uds_get_volume_index_record() is not a read-only operation. + */ + unsigned int zone = + get_volume_sub_index_zone(&volume_index->vi_hook, name); + struct mutex *mutex = &volume_index->zones[zone].hook_mutex; + + uds_lock_mutex(mutex); + result = get_volume_sub_index_record(&volume_index->vi_hook, name, + record); + uds_unlock_mutex(mutex); + /* Remember the mutex so that other operations on the index record can use it. */ + record->mutex = mutex; + } else { + result = get_volume_sub_index_record(&volume_index->vi_non_hook, name, + record); + } + + return result; +} + +int uds_put_volume_index_record(struct volume_index_record *record, u64 virtual_chapter) +{ + int result; + u32 address; + const struct volume_sub_index *sub_index = record->sub_index; + + if (!is_virtual_chapter_indexed(record, virtual_chapter)) { + u64 low = get_zone_for_record(record)->virtual_chapter_low; + u64 high = get_zone_for_record(record)->virtual_chapter_high; + + return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot put record into chapter number %llu that is out of the valid range %llu to %llu", + (unsigned long long) virtual_chapter, + (unsigned long long) low, + (unsigned long long) high); + } + address = extract_address(sub_index, record->name); + if (unlikely(record->mutex != NULL)) + uds_lock_mutex(record->mutex); + result = uds_put_delta_index_entry(&record->delta_entry, address, + convert_virtual_to_index(sub_index, + virtual_chapter), + record->is_found ? record->name->name : NULL); + if (unlikely(record->mutex != NULL)) + uds_unlock_mutex(record->mutex); + switch (result) { + case UDS_SUCCESS: + record->virtual_chapter = virtual_chapter; + record->is_collision = record->delta_entry.is_collision; + record->is_found = true; + break; + case UDS_OVERFLOW: + uds_log_ratelimit(uds_log_warning_strerror, UDS_OVERFLOW, + "Volume index entry dropped due to overflow condition"); + uds_log_delta_index_entry(&record->delta_entry); + break; + default: + break; + } + + return result; +} + +int uds_remove_volume_index_record(struct volume_index_record *record) +{ + int result; + + if (!record->is_found) + return uds_log_warning_strerror(UDS_BAD_STATE, + "illegal operation on new record"); + + /* Mark the record so that it cannot be used again */ + record->is_found = false; + if (unlikely(record->mutex != NULL)) + uds_lock_mutex(record->mutex); + result = uds_remove_delta_index_entry(&record->delta_entry); + if (unlikely(record->mutex != NULL)) + uds_unlock_mutex(record->mutex); + return result; +} + +static void set_volume_sub_index_zone_open_chapter(struct volume_sub_index *sub_index, + unsigned int zone_number, + u64 virtual_chapter) +{ + u64 used_bits = 0; + struct volume_sub_index_zone *zone = &sub_index->zones[zone_number]; + struct delta_zone *delta_zone; + u32 i; + + zone->virtual_chapter_low = (virtual_chapter >= sub_index->chapter_count ? + virtual_chapter - sub_index->chapter_count + 1 : + 0); + zone->virtual_chapter_high = virtual_chapter; + + /* Check to see if the new zone data is too large. */ + delta_zone = &sub_index->delta_index.delta_zones[zone_number]; + for (i = 1; i <= delta_zone->list_count; i++) + used_bits += delta_zone->delta_lists[i].size; + + if (used_bits > sub_index->max_zone_bits) { + /* Expire enough chapters to free the desired space. */ + u64 expire_count = + 1 + (used_bits - sub_index->max_zone_bits) / sub_index->chapter_zone_bits; + + if (expire_count == 1) { + uds_log_ratelimit(uds_log_info, + "zone %u: At chapter %llu, expiring chapter %llu early", + zone_number, + (unsigned long long) virtual_chapter, + (unsigned long long) zone->virtual_chapter_low); + zone->early_flushes++; + zone->virtual_chapter_low++; + } else { + u64 first_expired = zone->virtual_chapter_low; + + if (first_expired + expire_count < zone->virtual_chapter_high) { + zone->early_flushes += expire_count; + zone->virtual_chapter_low += expire_count; + } else { + zone->early_flushes += + zone->virtual_chapter_high - zone->virtual_chapter_low; + zone->virtual_chapter_low = zone->virtual_chapter_high; + } + uds_log_ratelimit(uds_log_info, + "zone %u: At chapter %llu, expiring chapters %llu to %llu early", + zone_number, + (unsigned long long) virtual_chapter, + (unsigned long long) first_expired, + (unsigned long long) zone->virtual_chapter_low - 1); + } + } +} + +void uds_set_volume_index_zone_open_chapter(struct volume_index *volume_index, + unsigned int zone_number, + u64 virtual_chapter) +{ + struct mutex *mutex = &volume_index->zones[zone_number].hook_mutex; + + set_volume_sub_index_zone_open_chapter(&volume_index->vi_non_hook, zone_number, + virtual_chapter); + + /* + * Other threads cannot be allowed to call uds_lookup_volume_index_name() while the open + * chapter number is changing. + */ + if (has_sparse(volume_index)) { + uds_lock_mutex(mutex); + set_volume_sub_index_zone_open_chapter(&volume_index->vi_hook, + zone_number, virtual_chapter); + uds_unlock_mutex(mutex); + } +} + +/* + * Set the newest open chapter number for the index, while also advancing the oldest valid chapter + * number. + */ +void uds_set_volume_index_open_chapter(struct volume_index *volume_index, + u64 virtual_chapter) +{ + unsigned int zone; + + for (zone = 0; zone < volume_index->zone_count; zone++) + uds_set_volume_index_zone_open_chapter(volume_index, zone, virtual_chapter); +} + +int uds_set_volume_index_record_chapter(struct volume_index_record *record, + u64 virtual_chapter) +{ + const struct volume_sub_index *sub_index = record->sub_index; + int result; + + if (!record->is_found) + return uds_log_warning_strerror(UDS_BAD_STATE, + "illegal operation on new record"); + + if (!is_virtual_chapter_indexed(record, virtual_chapter)) { + u64 low = get_zone_for_record(record)->virtual_chapter_low; + u64 high = get_zone_for_record(record)->virtual_chapter_high; + + return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot set chapter number %llu that is out of the valid range %llu to %llu", + (unsigned long long) virtual_chapter, + (unsigned long long) low, + (unsigned long long) high); + } + + if (unlikely(record->mutex != NULL)) + uds_lock_mutex(record->mutex); + result = uds_set_delta_entry_value(&record->delta_entry, + convert_virtual_to_index(sub_index, + virtual_chapter)); + if (unlikely(record->mutex != NULL)) + uds_unlock_mutex(record->mutex); + if (result != UDS_SUCCESS) + return result; + + record->virtual_chapter = virtual_chapter; + return UDS_SUCCESS; +} + +static u64 lookup_volume_sub_index_name(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + int result; + u32 address = extract_address(sub_index, name); + u32 delta_list_number = extract_dlist_num(sub_index, name); + unsigned int zone_number = get_volume_sub_index_zone(sub_index, name); + const struct volume_sub_index_zone *zone = &sub_index->zones[zone_number]; + u64 virtual_chapter; + u32 index_chapter; + u32 rolling_chapter; + struct delta_index_entry delta_entry; + + result = uds_get_delta_index_entry(&sub_index->delta_index, delta_list_number, + address, name->name, &delta_entry); + if (result != UDS_SUCCESS) + return NO_CHAPTER; + + if (delta_entry.at_end || (delta_entry.key != address)) + return NO_CHAPTER; + + index_chapter = uds_get_delta_entry_value(&delta_entry); + rolling_chapter = (index_chapter - zone->virtual_chapter_low) & sub_index->chapter_mask; + + virtual_chapter = zone->virtual_chapter_low + rolling_chapter; + if (virtual_chapter > zone->virtual_chapter_high) + return NO_CHAPTER; + + return virtual_chapter; +} + +/* Do a read-only lookup of the record name for sparse cache management. */ +u64 uds_lookup_volume_index_name(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + unsigned int zone_number = uds_get_volume_index_zone(volume_index, name); + struct mutex *mutex = &volume_index->zones[zone_number].hook_mutex; + u64 virtual_chapter; + + if (!uds_is_volume_index_sample(volume_index, name)) + return NO_CHAPTER; + + uds_lock_mutex(mutex); + virtual_chapter = lookup_volume_sub_index_name(&volume_index->vi_hook, name); + uds_unlock_mutex(mutex); + + return virtual_chapter; +} + +static void abort_restoring_volume_sub_index(struct volume_sub_index *sub_index) +{ + uds_reset_delta_index(&sub_index->delta_index); +} + +static void abort_restoring_volume_index(struct volume_index *volume_index) +{ + abort_restoring_volume_sub_index(&volume_index->vi_non_hook); + if (has_sparse(volume_index)) + abort_restoring_volume_sub_index(&volume_index->vi_hook); +} + +static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, + struct buffered_reader **readers, + unsigned int reader_count) +{ + unsigned int z; + int result; + u64 virtual_chapter_low = 0, virtual_chapter_high = 0; + unsigned int i; + + for (i = 0; i < reader_count; i++) { + struct sub_index_data header; + u8 buffer[sizeof(struct sub_index_data)]; + size_t offset = 0; + u32 j; + + result = uds_read_from_buffered_reader(readers[i], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read volume index header"); + } + + memcpy(&header.magic, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + decode_u64_le(buffer, &offset, &header.volume_nonce); + decode_u64_le(buffer, &offset, &header.virtual_chapter_low); + decode_u64_le(buffer, &offset, &header.virtual_chapter_high); + decode_u32_le(buffer, &offset, &header.first_list); + decode_u32_le(buffer, &offset, &header.list_count); + + result = ASSERT(offset = sizeof(buffer), + "%zu bytes decoded of %zu expected", offset, + sizeof(buffer)); + if (result != UDS_SUCCESS) + result = UDS_CORRUPT_DATA; + + if (memcmp(header.magic, MAGIC_START_5, MAGIC_SIZE) != 0) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "volume index file had bad magic number"); + } + + if (sub_index->volume_nonce == 0) { + sub_index->volume_nonce = header.volume_nonce; + } else if (header.volume_nonce != sub_index->volume_nonce) { + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "volume index volume nonce incorrect"); + } + + if (i == 0) { + virtual_chapter_low = header.virtual_chapter_low; + virtual_chapter_high = header.virtual_chapter_high; + } else if (virtual_chapter_high != header.virtual_chapter_high) { + u64 low = header.virtual_chapter_low; + u64 high = header.virtual_chapter_high; + + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "Inconsistent volume index zone files: Chapter range is [%llu,%llu], chapter range %d is [%llu,%llu]", + (unsigned long long) virtual_chapter_low, + (unsigned long long) virtual_chapter_high, + i, (unsigned long long) low, + (unsigned long long) high); + } else if (virtual_chapter_low < header.virtual_chapter_low) { + virtual_chapter_low = header.virtual_chapter_low; + } + + for (j = 0; j < header.list_count; j++) { + u8 decoded[sizeof(u64)]; + + result = uds_read_from_buffered_reader(readers[i], decoded, + sizeof(u64)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read volume index flush ranges"); + } + + sub_index->flush_chapters[header.first_list + j] = + get_unaligned_le64(decoded); + } + } + + for (z = 0; z < sub_index->zone_count; z++) { + memset(&sub_index->zones[z], 0, sizeof(struct volume_sub_index_zone)); + sub_index->zones[z].virtual_chapter_low = virtual_chapter_low; + sub_index->zones[z].virtual_chapter_high = virtual_chapter_high; + } + + result = uds_start_restoring_delta_index(&sub_index->delta_index, readers, + reader_count); + if (result != UDS_SUCCESS) + return uds_log_warning_strerror(result, "restoring delta index failed"); + + return UDS_SUCCESS; +} + +static int start_restoring_volume_index(struct volume_index *volume_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + unsigned int i; + int result; + + if (!has_sparse(volume_index)) { + return start_restoring_volume_sub_index(&volume_index->vi_non_hook, + buffered_readers, reader_count); + } + + for (i = 0; i < reader_count; i++) { + struct volume_index_data header; + u8 buffer[sizeof(struct volume_index_data)]; + size_t offset = 0; + + result = uds_read_from_buffered_reader(buffered_readers[i], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to read volume index header"); + } + + memcpy(&header.magic, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + decode_u32_le(buffer, &offset, &header.sparse_sample_rate); + + result = ASSERT(offset == sizeof(buffer), + "%zu bytes decoded of %zu expected", offset, + sizeof(buffer)); + if (result != UDS_SUCCESS) + result = UDS_CORRUPT_DATA; + + if (memcmp(header.magic, MAGIC_START_6, MAGIC_SIZE) != 0) + return uds_log_warning_strerror(UDS_CORRUPT_DATA, + "volume index file had bad magic number"); + + if (i == 0) { + volume_index->sparse_sample_rate = header.sparse_sample_rate; + } else if (volume_index->sparse_sample_rate != header.sparse_sample_rate) { + uds_log_warning_strerror(UDS_CORRUPT_DATA, + "Inconsistent sparse sample rate in delta index zone files: %u vs. %u", + volume_index->sparse_sample_rate, + header.sparse_sample_rate); + return UDS_CORRUPT_DATA; + } + } + + result = start_restoring_volume_sub_index(&volume_index->vi_non_hook, + buffered_readers, reader_count); + if (result != UDS_SUCCESS) + return result; + + return start_restoring_volume_sub_index(&volume_index->vi_hook, buffered_readers, + reader_count); +} + +static int finish_restoring_volume_sub_index(struct volume_sub_index *sub_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + return uds_finish_restoring_delta_index(&sub_index->delta_index, + buffered_readers, reader_count); +} + +static int finish_restoring_volume_index(struct volume_index *volume_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + + result = finish_restoring_volume_sub_index(&volume_index->vi_non_hook, + buffered_readers, reader_count); + if ((result == UDS_SUCCESS) && has_sparse(volume_index)) { + result = finish_restoring_volume_sub_index(&volume_index->vi_hook, + buffered_readers, + reader_count); + } + + return result; +} + +int uds_load_volume_index(struct volume_index *volume_index, + struct buffered_reader **readers, unsigned int reader_count) +{ + int result; + + /* Start by reading the header section of the stream. */ + result = start_restoring_volume_index(volume_index, readers, reader_count); + if (result != UDS_SUCCESS) + return result; + + result = finish_restoring_volume_index(volume_index, readers, reader_count); + if (result != UDS_SUCCESS) { + abort_restoring_volume_index(volume_index); + return result; + } + + /* Check the final guard lists to make sure there is no extra data. */ + result = uds_check_guard_delta_lists(readers, reader_count); + if (result != UDS_SUCCESS) + abort_restoring_volume_index(volume_index); + + return result; +} + +static int start_saving_volume_sub_index(const struct volume_sub_index *sub_index, + unsigned int zone_number, + struct buffered_writer *buffered_writer) +{ + int result; + struct volume_sub_index_zone *volume_index_zone = &sub_index->zones[zone_number]; + u32 first_list = sub_index->delta_index.delta_zones[zone_number].first_list; + u32 list_count = sub_index->delta_index.delta_zones[zone_number].list_count; + u8 buffer[sizeof(struct sub_index_data)]; + size_t offset = 0; + u32 i; + + memcpy(buffer, MAGIC_START_5, MAGIC_SIZE); + offset += MAGIC_SIZE; + encode_u64_le(buffer, &offset, sub_index->volume_nonce); + encode_u64_le(buffer, &offset, volume_index_zone->virtual_chapter_low); + encode_u64_le(buffer, &offset, volume_index_zone->virtual_chapter_high); + encode_u32_le(buffer, &offset, first_list); + encode_u32_le(buffer, &offset, list_count); + + result = ASSERT(offset == sizeof(struct sub_index_data), + "%zu bytes of config written, of %zu expected", offset, + sizeof(struct sub_index_data)); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); + if (result != UDS_SUCCESS) + return uds_log_warning_strerror(result, + "failed to write volume index header"); + + for (i = 0; i < list_count; i++) { + u8 encoded[sizeof(u64)]; + + put_unaligned_le64(sub_index->flush_chapters[first_list + i], &encoded); + result = uds_write_to_buffered_writer(buffered_writer, encoded, + sizeof(u64)); + if (result != UDS_SUCCESS) { + return uds_log_warning_strerror(result, + "failed to write volume index flush ranges"); + } + } + + return uds_start_saving_delta_index(&sub_index->delta_index, zone_number, + buffered_writer); +} + +static int start_saving_volume_index(const struct volume_index *volume_index, + unsigned int zone_number, + struct buffered_writer *writer) +{ + u8 buffer[sizeof(struct volume_index_data)]; + size_t offset = 0; + int result; + + if (!has_sparse(volume_index)) { + return start_saving_volume_sub_index(&volume_index->vi_non_hook, + zone_number, writer); + } + + memcpy(buffer, MAGIC_START_6, MAGIC_SIZE); + offset += MAGIC_SIZE; + encode_u32_le(buffer, &offset, volume_index->sparse_sample_rate); + result = ASSERT(offset == sizeof(struct volume_index_data), + "%zu bytes of header written, of %zu expected", offset, + sizeof(struct volume_index_data)); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(writer, buffer, offset); + if (result != UDS_SUCCESS) { + uds_log_warning_strerror(result, "failed to write volume index header"); + return result; + } + + result = start_saving_volume_sub_index(&volume_index->vi_non_hook, zone_number, + writer); + if (result != UDS_SUCCESS) + return result; + + return start_saving_volume_sub_index(&volume_index->vi_hook, zone_number, + writer); +} + +static int finish_saving_volume_sub_index(const struct volume_sub_index *sub_index, + unsigned int zone_number) +{ + return uds_finish_saving_delta_index(&sub_index->delta_index, zone_number); +} + +static int finish_saving_volume_index(const struct volume_index *volume_index, + unsigned int zone_number) +{ + int result; + + result = finish_saving_volume_sub_index(&volume_index->vi_non_hook, zone_number); + if ((result == UDS_SUCCESS) && has_sparse(volume_index)) + result = finish_saving_volume_sub_index(&volume_index->vi_hook, zone_number); + return result; +} + +int uds_save_volume_index(struct volume_index *volume_index, + struct buffered_writer **writers, unsigned int writer_count) +{ + int result = UDS_SUCCESS; + unsigned int zone; + + for (zone = 0; zone < writer_count; zone++) { + result = start_saving_volume_index(volume_index, zone, writers[zone]); + if (result != UDS_SUCCESS) + break; + + result = finish_saving_volume_index(volume_index, zone); + if (result != UDS_SUCCESS) + break; + + result = uds_write_guard_delta_list(writers[zone]); + if (result != UDS_SUCCESS) + break; + + result = uds_flush_buffered_writer(writers[zone]); + if (result != UDS_SUCCESS) + break; + } + + return result; +} + +static void get_volume_sub_index_stats(const struct volume_sub_index *sub_index, + struct volume_index_stats *stats) +{ + struct delta_index_stats dis; + unsigned int z; + + uds_get_delta_index_stats(&sub_index->delta_index, &dis); + stats->rebalance_time = dis.rebalance_time; + stats->rebalance_count = dis.rebalance_count; + stats->record_count = dis.record_count; + stats->collision_count = dis.collision_count; + stats->discard_count = dis.discard_count; + stats->overflow_count = dis.overflow_count; + stats->delta_lists = dis.list_count; + stats->early_flushes = 0; + for (z = 0; z < sub_index->zone_count; z++) + stats->early_flushes += sub_index->zones[z].early_flushes; +} + +void uds_get_volume_index_stats(const struct volume_index *volume_index, + struct volume_index_stats *stats) +{ + struct volume_index_stats sparse_stats; + + get_volume_sub_index_stats(&volume_index->vi_non_hook, stats); + if (!has_sparse(volume_index)) + return; + + get_volume_sub_index_stats(&volume_index->vi_hook, &sparse_stats); + stats->rebalance_time += sparse_stats.rebalance_time; + stats->rebalance_count += sparse_stats.rebalance_count; + stats->record_count += sparse_stats.record_count; + stats->collision_count += sparse_stats.collision_count; + stats->discard_count += sparse_stats.discard_count; + stats->overflow_count += sparse_stats.overflow_count; + stats->delta_lists += sparse_stats.delta_lists; + stats->early_flushes += sparse_stats.early_flushes; +} + +static int initialize_volume_sub_index(const struct configuration *config, + u64 volume_nonce, u8 tag, + struct volume_sub_index *sub_index) +{ + struct sub_index_parameters params = { .address_bits = 0 }; + unsigned int zone_count = config->zone_count; + u64 available_bytes = 0; + unsigned int z; + int result; + + result = compute_volume_sub_index_parameters(config, ¶ms); + if (result != UDS_SUCCESS) + return result; + + sub_index->address_bits = params.address_bits; + sub_index->address_mask = (1u << params.address_bits) - 1; + sub_index->chapter_bits = params.chapter_bits; + sub_index->chapter_mask = (1u << params.chapter_bits) - 1; + sub_index->chapter_count = params.chapter_count; + sub_index->list_count = params.list_count; + sub_index->zone_count = zone_count; + sub_index->chapter_zone_bits = params.chapter_size_in_bits / zone_count; + sub_index->volume_nonce = volume_nonce; + + result = uds_initialize_delta_index(&sub_index->delta_index, zone_count, + params.list_count, params.mean_delta, + params.chapter_bits, params.memory_size, + tag); + if (result != UDS_SUCCESS) + return result; + + for (z = 0; z < sub_index->delta_index.zone_count; z++) + available_bytes += sub_index->delta_index.delta_zones[z].size; + available_bytes -= params.target_free_bytes; + sub_index->max_zone_bits = (available_bytes * BITS_PER_BYTE) / zone_count; + sub_index->memory_size = (sub_index->delta_index.memory_size + + sizeof(struct volume_sub_index) + + (params.list_count * sizeof(u64)) + + (zone_count * sizeof(struct volume_sub_index_zone))); + + /* The following arrays are initialized to all zeros. */ + result = uds_allocate(params.list_count, u64, "first chapter to flush", + &sub_index->flush_chapters); + if (result != UDS_SUCCESS) + return result; + + return uds_allocate(zone_count, struct volume_sub_index_zone, + "volume index zones", &sub_index->zones); +} + +int uds_make_volume_index(const struct configuration *config, u64 volume_nonce, + struct volume_index **volume_index_ptr) +{ + struct split_config split; + unsigned int zone; + struct volume_index *volume_index; + int result; + + result = uds_allocate(1, struct volume_index, "volume index", &volume_index); + if (result != UDS_SUCCESS) + return result; + + volume_index->zone_count = config->zone_count; + + if (!uds_is_sparse_geometry(config->geometry)) { + result = initialize_volume_sub_index(config, volume_nonce, 'm', + &volume_index->vi_non_hook); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return result; + } + + volume_index->memory_size = volume_index->vi_non_hook.memory_size; + *volume_index_ptr = volume_index; + return UDS_SUCCESS; + } + + volume_index->sparse_sample_rate = config->sparse_sample_rate; + + result = uds_allocate(config->zone_count, struct volume_index_zone, + "volume index zones", &volume_index->zones); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return result; + } + + for (zone = 0; zone < config->zone_count; zone++) { + result = uds_init_mutex(&volume_index->zones[zone].hook_mutex); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return result; + } + } + + split_configuration(config, &split); + result = initialize_volume_sub_index(&split.non_hook_config, volume_nonce, 'd', + &volume_index->vi_non_hook); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return uds_log_error_strerror(result, + "Error creating non hook volume index"); + } + + result = initialize_volume_sub_index(&split.hook_config, volume_nonce, 's', + &volume_index->vi_hook); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return uds_log_error_strerror(result, + "Error creating hook volume index"); + } + + volume_index->memory_size = + volume_index->vi_non_hook.memory_size + volume_index->vi_hook.memory_size; + *volume_index_ptr = volume_index; + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/volume-index.h b/drivers/md/dm-vdo/volume-index.h new file mode 100644 index 000000000000..7fbe11fe3284 --- /dev/null +++ b/drivers/md/dm-vdo/volume-index.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_VOLUME_INDEX_H +#define UDS_VOLUME_INDEX_H + +#include + +#include "config.h" +#include "delta-index.h" +#include "uds.h" +#include "uds-threads.h" + +/* + * The volume index is the primary top-level index for UDS. It contains records which map a record + * name to the chapter where a record with that name is stored. This mapping can definitively say + * when no record exists. However, because we only use a subset of the name for this index, it + * cannot definitively say that a record for the entry does exist. It can only say that if a record + * exists, it will be in a particular chapter. The request can then be dispatched to that chapter + * for further processing. + * + * If the volume_index_record does not actually match the record name, the index can store a more + * specific collision record to disambiguate the new entry from the existing one. Index entries are + * managed with volume_index_record structures. + */ + +static const u64 NO_CHAPTER = U64_MAX; + +struct volume_index_stats { + /* Nanoseconds spent rebalancing */ + ktime_t rebalance_time; + /* Number of memory rebalances */ + u32 rebalance_count; + /* The number of records in the index */ + u64 record_count; + /* The number of collision records */ + u64 collision_count; + /* The number of records removed */ + u64 discard_count; + /* The number of UDS_OVERFLOWs detected */ + u64 overflow_count; + /* The number of delta lists */ + u32 delta_lists; + /* Number of early flushes */ + u64 early_flushes; +}; + +struct volume_sub_index_zone { + u64 virtual_chapter_low; + u64 virtual_chapter_high; + u64 early_flushes; +} __aligned(L1_CACHE_BYTES); + +struct volume_sub_index { + /* The delta index */ + struct delta_index delta_index; + /* The first chapter to be flushed in each zone */ + u64 *flush_chapters; + /* The zones */ + struct volume_sub_index_zone *zones; + /* The volume nonce */ + u64 volume_nonce; + /* Expected size of a chapter (per zone) */ + u64 chapter_zone_bits; + /* Maximum size of the index (per zone) */ + u64 max_zone_bits; + /* The number of bits in address mask */ + u8 address_bits; + /* Mask to get address within delta list */ + u32 address_mask; + /* The number of bits in chapter number */ + u8 chapter_bits; + /* The largest storable chapter number */ + u32 chapter_mask; + /* The number of chapters used */ + u32 chapter_count; + /* The number of delta lists */ + u32 list_count; + /* The number of zones */ + unsigned int zone_count; + /* The amount of memory allocated */ + u64 memory_size; +}; + +struct volume_index_zone { + /* Protects the sampled index in this zone */ + struct mutex hook_mutex; +} __aligned(L1_CACHE_BYTES); + +struct volume_index { + u32 sparse_sample_rate; + unsigned int zone_count; + u64 memory_size; + struct volume_sub_index vi_non_hook; + struct volume_sub_index vi_hook; + struct volume_index_zone *zones; +}; + +/* + * The volume_index_record structure is used to facilitate processing of a record name. A client + * first calls uds_get_volume_index_record() to find the volume index record for a record name. The + * fields of the record can then be examined to determine the state of the record. + * + * If is_found is false, then the index did not find an entry for the record name. Calling + * uds_put_volume_index_record() will insert a new entry for that name at the proper place. + * + * If is_found is true, then we did find an entry for the record name, and the virtual_chapter and + * is_collision fields reflect the entry found. Subsequently, a call to + * uds_remove_volume_index_record() will remove the entry, a call to + * uds_set_volume_index_record_chapter() will update the existing entry, and a call to + * uds_put_volume_index_record() will insert a new collision record after the existing entry. + */ +struct volume_index_record { + /* Public fields */ + + /* Chapter where the record info is found */ + u64 virtual_chapter; + /* This record is a collision */ + bool is_collision; + /* This record is the requested record */ + bool is_found; + + /* Private fields */ + + /* Zone that contains this name */ + unsigned int zone_number; + /* The volume index */ + struct volume_sub_index *sub_index; + /* Mutex for accessing this delta index entry in the hook index */ + struct mutex *mutex; + /* The record name to which this record refers */ + const struct uds_record_name *name; + /* The delta index entry for this record */ + struct delta_index_entry delta_entry; +}; + +int __must_check uds_make_volume_index(const struct configuration *config, + u64 volume_nonce, + struct volume_index **volume_index); + +void uds_free_volume_index(struct volume_index *volume_index); + +int __must_check uds_compute_volume_index_save_blocks(const struct configuration *config, + size_t block_size, + u64 *block_count); + +unsigned int __must_check uds_get_volume_index_zone(const struct volume_index *volume_index, + const struct uds_record_name *name); + +bool __must_check uds_is_volume_index_sample(const struct volume_index *volume_index, + const struct uds_record_name *name); + +/* + * This function is only used to manage sparse cache membership. Most requests should use + * uds_get_volume_index_record() to look up index records instead. + */ +u64 __must_check uds_lookup_volume_index_name(const struct volume_index *volume_index, + const struct uds_record_name *name); + +int __must_check uds_get_volume_index_record(struct volume_index *volume_index, + const struct uds_record_name *name, + struct volume_index_record *record); + +int __must_check uds_put_volume_index_record(struct volume_index_record *record, + u64 virtual_chapter); + +int __must_check uds_remove_volume_index_record(struct volume_index_record *record); + +int __must_check uds_set_volume_index_record_chapter(struct volume_index_record *record, + u64 virtual_chapter); + +void uds_set_volume_index_open_chapter(struct volume_index *volume_index, + u64 virtual_chapter); + +void uds_set_volume_index_zone_open_chapter(struct volume_index *volume_index, + unsigned int zone_number, + u64 virtual_chapter); + +int __must_check uds_load_volume_index(struct volume_index *volume_index, + struct buffered_reader **readers, + unsigned int reader_count); + +int __must_check uds_save_volume_index(struct volume_index *volume_index, + struct buffered_writer **writers, + unsigned int writer_count); + +void uds_get_volume_index_stats(const struct volume_index *volume_index, + struct volume_index_stats *stats); + +#endif /* UDS_VOLUME_INDEX_H */ From 6afc7bcaa40d7ec6bf37483b35e68615cfaad524 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:54:29 -0500 Subject: [PATCH 0561/1038] dm vdo: implement the open chapter and chapter indexes Deduplication records are stored in groups called chapters. New records are collected in a structure called the open chapter, which is optimized for adding, removing, and sorting records. When a chapter fills, it is packed into a read-only structure called a closed chapter, which is optimized for searching and reading. The closed chapter includes a delta index, called the chapter index, which maps each record name to the record page containing the record and allows the index to read at most one record page when looking up a record. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/chapter-index.c | 291 ++++++++++++++++++++ drivers/md/dm-vdo/chapter-index.h | 63 +++++ drivers/md/dm-vdo/open-chapter.c | 427 ++++++++++++++++++++++++++++++ drivers/md/dm-vdo/open-chapter.h | 79 ++++++ 4 files changed, 860 insertions(+) create mode 100644 drivers/md/dm-vdo/chapter-index.c create mode 100644 drivers/md/dm-vdo/chapter-index.h create mode 100644 drivers/md/dm-vdo/open-chapter.c create mode 100644 drivers/md/dm-vdo/open-chapter.h diff --git a/drivers/md/dm-vdo/chapter-index.c b/drivers/md/dm-vdo/chapter-index.c new file mode 100644 index 000000000000..27a2d95a2bef --- /dev/null +++ b/drivers/md/dm-vdo/chapter-index.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "chapter-index.h" + +#include "errors.h" +#include "hash-utils.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "uds.h" + +int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, + const struct geometry *geometry, u64 volume_nonce) +{ + int result; + size_t memory_size; + struct open_chapter_index *index; + + result = uds_allocate(1, struct open_chapter_index, "open chapter index", + &index); + if (result != UDS_SUCCESS) + return result; + + /* + * The delta index will rebalance delta lists when memory gets tight, + * so give the chapter index one extra page. + */ + memory_size = ((geometry->index_pages_per_chapter + 1) * geometry->bytes_per_page); + index->geometry = geometry; + index->volume_nonce = volume_nonce; + result = uds_initialize_delta_index(&index->delta_index, 1, + geometry->delta_lists_per_chapter, + geometry->chapter_mean_delta, + geometry->chapter_payload_bits, + memory_size, 'm'); + if (result != UDS_SUCCESS) { + uds_free(index); + return result; + } + + index->memory_size = index->delta_index.memory_size + sizeof(struct open_chapter_index); + *chapter_index = index; + return UDS_SUCCESS; +} + +void uds_free_open_chapter_index(struct open_chapter_index *chapter_index) +{ + if (chapter_index == NULL) + return; + + uds_uninitialize_delta_index(&chapter_index->delta_index); + uds_free(chapter_index); +} + +/* Re-initialize an open chapter index for a new chapter. */ +void uds_empty_open_chapter_index(struct open_chapter_index *chapter_index, + u64 virtual_chapter_number) +{ + uds_reset_delta_index(&chapter_index->delta_index); + chapter_index->virtual_chapter_number = virtual_chapter_number; +} + +static inline bool was_entry_found(const struct delta_index_entry *entry, u32 address) +{ + return (!entry->at_end) && (entry->key == address); +} + +/* Associate a record name with the record page containing its metadata. */ +int uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, + const struct uds_record_name *name, + u32 page_number) +{ + int result; + struct delta_index_entry entry; + u32 address; + u32 list_number; + const u8 *found_name; + bool found; + const struct geometry *geometry = chapter_index->geometry; + u64 chapter_number = chapter_index->virtual_chapter_number; + u32 record_pages = geometry->record_pages_per_chapter; + + result = ASSERT(page_number < record_pages, + "Page number within chapter (%u) exceeds the maximum value %u", + page_number, record_pages); + if (result != UDS_SUCCESS) + return UDS_INVALID_ARGUMENT; + + address = uds_hash_to_chapter_delta_address(name, geometry); + list_number = uds_hash_to_chapter_delta_list(name, geometry); + result = uds_get_delta_index_entry(&chapter_index->delta_index, list_number, + address, name->name, &entry); + if (result != UDS_SUCCESS) + return result; + + found = was_entry_found(&entry, address); + result = ASSERT(!(found && entry.is_collision), + "Chunk appears more than once in chapter %llu", + (unsigned long long) chapter_number); + if (result != UDS_SUCCESS) + return UDS_BAD_STATE; + + found_name = (found ? name->name : NULL); + return uds_put_delta_index_entry(&entry, address, page_number, found_name); +} + +/* + * Pack a section of an open chapter index into a chapter index page. A range of delta lists + * (starting with a specified list index) is copied from the open chapter index into a memory page. + * The number of lists copied onto the page is returned to the caller on success. + * + * @chapter_index: The open chapter index + * @memory: The memory page to use + * @first_list: The first delta list number to be copied + * @last_page: If true, this is the last page of the chapter index and all the remaining lists must + * be packed onto this page + * @lists_packed: The number of delta lists that were packed onto this page + */ +int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, + u8 *memory, u32 first_list, bool last_page, + u32 *lists_packed) +{ + int result; + struct delta_index *delta_index = &chapter_index->delta_index; + struct delta_index_stats stats; + u64 nonce = chapter_index->volume_nonce; + u64 chapter_number = chapter_index->virtual_chapter_number; + const struct geometry *geometry = chapter_index->geometry; + u32 list_count = geometry->delta_lists_per_chapter; + unsigned int removals = 0; + struct delta_index_entry entry; + u32 next_list; + s32 list_number; + + for (;;) { + result = uds_pack_delta_index_page(delta_index, nonce, memory, + geometry->bytes_per_page, + chapter_number, first_list, + lists_packed); + if (result != UDS_SUCCESS) + return result; + + if ((first_list + *lists_packed) == list_count) { + /* All lists are packed. */ + break; + } else if (*lists_packed == 0) { + /* + * The next delta list does not fit on a page. This delta list will be + * removed. + */ + } else if (last_page) { + /* + * This is the last page and there are lists left unpacked, but all of the + * remaining lists must fit on the page. Find a list that contains entries + * and remove the entire list. Try the first list that does not fit. If it + * is empty, we will select the last list that already fits and has any + * entries. + */ + } else { + /* This page is done. */ + break; + } + + if (removals == 0) { + uds_get_delta_index_stats(delta_index, &stats); + uds_log_warning("The chapter index for chapter %llu contains %llu entries with %llu collisions", + (unsigned long long) chapter_number, + (unsigned long long) stats.record_count, + (unsigned long long) stats.collision_count); + } + + list_number = *lists_packed; + do { + if (list_number < 0) + return UDS_OVERFLOW; + + next_list = first_list + list_number--, + result = uds_start_delta_index_search(delta_index, next_list, 0, + &entry); + if (result != UDS_SUCCESS) + return result; + + result = uds_next_delta_index_entry(&entry); + if (result != UDS_SUCCESS) + return result; + } while (entry.at_end); + + do { + result = uds_remove_delta_index_entry(&entry); + if (result != UDS_SUCCESS) + return result; + + removals++; + } while (!entry.at_end); + } + + if (removals > 0) + uds_log_warning("To avoid chapter index page overflow in chapter %llu, %u entries were removed from the chapter index", + (unsigned long long) chapter_number, removals); + + return UDS_SUCCESS; +} + +/* Make a new chapter index page, initializing it with the data from a given index_page buffer. */ +int uds_initialize_chapter_index_page(struct delta_index_page *index_page, + const struct geometry *geometry, u8 *page_buffer, + u64 volume_nonce) +{ + return uds_initialize_delta_index_page(index_page, volume_nonce, + geometry->chapter_mean_delta, + geometry->chapter_payload_bits, + page_buffer, geometry->bytes_per_page); +} + +/* Validate a chapter index page read during rebuild. */ +int uds_validate_chapter_index_page(const struct delta_index_page *index_page, + const struct geometry *geometry) +{ + int result; + const struct delta_index *delta_index = &index_page->delta_index; + u32 first = index_page->lowest_list_number; + u32 last = index_page->highest_list_number; + u32 list_number; + + /* We walk every delta list from start to finish. */ + for (list_number = first; list_number <= last; list_number++) { + struct delta_index_entry entry; + + result = uds_start_delta_index_search(delta_index, list_number - first, + 0, &entry); + if (result != UDS_SUCCESS) + return result; + + for (;;) { + result = uds_next_delta_index_entry(&entry); + if (result != UDS_SUCCESS) { + /* + * A random bit stream is highly likely to arrive here when we go + * past the end of the delta list. + */ + return result; + } + + if (entry.at_end) + break; + + /* Also make sure that the record page field contains a plausible value. */ + if (uds_get_delta_entry_value(&entry) >= + geometry->record_pages_per_chapter) + /* + * Do not log this as an error. It happens in normal operation when + * we are doing a rebuild but haven't written the entire volume + * once. + */ + return UDS_CORRUPT_DATA; + } + } + return UDS_SUCCESS; +} + +/* + * Search a chapter index page for a record name, returning the record page number that may contain + * the name. + */ +int uds_search_chapter_index_page(struct delta_index_page *index_page, + const struct geometry *geometry, + const struct uds_record_name *name, + u16 *record_page_ptr) +{ + int result; + struct delta_index *delta_index = &index_page->delta_index; + u32 address = uds_hash_to_chapter_delta_address(name, geometry); + u32 delta_list_number = uds_hash_to_chapter_delta_list(name, geometry); + u32 sub_list_number = delta_list_number - index_page->lowest_list_number; + struct delta_index_entry entry; + + result = uds_get_delta_index_entry(delta_index, sub_list_number, address, + name->name, &entry); + if (result != UDS_SUCCESS) + return result; + + if (was_entry_found(&entry, address)) + *record_page_ptr = uds_get_delta_entry_value(&entry); + else + *record_page_ptr = NO_CHAPTER_INDEX_ENTRY; + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/chapter-index.h b/drivers/md/dm-vdo/chapter-index.h new file mode 100644 index 000000000000..ad1f61b6ab9a --- /dev/null +++ b/drivers/md/dm-vdo/chapter-index.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_CHAPTER_INDEX_H +#define UDS_CHAPTER_INDEX_H + +#include + +#include "delta-index.h" +#include "geometry.h" + +/* + * A chapter index for an open chapter is a mutable structure that tracks all the records that have + * been added to the chapter. A chapter index for a closed chapter is similar except that it is + * immutable because the contents of a closed chapter can never change, and the immutable structure + * is more efficient. Both types of chapter index are implemented with a delta index. + */ + +enum { + /* The value returned when no entry is found in the chapter index. */ + NO_CHAPTER_INDEX_ENTRY = U16_MAX, +}; + +struct open_chapter_index { + const struct geometry *geometry; + struct delta_index delta_index; + u64 virtual_chapter_number; + u64 volume_nonce; + size_t memory_size; +}; + +int __must_check uds_make_open_chapter_index(struct open_chapter_index **chapter_index, + const struct geometry *geometry, + u64 volume_nonce); + +void uds_free_open_chapter_index(struct open_chapter_index *chapter_index); + +void uds_empty_open_chapter_index(struct open_chapter_index *chapter_index, + u64 virtual_chapter_number); + +int __must_check uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, + const struct uds_record_name *name, + u32 page_number); + +int __must_check uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, + u8 *memory, u32 first_list, + bool last_page, u32 *lists_packed); + +int __must_check uds_initialize_chapter_index_page(struct delta_index_page *index_page, + const struct geometry *geometry, + u8 *page_buffer, u64 volume_nonce); + +int __must_check uds_validate_chapter_index_page(const struct delta_index_page *index_page, + const struct geometry *geometry); + +int __must_check uds_search_chapter_index_page(struct delta_index_page *index_page, + const struct geometry *geometry, + const struct uds_record_name *name, + u16 *record_page_ptr); + +#endif /* UDS_CHAPTER_INDEX_H */ diff --git a/drivers/md/dm-vdo/open-chapter.c b/drivers/md/dm-vdo/open-chapter.c new file mode 100644 index 000000000000..45cec2c3edb2 --- /dev/null +++ b/drivers/md/dm-vdo/open-chapter.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "open-chapter.h" + +#include + +#include "config.h" +#include "hash-utils.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" + +/* + * Each index zone has a dedicated open chapter zone structure which gets an equal share of the + * open chapter space. Records are assigned to zones based on their record name. Within each zone, + * records are stored in an array in the order they arrive. Additionally, a reference to each + * record is stored in a hash table to help determine if a new record duplicates an existing one. + * If new metadata for an existing name arrives, the record is altered in place. The array of + * records is 1-based so that record number 0 can be used to indicate an unused hash slot. + * + * Deleted records are marked with a flag rather than actually removed to simplify hash table + * management. The array of deleted flags overlays the array of hash slots, but the flags are + * indexed by record number instead of by record name. The number of hash slots will always be a + * power of two that is greater than the number of records to be indexed, guaranteeing that hash + * insertion cannot fail, and that there are sufficient flags for all records. + * + * Once any open chapter zone fills its available space, the chapter is closed. The records from + * each zone are interleaved to attempt to preserve temporal locality and assigned to record pages. + * Empty or deleted records are replaced by copies of a valid record so that the record pages only + * contain valid records. The chapter then constructs a delta index which maps each record name to + * the record page on which that record can be found, which is split into index pages. These + * structures are then passed to the volume to be recorded on storage. + * + * When the index is saved, the open chapter records are saved in a single array, once again + * interleaved to attempt to preserve temporal locality. When the index is reloaded, there may be a + * different number of zones than previously, so the records must be parcelled out to their new + * zones. In addition, depending on the distribution of record names, a new zone may have more + * records than it has space. In this case, the latest records for that zone will be discarded. + */ + +static const u8 OPEN_CHAPTER_MAGIC[] = "ALBOC"; +static const u8 OPEN_CHAPTER_VERSION[] = "02.00"; + +enum { + OPEN_CHAPTER_MAGIC_LENGTH = sizeof(OPEN_CHAPTER_MAGIC) - 1, + OPEN_CHAPTER_VERSION_LENGTH = sizeof(OPEN_CHAPTER_VERSION) - 1, + LOAD_RATIO = 2, +}; + +static inline size_t records_size(const struct open_chapter_zone *open_chapter) +{ + return sizeof(struct uds_volume_record) * (1 + open_chapter->capacity); +} + +static inline size_t slots_size(size_t slot_count) +{ + return sizeof(struct open_chapter_zone_slot) * slot_count; +} + +int uds_make_open_chapter(const struct geometry *geometry, unsigned int zone_count, + struct open_chapter_zone **open_chapter_ptr) +{ + int result; + struct open_chapter_zone *open_chapter; + size_t capacity = geometry->records_per_chapter / zone_count; + size_t slot_count = (1 << bits_per(capacity * LOAD_RATIO)); + + result = uds_allocate_extended(struct open_chapter_zone, slot_count, + struct open_chapter_zone_slot, "open chapter", + &open_chapter); + if (result != UDS_SUCCESS) + return result; + + open_chapter->slot_count = slot_count; + open_chapter->capacity = capacity; + result = uds_allocate_cache_aligned(records_size(open_chapter), "record pages", + &open_chapter->records); + if (result != UDS_SUCCESS) { + uds_free_open_chapter(open_chapter); + return result; + } + + *open_chapter_ptr = open_chapter; + return UDS_SUCCESS; +} + +void uds_reset_open_chapter(struct open_chapter_zone *open_chapter) +{ + open_chapter->size = 0; + open_chapter->deletions = 0; + + memset(open_chapter->records, 0, records_size(open_chapter)); + memset(open_chapter->slots, 0, slots_size(open_chapter->slot_count)); +} + +static unsigned int probe_chapter_slots(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name) +{ + struct uds_volume_record *record; + unsigned int slot_count = open_chapter->slot_count; + unsigned int slot = uds_name_to_hash_slot(name, slot_count); + unsigned int record_number; + unsigned int attempts = 1; + + while (true) { + record_number = open_chapter->slots[slot].record_number; + + /* + * If the hash slot is empty, we've reached the end of a chain without finding the + * record and should terminate the search. + */ + if (record_number == 0) + return slot; + + /* + * If the name of the record referenced by the slot matches and has not been + * deleted, then we've found the requested name. + */ + record = &open_chapter->records[record_number]; + if ((memcmp(&record->name, name, UDS_RECORD_NAME_SIZE) == 0) && + !open_chapter->slots[record_number].deleted) + return slot; + + /* + * Quadratic probing: advance the probe by 1, 2, 3, etc. and try again. This + * performs better than linear probing and works best for 2^N slots. + */ + slot = (slot + attempts++) % slot_count; + } +} + +void uds_search_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + struct uds_record_data *metadata, bool *found) +{ + unsigned int slot; + unsigned int record_number; + + slot = probe_chapter_slots(open_chapter, name); + record_number = open_chapter->slots[slot].record_number; + if (record_number == 0) { + *found = false; + } else { + *found = true; + *metadata = open_chapter->records[record_number].data; + } +} + +/* Add a record to the open chapter zone and return the remaining space. */ +int uds_put_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + const struct uds_record_data *metadata) +{ + unsigned int slot; + unsigned int record_number; + struct uds_volume_record *record; + + if (open_chapter->size >= open_chapter->capacity) + return 0; + + slot = probe_chapter_slots(open_chapter, name); + record_number = open_chapter->slots[slot].record_number; + + if (record_number == 0) { + record_number = ++open_chapter->size; + open_chapter->slots[slot].record_number = record_number; + } + + record = &open_chapter->records[record_number]; + record->name = *name; + record->data = *metadata; + + return open_chapter->capacity - open_chapter->size; +} + +void uds_remove_from_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name) +{ + unsigned int slot; + unsigned int record_number; + + slot = probe_chapter_slots(open_chapter, name); + record_number = open_chapter->slots[slot].record_number; + + if (record_number > 0) { + open_chapter->slots[record_number].deleted = true; + open_chapter->deletions += 1; + } +} + +void uds_free_open_chapter(struct open_chapter_zone *open_chapter) +{ + if (open_chapter != NULL) { + uds_free(open_chapter->records); + uds_free(open_chapter); + } +} + +/* Map each record name to its record page number in the delta chapter index. */ +static int fill_delta_chapter_index(struct open_chapter_zone **chapter_zones, + unsigned int zone_count, + struct open_chapter_index *index, + struct uds_volume_record *collated_records) +{ + int result; + unsigned int records_per_chapter; + unsigned int records_per_page; + unsigned int record_index; + unsigned int records = 0; + u32 page_number; + unsigned int z; + int overflow_count = 0; + struct uds_volume_record *fill_record = NULL; + + /* + * The record pages should not have any empty space, so find a record with which to fill + * the chapter zone if it was closed early, and also to replace any deleted records. The + * last record in any filled zone is guaranteed to not have been deleted, so use one of + * those. + */ + for (z = 0; z < zone_count; z++) { + struct open_chapter_zone *zone = chapter_zones[z]; + + if (zone->size == zone->capacity) { + fill_record = &zone->records[zone->size]; + break; + } + } + + records_per_chapter = index->geometry->records_per_chapter; + records_per_page = index->geometry->records_per_page; + + for (records = 0; records < records_per_chapter; records++) { + struct uds_volume_record *record = &collated_records[records]; + struct open_chapter_zone *open_chapter; + + /* The record arrays in the zones are 1-based. */ + record_index = 1 + (records / zone_count); + page_number = records / records_per_page; + open_chapter = chapter_zones[records % zone_count]; + + /* Use the fill record in place of an unused record. */ + if (record_index > open_chapter->size || + open_chapter->slots[record_index].deleted) { + *record = *fill_record; + continue; + } + + *record = open_chapter->records[record_index]; + result = uds_put_open_chapter_index_record(index, &record->name, + page_number); + switch (result) { + case UDS_SUCCESS: + break; + case UDS_OVERFLOW: + overflow_count++; + break; + default: + uds_log_error_strerror(result, + "failed to build open chapter index"); + return result; + } + } + + if (overflow_count > 0) + uds_log_warning("Failed to add %d entries to chapter index", + overflow_count); + + return UDS_SUCCESS; +} + +int uds_close_open_chapter(struct open_chapter_zone **chapter_zones, + unsigned int zone_count, struct volume *volume, + struct open_chapter_index *chapter_index, + struct uds_volume_record *collated_records, + u64 virtual_chapter_number) +{ + int result; + + uds_empty_open_chapter_index(chapter_index, virtual_chapter_number); + result = fill_delta_chapter_index(chapter_zones, zone_count, chapter_index, + collated_records); + if (result != UDS_SUCCESS) + return result; + + return uds_write_chapter(volume, chapter_index, collated_records); +} + +int uds_save_open_chapter(struct uds_index *index, struct buffered_writer *writer) +{ + int result; + struct open_chapter_zone *open_chapter; + struct uds_volume_record *record; + u8 record_count_data[sizeof(u32)]; + u32 record_count = 0; + unsigned int record_index; + unsigned int z; + + result = uds_write_to_buffered_writer(writer, OPEN_CHAPTER_MAGIC, + OPEN_CHAPTER_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(writer, OPEN_CHAPTER_VERSION, + OPEN_CHAPTER_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return result; + + for (z = 0; z < index->zone_count; z++) { + open_chapter = index->zones[z]->open_chapter; + record_count += open_chapter->size - open_chapter->deletions; + } + + put_unaligned_le32(record_count, record_count_data); + result = uds_write_to_buffered_writer(writer, record_count_data, + sizeof(record_count_data)); + if (result != UDS_SUCCESS) + return result; + + record_index = 1; + while (record_count > 0) { + for (z = 0; z < index->zone_count; z++) { + open_chapter = index->zones[z]->open_chapter; + if (record_index > open_chapter->size) + continue; + + if (open_chapter->slots[record_index].deleted) + continue; + + record = &open_chapter->records[record_index]; + result = uds_write_to_buffered_writer(writer, (u8 *) record, + sizeof(*record)); + if (result != UDS_SUCCESS) + return result; + + record_count--; + } + + record_index++; + } + + return uds_flush_buffered_writer(writer); +} + +u64 uds_compute_saved_open_chapter_size(struct geometry *geometry) +{ + unsigned int records_per_chapter = geometry->records_per_chapter; + + return OPEN_CHAPTER_MAGIC_LENGTH + OPEN_CHAPTER_VERSION_LENGTH + sizeof(u32) + + records_per_chapter * sizeof(struct uds_volume_record); +} + +static int load_version20(struct uds_index *index, struct buffered_reader *reader) +{ + int result; + u32 record_count; + u8 record_count_data[sizeof(u32)]; + struct uds_volume_record record; + + /* + * Track which zones cannot accept any more records. If the open chapter had a different + * number of zones previously, some new zones may have more records than they have space + * for. These overflow records will be discarded. + */ + bool full_flags[MAX_ZONES] = { + false, + }; + + result = uds_read_from_buffered_reader(reader, (u8 *) &record_count_data, + sizeof(record_count_data)); + if (result != UDS_SUCCESS) + return result; + + record_count = get_unaligned_le32(record_count_data); + while (record_count-- > 0) { + unsigned int zone = 0; + + result = uds_read_from_buffered_reader(reader, (u8 *) &record, + sizeof(record)); + if (result != UDS_SUCCESS) + return result; + + if (index->zone_count > 1) + zone = uds_get_volume_index_zone(index->volume_index, + &record.name); + + if (!full_flags[zone]) { + struct open_chapter_zone *open_chapter; + unsigned int remaining; + + open_chapter = index->zones[zone]->open_chapter; + remaining = uds_put_open_chapter(open_chapter, &record.name, + &record.data); + /* Do not allow any zone to fill completely. */ + full_flags[zone] = (remaining <= 1); + } + } + + return UDS_SUCCESS; +} + +int uds_load_open_chapter(struct uds_index *index, struct buffered_reader *reader) +{ + u8 version[OPEN_CHAPTER_VERSION_LENGTH]; + int result; + + result = uds_verify_buffered_data(reader, OPEN_CHAPTER_MAGIC, + OPEN_CHAPTER_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, version, sizeof(version)); + if (result != UDS_SUCCESS) + return result; + + if (memcmp(OPEN_CHAPTER_VERSION, version, sizeof(version)) != 0) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "Invalid open chapter version: %.*s", + (int) sizeof(version), version); + } + + return load_version20(index, reader); +} diff --git a/drivers/md/dm-vdo/open-chapter.h b/drivers/md/dm-vdo/open-chapter.h new file mode 100644 index 000000000000..eb11d897f20a --- /dev/null +++ b/drivers/md/dm-vdo/open-chapter.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_OPEN_CHAPTER_H +#define UDS_OPEN_CHAPTER_H + +#include "chapter-index.h" +#include "geometry.h" +#include "index.h" +#include "volume.h" + +/* + * The open chapter tracks the newest records in memory. Like the index as a whole, each open + * chapter is divided into a number of independent zones which are interleaved when the chapter is + * committed to the volume. + */ + +enum { + OPEN_CHAPTER_RECORD_NUMBER_BITS = 23, +}; + +struct open_chapter_zone_slot { + /* If non-zero, the record number addressed by this hash slot */ + unsigned int record_number : OPEN_CHAPTER_RECORD_NUMBER_BITS; + /* If true, the record at the index of this hash slot was deleted */ + bool deleted : 1; +} __packed; + +struct open_chapter_zone { + /* The maximum number of records that can be stored */ + unsigned int capacity; + /* The number of records stored */ + unsigned int size; + /* The number of deleted records */ + unsigned int deletions; + /* Array of chunk records, 1-based */ + struct uds_volume_record *records; + /* The number of slots in the hash table */ + unsigned int slot_count; + /* The hash table slots, referencing virtual record numbers */ + struct open_chapter_zone_slot slots[]; +}; + +int __must_check uds_make_open_chapter(const struct geometry *geometry, + unsigned int zone_count, + struct open_chapter_zone **open_chapter_ptr); + +void uds_reset_open_chapter(struct open_chapter_zone *open_chapter); + +void uds_search_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + struct uds_record_data *metadata, bool *found); + +int __must_check uds_put_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + const struct uds_record_data *metadata); + +void uds_remove_from_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name); + +void uds_free_open_chapter(struct open_chapter_zone *open_chapter); + +int __must_check uds_close_open_chapter(struct open_chapter_zone **chapter_zones, + unsigned int zone_count, struct volume *volume, + struct open_chapter_index *chapter_index, + struct uds_volume_record *collated_records, + u64 virtual_chapter_number); + +int __must_check uds_save_open_chapter(struct uds_index *index, + struct buffered_writer *writer); + +int __must_check uds_load_open_chapter(struct uds_index *index, + struct buffered_reader *reader); + +u64 uds_compute_saved_open_chapter_size(struct geometry *geometry); + +#endif /* UDS_OPEN_CHAPTER_H */ From 4e7ff03922cba20997680dbe78531997760d6117 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:55:23 -0500 Subject: [PATCH 0562/1038] dm vdo: implement the chapter volume store The volume store structures manage the reading and writing of chapter pages. When a chapter is closed, it is packed into a read-only structure, split across several pages, and written to storage. The volume store also contains a cache and specialized queues that sort and batch requests by the page they need, in order to minimize latency and I/O requests when records have to be read from storage. The cache and queues also coordinate with the volume index to ensure that the volume does not waste resources reading pages that are no longer valid. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: John Wiele Signed-off-by: John Wiele Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/index-page-map.c | 175 +++ drivers/md/dm-vdo/index-page-map.h | 50 + drivers/md/dm-vdo/radix-sort.c | 332 ++++++ drivers/md/dm-vdo/radix-sort.h | 26 + drivers/md/dm-vdo/volume.c | 1711 ++++++++++++++++++++++++++++ drivers/md/dm-vdo/volume.h | 171 +++ 6 files changed, 2465 insertions(+) create mode 100644 drivers/md/dm-vdo/index-page-map.c create mode 100644 drivers/md/dm-vdo/index-page-map.h create mode 100644 drivers/md/dm-vdo/radix-sort.c create mode 100644 drivers/md/dm-vdo/radix-sort.h create mode 100644 drivers/md/dm-vdo/volume.c create mode 100644 drivers/md/dm-vdo/volume.h diff --git a/drivers/md/dm-vdo/index-page-map.c b/drivers/md/dm-vdo/index-page-map.c new file mode 100644 index 000000000000..190d12848ba2 --- /dev/null +++ b/drivers/md/dm-vdo/index-page-map.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "index-page-map.h" + +#include "errors.h" +#include "hash-utils.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" +#include "uds-threads.h" +#include "uds.h" + +/* + * The index page map is conceptually a two-dimensional array indexed by chapter number and index + * page number within the chapter. Each entry contains the number of the last delta list on that + * index page. In order to save memory, the information for the last page in each chapter is not + * recorded, as it is known from the geometry. + */ + +static const u8 PAGE_MAP_MAGIC[] = "ALBIPM02"; + +enum { + PAGE_MAP_MAGIC_LENGTH = sizeof(PAGE_MAP_MAGIC) - 1, +}; + +static inline u32 get_entry_count(const struct geometry *geometry) +{ + return geometry->chapters_per_volume * (geometry->index_pages_per_chapter - 1); +} + +int uds_make_index_page_map(const struct geometry *geometry, + struct index_page_map **map_ptr) +{ + int result; + struct index_page_map *map; + + result = uds_allocate(1, struct index_page_map, "page map", &map); + if (result != UDS_SUCCESS) + return result; + + map->geometry = geometry; + map->entries_per_chapter = geometry->index_pages_per_chapter - 1; + result = uds_allocate(get_entry_count(geometry), u16, "Index Page Map Entries", + &map->entries); + if (result != UDS_SUCCESS) { + uds_free_index_page_map(map); + return result; + } + + *map_ptr = map; + return UDS_SUCCESS; +} + +void uds_free_index_page_map(struct index_page_map *map) +{ + if (map != NULL) { + uds_free(map->entries); + uds_free(map); + } +} + +void uds_update_index_page_map(struct index_page_map *map, u64 virtual_chapter_number, + u32 chapter_number, u32 index_page_number, + u32 delta_list_number) +{ + size_t slot; + + map->last_update = virtual_chapter_number; + if (index_page_number == map->entries_per_chapter) + return; + + slot = (chapter_number * map->entries_per_chapter) + index_page_number; + map->entries[slot] = delta_list_number; +} + +u32 uds_find_index_page_number(const struct index_page_map *map, + const struct uds_record_name *name, u32 chapter_number) +{ + u32 delta_list_number = uds_hash_to_chapter_delta_list(name, map->geometry); + u32 slot = chapter_number * map->entries_per_chapter; + u32 page; + + for (page = 0; page < map->entries_per_chapter; page++) { + if (delta_list_number <= map->entries[slot + page]) + break; + } + + return page; +} + +void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_number, + u32 index_page_number, u32 *lowest_list, + u32 *highest_list) +{ + u32 slot = chapter_number * map->entries_per_chapter; + + *lowest_list = ((index_page_number == 0) ? + 0 : + map->entries[slot + index_page_number - 1] + 1); + *highest_list = ((index_page_number < map->entries_per_chapter) ? + map->entries[slot + index_page_number] : + map->geometry->delta_lists_per_chapter - 1); +} + +u64 uds_compute_index_page_map_save_size(const struct geometry *geometry) +{ + return PAGE_MAP_MAGIC_LENGTH + sizeof(u64) + sizeof(u16) * get_entry_count(geometry); +} + +int uds_write_index_page_map(struct index_page_map *map, struct buffered_writer *writer) +{ + int result; + u8 *buffer; + size_t offset = 0; + u64 saved_size = uds_compute_index_page_map_save_size(map->geometry); + u32 i; + + result = uds_allocate(saved_size, u8, "page map data", &buffer); + if (result != UDS_SUCCESS) + return result; + + memcpy(buffer, PAGE_MAP_MAGIC, PAGE_MAP_MAGIC_LENGTH); + offset += PAGE_MAP_MAGIC_LENGTH; + encode_u64_le(buffer, &offset, map->last_update); + for (i = 0; i < get_entry_count(map->geometry); i++) + encode_u16_le(buffer, &offset, map->entries[i]); + + result = uds_write_to_buffered_writer(writer, buffer, offset); + uds_free(buffer); + if (result != UDS_SUCCESS) + return result; + + return uds_flush_buffered_writer(writer); +} + +int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader *reader) +{ + int result; + u8 magic[PAGE_MAP_MAGIC_LENGTH]; + u8 *buffer; + size_t offset = 0; + u64 saved_size = uds_compute_index_page_map_save_size(map->geometry); + u32 i; + + result = uds_allocate(saved_size, u8, "page map data", &buffer); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, buffer, saved_size); + if (result != UDS_SUCCESS) { + uds_free(buffer); + return result; + } + + memcpy(&magic, buffer, PAGE_MAP_MAGIC_LENGTH); + offset += PAGE_MAP_MAGIC_LENGTH; + if (memcmp(magic, PAGE_MAP_MAGIC, PAGE_MAP_MAGIC_LENGTH) != 0) { + uds_free(buffer); + return UDS_CORRUPT_DATA; + } + + decode_u64_le(buffer, &offset, &map->last_update); + for (i = 0; i < get_entry_count(map->geometry); i++) + decode_u16_le(buffer, &offset, &map->entries[i]); + + uds_free(buffer); + uds_log_debug("read index page map, last update %llu", + (unsigned long long) map->last_update); + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/index-page-map.h b/drivers/md/dm-vdo/index-page-map.h new file mode 100644 index 000000000000..51ac9b81362f --- /dev/null +++ b/drivers/md/dm-vdo/index-page-map.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_PAGE_MAP_H +#define UDS_INDEX_PAGE_MAP_H + +#include "geometry.h" +#include "io-factory.h" + +/* + * The index maintains a page map which records how the chapter delta lists are distributed among + * the index pages for each chapter, allowing the volume to be efficient about reading only pages + * that it knows it will need. + */ + +struct index_page_map { + const struct geometry *geometry; + u64 last_update; + u32 entries_per_chapter; + u16 *entries; +}; + +int __must_check uds_make_index_page_map(const struct geometry *geometry, + struct index_page_map **map_ptr); + +void uds_free_index_page_map(struct index_page_map *map); + +int __must_check uds_read_index_page_map(struct index_page_map *map, + struct buffered_reader *reader); + +int __must_check uds_write_index_page_map(struct index_page_map *map, + struct buffered_writer *writer); + +void uds_update_index_page_map(struct index_page_map *map, u64 virtual_chapter_number, + u32 chapter_number, u32 index_page_number, + u32 delta_list_number); + +u32 __must_check uds_find_index_page_number(const struct index_page_map *map, + const struct uds_record_name *name, + u32 chapter_number); + +void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_number, + u32 index_page_number, u32 *lowest_list, + u32 *highest_list); + +u64 uds_compute_index_page_map_save_size(const struct geometry *geometry); + +#endif /* UDS_INDEX_PAGE_MAP_H */ diff --git a/drivers/md/dm-vdo/radix-sort.c b/drivers/md/dm-vdo/radix-sort.c new file mode 100644 index 000000000000..1f17c708a652 --- /dev/null +++ b/drivers/md/dm-vdo/radix-sort.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "radix-sort.h" + +#include +#include + +#include "memory-alloc.h" +#include "string-utils.h" + +/* + * This implementation allocates one large object to do the sorting, which can be reused as many + * times as desired. The amount of memory required is logarithmically proportional to the number of + * keys to be sorted. + */ + +enum { + /* Piles smaller than this are handled with a simple insertion sort. */ + INSERTION_SORT_THRESHOLD = 12, +}; + +/* Sort keys are pointers to immutable fixed-length arrays of bytes. */ +typedef const u8 *sort_key_t; + +/* + * The keys are separated into piles based on the byte in each keys at the current offset, so the + * number of keys with each byte must be counted. + */ +struct histogram { + /* The number of non-empty bins */ + u16 used; + /* The index (key byte) of the first non-empty bin */ + u16 first; + /* The index (key byte) of the last non-empty bin */ + u16 last; + /* The number of occurrences of each specific byte */ + u32 size[256]; +}; + +/* + * Sub-tasks are manually managed on a stack, both for performance and to put a logarithmic bound + * on the stack space needed. + */ +struct task { + /* Pointer to the first key to sort. */ + sort_key_t *first_key; + /* Pointer to the last key to sort. */ + sort_key_t *last_key; + /* The offset into the key at which to continue sorting. */ + u16 offset; + /* The number of bytes remaining in the sort keys. */ + u16 length; +}; + +struct radix_sorter { + unsigned int count; + struct histogram bins; + sort_key_t *pile[256]; + struct task *end_of_stack; + struct task insertion_list[256]; + struct task stack[]; +}; + +/* Compare a segment of two fixed-length keys starting at an offset. */ +static inline int compare(sort_key_t key1, sort_key_t key2, u16 offset, u16 length) +{ + return memcmp(&key1[offset], &key2[offset], length); +} + +/* Insert the next unsorted key into an array of sorted keys. */ +static inline void insert_key(const struct task task, sort_key_t *next) +{ + /* Pull the unsorted key out, freeing up the array slot. */ + sort_key_t unsorted = *next; + + /* Compare the key to the preceding sorted entries, shifting down ones that are larger. */ + while ((--next >= task.first_key) && + (compare(unsorted, next[0], task.offset, task.length) < 0)) + next[1] = next[0]; + + /* Insert the key into the last slot that was cleared, sorting it. */ + next[1] = unsorted; +} + +/* + * Sort a range of key segments using an insertion sort. This simple sort is faster than the + * 256-way radix sort when the number of keys to sort is small. + */ +static inline void insertion_sort(const struct task task) +{ + sort_key_t *next; + + for (next = task.first_key + 1; next <= task.last_key; next++) + insert_key(task, next); +} + +/* Push a sorting task onto a task stack. */ +static inline void push_task(struct task **stack_pointer, sort_key_t *first_key, + u32 count, u16 offset, u16 length) +{ + struct task *task = (*stack_pointer)++; + + task->first_key = first_key; + task->last_key = &first_key[count - 1]; + task->offset = offset; + task->length = length; +} + +static inline void swap_keys(sort_key_t *a, sort_key_t *b) +{ + sort_key_t c = *a; + *a = *b; + *b = c; +} + +/* + * Count the number of times each byte value appears in the arrays of keys to sort at the current + * offset, keeping track of the number of non-empty bins, and the index of the first and last + * non-empty bin. + */ +static inline void measure_bins(const struct task task, struct histogram *bins) +{ + sort_key_t *key_ptr; + + /* + * Subtle invariant: bins->used and bins->size[] are zero because the sorting code clears + * it all out as it goes. Even though this structure is re-used, we don't need to pay to + * zero it before starting a new tally. + */ + bins->first = U8_MAX; + bins->last = 0; + + for (key_ptr = task.first_key; key_ptr <= task.last_key; key_ptr++) { + /* Increment the count for the byte in the key at the current offset. */ + u8 bin = (*key_ptr)[task.offset]; + u32 size = ++bins->size[bin]; + + /* Track non-empty bins. */ + if (size == 1) { + bins->used += 1; + if (bin < bins->first) + bins->first = bin; + + if (bin > bins->last) + bins->last = bin; + } + } +} + +/* + * Convert the bin sizes to pointers to where each pile goes. + * + * pile[0] = first_key + bin->size[0], + * pile[1] = pile[0] + bin->size[1], etc. + * + * After the keys are moved to the appropriate pile, we'll need to sort each of the piles by the + * next radix position. A new task is put on the stack for each pile containing lots of keys, or a + * new task is put on the list for each pile containing few keys. + * + * @stack: pointer the top of the stack + * @end_of_stack: the end of the stack + * @list: pointer the head of the list + * @pile: array for pointers to the end of each pile + * @bins: the histogram of the sizes of each pile + * @first_key: the first key of the stack + * @offset: the next radix position to sort by + * @length: the number of bytes remaining in the sort keys + * + * Return: UDS_SUCCESS or an error code + */ +static inline int push_bins(struct task **stack, struct task *end_of_stack, + struct task **list, sort_key_t *pile[], + struct histogram *bins, sort_key_t *first_key, + u16 offset, u16 length) +{ + sort_key_t *pile_start = first_key; + int bin; + + for (bin = bins->first; ; bin++) { + u32 size = bins->size[bin]; + + /* Skip empty piles. */ + if (size == 0) + continue; + + /* There's no need to sort empty keys. */ + if (length > 0) { + if (size > INSERTION_SORT_THRESHOLD) { + if (*stack >= end_of_stack) + return UDS_BAD_STATE; + + push_task(stack, pile_start, size, offset, length); + } else if (size > 1) { + push_task(list, pile_start, size, offset, length); + } + } + + pile_start += size; + pile[bin] = pile_start; + if (--bins->used == 0) + break; + } + + return UDS_SUCCESS; +} + +int uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter) +{ + int result; + unsigned int stack_size = count / INSERTION_SORT_THRESHOLD; + struct radix_sorter *radix_sorter; + + result = uds_allocate_extended(struct radix_sorter, stack_size, struct task, + __func__, &radix_sorter); + if (result != UDS_SUCCESS) + return result; + + radix_sorter->count = count; + radix_sorter->end_of_stack = radix_sorter->stack + stack_size; + *sorter = radix_sorter; + return UDS_SUCCESS; +} + +void uds_free_radix_sorter(struct radix_sorter *sorter) +{ + uds_free(sorter); +} + +/* + * Sort pointers to fixed-length keys (arrays of bytes) using a radix sort. The sort implementation + * is unstable, so the relative ordering of equal keys is not preserved. + */ +int uds_radix_sort(struct radix_sorter *sorter, const unsigned char *keys[], + unsigned int count, unsigned short length) +{ + struct task start; + struct histogram *bins = &sorter->bins; + sort_key_t **pile = sorter->pile; + struct task *task_stack = sorter->stack; + + /* All zero-length keys are identical and therefore already sorted. */ + if ((count == 0) || (length == 0)) + return UDS_SUCCESS; + + /* The initial task is to sort the entire length of all the keys. */ + start = (struct task) { + .first_key = keys, + .last_key = &keys[count - 1], + .offset = 0, + .length = length, + }; + + if (count <= INSERTION_SORT_THRESHOLD) { + insertion_sort(start); + return UDS_SUCCESS; + } + + if (count > sorter->count) + return UDS_INVALID_ARGUMENT; + + /* + * Repeatedly consume a sorting task from the stack and process it, pushing new sub-tasks + * onto the stack for each radix-sorted pile. When all tasks and sub-tasks have been + * processed, the stack will be empty and all the keys in the starting task will be fully + * sorted. + */ + for (*task_stack = start; task_stack >= sorter->stack; task_stack--) { + const struct task task = *task_stack; + struct task *insertion_task_list; + int result; + sort_key_t *fence; + sort_key_t *end; + + measure_bins(task, bins); + + /* + * Now that we know how large each bin is, generate pointers for each of the piles + * and push a new task to sort each pile by the next radix byte. + */ + insertion_task_list = sorter->insertion_list; + result = push_bins(&task_stack, sorter->end_of_stack, + &insertion_task_list, pile, bins, task.first_key, + task.offset + 1, task.length - 1); + if (result != UDS_SUCCESS) { + memset(bins, 0, sizeof(*bins)); + return result; + } + + /* Now bins->used is zero again. */ + + /* + * Don't bother processing the last pile: when piles 0..N-1 are all in place, then + * pile N must also be in place. + */ + end = task.last_key - bins->size[bins->last]; + bins->size[bins->last] = 0; + + for (fence = task.first_key; fence <= end; ) { + u8 bin; + sort_key_t key = *fence; + + /* + * The radix byte of the key tells us which pile it belongs in. Swap it for + * an unprocessed item just below that pile, and repeat. + */ + while (--pile[bin = key[task.offset]] > fence) + swap_keys(pile[bin], &key); + + /* + * The pile reached the fence. Put the key at the bottom of that pile, + * completing it, and advance the fence to the next pile. + */ + *fence = key; + fence += bins->size[bin]; + bins->size[bin] = 0; + } + + /* Now bins->size[] is all zero again. */ + + /* + * When the number of keys in a task gets small enough, it is faster to use an + * insertion sort than to keep subdividing into tiny piles. + */ + while (--insertion_task_list >= sorter->insertion_list) + insertion_sort(*insertion_task_list); + } + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/radix-sort.h b/drivers/md/dm-vdo/radix-sort.h new file mode 100644 index 000000000000..812949bc2cee --- /dev/null +++ b/drivers/md/dm-vdo/radix-sort.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_RADIX_SORT_H +#define UDS_RADIX_SORT_H + +/* + * Radix sort is implemented using an American Flag sort, an unstable, in-place 8-bit radix + * exchange sort. This is adapted from the algorithm in the paper by Peter M. McIlroy, Keith + * Bostic, and M. Douglas McIlroy, "Engineering Radix Sort". + * + * http://www.usenix.org/publications/compsystems/1993/win_mcilroy.pdf + */ + +struct radix_sorter; + +int __must_check uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter); + +void uds_free_radix_sorter(struct radix_sorter *sorter); + +int __must_check uds_radix_sort(struct radix_sorter *sorter, const unsigned char *keys[], + unsigned int count, unsigned short length); + +#endif /* UDS_RADIX_SORT_H */ diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c new file mode 100644 index 000000000000..65bf6d32e5d0 --- /dev/null +++ b/drivers/md/dm-vdo/volume.c @@ -0,0 +1,1711 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "volume.h" + +#include +#include +#include + +#include "chapter-index.h" +#include "config.h" +#include "errors.h" +#include "geometry.h" +#include "hash-utils.h" +#include "index.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "sparse-cache.h" +#include "string-utils.h" +#include "uds-threads.h" + +/* + * The first block of the volume layout is reserved for the volume header, which is no longer used. + * The remainder of the volume is divided into chapters consisting of several pages of records, and + * several pages of static index to use to find those records. The index pages are recorded first, + * followed by the record pages. The chapters are written in order as they are filled, so the + * volume storage acts as a circular log of the most recent chapters, with each new chapter + * overwriting the oldest saved one. + * + * When a new chapter is filled and closed, the records from that chapter are sorted and + * interleaved in approximate temporal order, and assigned to record pages. Then a static delta + * index is generated to store which record page contains each record. The in-memory index page map + * is also updated to indicate which delta lists fall on each chapter index page. This means that + * when a record is read, the volume only has to load a single index page and a single record page, + * rather than search the entire chapter. These index and record pages are written to storage, and + * the index pages are transferred to the page cache under the theory that the most recently + * written chapter is likely to be accessed again soon. + * + * When reading a record, the volume index will indicate which chapter should contain it. The + * volume uses the index page map to determine which chapter index page needs to be loaded, and + * then reads the relevant record page number from the chapter index. Both index and record pages + * are stored in a page cache when read for the common case that subsequent records need the same + * pages. The page cache evicts the least recently accessed entries when caching new pages. In + * addition, the volume uses dm-bufio to manage access to the storage, which may allow for + * additional caching depending on available system resources. + * + * Record requests are handled from cached pages when possible. If a page needs to be read, it is + * placed on a queue along with the request that wants to read it. Any requests for the same page + * that arrive while the read is pending are added to the queue entry. A separate reader thread + * handles the queued reads, adding the page to the cache and updating any requests queued with it + * so they can continue processing. This allows the index zone threads to continue processing new + * requests rather than wait for the storage reads. + * + * When an index rebuild is necessary, the volume reads each stored chapter to determine which + * range of chapters contain valid records, so that those records can be used to reconstruct the + * in-memory volume index. + */ + +enum { + /* The maximum allowable number of contiguous bad chapters */ + MAX_BAD_CHAPTERS = 100, + VOLUME_CACHE_MAX_ENTRIES = (U16_MAX >> 1), + VOLUME_CACHE_QUEUED_FLAG = (1 << 15), + VOLUME_CACHE_MAX_QUEUED_READS = 4096, +}; + +static const u64 BAD_CHAPTER = U64_MAX; + +/* + * The invalidate counter is two 32 bits fields stored together atomically. The low order 32 bits + * are the physical page number of the cached page being read. The high order 32 bits are a + * sequence number. This value is written when the zone that owns it begins or completes a cache + * search. Any other thread will only read the counter in wait_for_pending_searches() while waiting + * to update the cache contents. + */ +union invalidate_counter { + u64 value; + struct { + u32 page; + u32 counter; + }; +}; + +static inline u32 map_to_page_number(struct geometry *geometry, u32 physical_page) +{ + return (physical_page - HEADER_PAGES_PER_VOLUME) % geometry->pages_per_chapter; +} + +static inline u32 map_to_chapter_number(struct geometry *geometry, u32 physical_page) +{ + return (physical_page - HEADER_PAGES_PER_VOLUME) / geometry->pages_per_chapter; +} + +static inline bool is_record_page(struct geometry *geometry, u32 physical_page) +{ + return map_to_page_number(geometry, physical_page) >= geometry->index_pages_per_chapter; +} + +static u32 map_to_physical_page(const struct geometry *geometry, u32 chapter, u32 page) +{ + /* Page zero is the header page, so the first chapter index page is page one. */ + return HEADER_PAGES_PER_VOLUME + (geometry->pages_per_chapter * chapter) + page; +} + +static inline union invalidate_counter get_invalidate_counter(struct page_cache *cache, + unsigned int zone_number) +{ + return (union invalidate_counter) { + .value = READ_ONCE(cache->search_pending_counters[zone_number].atomic_value), + }; +} + +static inline void set_invalidate_counter(struct page_cache *cache, + unsigned int zone_number, + union invalidate_counter invalidate_counter) +{ + WRITE_ONCE(cache->search_pending_counters[zone_number].atomic_value, + invalidate_counter.value); +} + +static inline bool search_pending(union invalidate_counter invalidate_counter) +{ + return (invalidate_counter.counter & 1) != 0; +} + +/* Lock the cache for a zone in order to search for a page. */ +static void begin_pending_search(struct page_cache *cache, u32 physical_page, + unsigned int zone_number) +{ + union invalidate_counter invalidate_counter = + get_invalidate_counter(cache, zone_number); + + invalidate_counter.page = physical_page; + invalidate_counter.counter++; + set_invalidate_counter(cache, zone_number, invalidate_counter); + ASSERT_LOG_ONLY(search_pending(invalidate_counter), + "Search is pending for zone %u", zone_number); + /* + * This memory barrier ensures that the write to the invalidate counter is seen by other + * threads before this thread accesses the cached page. The corresponding read memory + * barrier is in wait_for_pending_searches(). + */ + smp_mb(); +} + +/* Unlock the cache for a zone by clearing its invalidate counter. */ +static void end_pending_search(struct page_cache *cache, unsigned int zone_number) +{ + union invalidate_counter invalidate_counter; + + /* + * This memory barrier ensures that this thread completes reads of the + * cached page before other threads see the write to the invalidate + * counter. + */ + smp_mb(); + + invalidate_counter = get_invalidate_counter(cache, zone_number); + ASSERT_LOG_ONLY(search_pending(invalidate_counter), + "Search is pending for zone %u", zone_number); + invalidate_counter.counter++; + set_invalidate_counter(cache, zone_number, invalidate_counter); +} + +static void wait_for_pending_searches(struct page_cache *cache, u32 physical_page) +{ + union invalidate_counter initial_counters[MAX_ZONES]; + unsigned int i; + + /* + * We hold the read_threads_mutex. We are waiting for threads that do not hold the + * read_threads_mutex. Those threads have "locked" their targeted page by setting the + * search_pending_counter. The corresponding write memory barrier is in + * begin_pending_search(). + */ + smp_mb(); + + for (i = 0; i < cache->zone_count; i++) + initial_counters[i] = get_invalidate_counter(cache, i); + for (i = 0; i < cache->zone_count; i++) { + if (search_pending(initial_counters[i]) && + (initial_counters[i].page == physical_page)) { + /* + * There is an active search using the physical page. We need to wait for + * the search to finish. + * + * FIXME: Investigate using wait_event() to wait for the search to finish. + */ + while (initial_counters[i].value == + get_invalidate_counter(cache, i).value) + cond_resched(); + } + } +} + +static void release_page_buffer(struct cached_page *page) +{ + if (page->buffer != NULL) + dm_bufio_release(uds_forget(page->buffer)); +} + +static void clear_cache_page(struct page_cache *cache, struct cached_page *page) +{ + /* Do not clear read_pending because the read queue relies on it. */ + release_page_buffer(page); + page->physical_page = cache->indexable_pages; + WRITE_ONCE(page->last_used, 0); +} + +static void make_page_most_recent(struct page_cache *cache, struct cached_page *page) +{ + /* + * ASSERTION: We are either a zone thread holding a search_pending_counter, or we are any + * thread holding the read_threads_mutex. + */ + if (atomic64_read(&cache->clock) != READ_ONCE(page->last_used)) + WRITE_ONCE(page->last_used, atomic64_inc_return(&cache->clock)); +} + +/* Select a page to remove from the cache to make space for a new entry. */ +static struct cached_page *select_victim_in_cache(struct page_cache *cache) +{ + struct cached_page *page; + int oldest_index = 0; + s64 oldest_time = S64_MAX; + s64 last_used; + u16 i; + + /* Find the oldest unclaimed page. We hold the read_threads_mutex. */ + for (i = 0; i < cache->cache_slots; i++) { + /* A page with a pending read must not be replaced. */ + if (cache->cache[i].read_pending) + continue; + + last_used = READ_ONCE(cache->cache[i].last_used); + if (last_used <= oldest_time) { + oldest_time = last_used; + oldest_index = i; + } + } + + page = &cache->cache[oldest_index]; + if (page->physical_page != cache->indexable_pages) { + WRITE_ONCE(cache->index[page->physical_page], cache->cache_slots); + wait_for_pending_searches(cache, page->physical_page); + } + + page->read_pending = true; + clear_cache_page(cache, page); + return page; +} + +/* Make a newly filled cache entry available to other threads. */ +static int put_page_in_cache(struct page_cache *cache, u32 physical_page, + struct cached_page *page) +{ + int result; + + /* We hold the read_threads_mutex. */ + result = ASSERT((page->read_pending), "page to install has a pending read"); + if (result != UDS_SUCCESS) + return result; + + page->physical_page = physical_page; + make_page_most_recent(cache, page); + page->read_pending = false; + + /* + * We hold the read_threads_mutex, but we must have a write memory barrier before making + * the cached_page available to the readers that do not hold the mutex. The corresponding + * read memory barrier is in get_page_and_index(). + */ + smp_wmb(); + + /* This assignment also clears the queued flag. */ + WRITE_ONCE(cache->index[physical_page], page - cache->cache); + return UDS_SUCCESS; +} + +static void cancel_page_in_cache(struct page_cache *cache, u32 physical_page, + struct cached_page *page) +{ + int result; + + /* We hold the read_threads_mutex. */ + result = ASSERT((page->read_pending), "page to install has a pending read"); + if (result != UDS_SUCCESS) + return; + + clear_cache_page(cache, page); + page->read_pending = false; + + /* Clear the mapping and the queued flag for the new page. */ + WRITE_ONCE(cache->index[physical_page], cache->cache_slots); +} + +static inline u16 next_queue_position(u16 position) +{ + return (position + 1) % VOLUME_CACHE_MAX_QUEUED_READS; +} + +static inline void advance_queue_position(u16 *position) +{ + *position = next_queue_position(*position); +} + +static inline bool read_queue_is_full(struct page_cache *cache) +{ + return cache->read_queue_first == next_queue_position(cache->read_queue_last); +} + +static bool enqueue_read(struct page_cache *cache, struct uds_request *request, + u32 physical_page) +{ + struct queued_read *queue_entry; + u16 last = cache->read_queue_last; + u16 read_queue_index; + + /* We hold the read_threads_mutex. */ + if ((cache->index[physical_page] & VOLUME_CACHE_QUEUED_FLAG) == 0) { + /* This page has no existing entry in the queue. */ + if (read_queue_is_full(cache)) + return false; + + /* Fill in the read queue entry. */ + cache->read_queue[last].physical_page = physical_page; + cache->read_queue[last].invalid = false; + cache->read_queue[last].first_request = NULL; + cache->read_queue[last].last_request = NULL; + + /* Point the cache index to the read queue entry. */ + read_queue_index = last; + WRITE_ONCE(cache->index[physical_page], + read_queue_index | VOLUME_CACHE_QUEUED_FLAG); + + advance_queue_position(&cache->read_queue_last); + } else { + /* It's already queued, so add this request to the existing entry. */ + read_queue_index = cache->index[physical_page] & ~VOLUME_CACHE_QUEUED_FLAG; + } + + request->next_request = NULL; + queue_entry = &cache->read_queue[read_queue_index]; + if (queue_entry->first_request == NULL) + queue_entry->first_request = request; + else + queue_entry->last_request->next_request = request; + queue_entry->last_request = request; + + return true; +} + +static void enqueue_page_read(struct volume *volume, struct uds_request *request, + u32 physical_page) +{ + /* Mark the page as queued, so that chapter invalidation knows to cancel a read. */ + while (!enqueue_read(&volume->page_cache, request, physical_page)) { + uds_log_debug("Read queue full, waiting for reads to finish"); + uds_wait_cond(&volume->read_threads_read_done_cond, + &volume->read_threads_mutex); + } + + uds_signal_cond(&volume->read_threads_cond); +} + +/* + * Reserve the next read queue entry for processing, but do not actually remove it from the queue. + * Must be followed by release_queued_requests(). + */ +static struct queued_read *reserve_read_queue_entry(struct page_cache *cache) +{ + /* We hold the read_threads_mutex. */ + struct queued_read *entry; + u16 index_value; + bool queued; + + /* No items to dequeue */ + if (cache->read_queue_next_read == cache->read_queue_last) + return NULL; + + entry = &cache->read_queue[cache->read_queue_next_read]; + index_value = cache->index[entry->physical_page]; + queued = (index_value & VOLUME_CACHE_QUEUED_FLAG) != 0; + /* Check to see if it's still queued before resetting. */ + if (entry->invalid && queued) + WRITE_ONCE(cache->index[entry->physical_page], cache->cache_slots); + + /* + * If a synchronous read has taken this page, set invalid to true so it doesn't get + * overwritten. Requests will just be requeued. + */ + if (!queued) + entry->invalid = true; + + entry->reserved = true; + advance_queue_position(&cache->read_queue_next_read); + return entry; +} + +static inline struct queued_read *wait_to_reserve_read_queue_entry(struct volume *volume) +{ + struct queued_read *queue_entry = NULL; + + while (!volume->read_threads_exiting) { + queue_entry = reserve_read_queue_entry(&volume->page_cache); + if (queue_entry != NULL) + break; + + uds_wait_cond(&volume->read_threads_cond, &volume->read_threads_mutex); + } + + return queue_entry; +} + +static int init_chapter_index_page(const struct volume *volume, u8 *index_page, + u32 chapter, u32 index_page_number, + struct delta_index_page *chapter_index_page) +{ + u64 ci_virtual; + u32 ci_chapter; + u32 lowest_list; + u32 highest_list; + struct geometry *geometry = volume->geometry; + int result; + + result = uds_initialize_chapter_index_page(chapter_index_page, geometry, + index_page, volume->nonce); + if (volume->lookup_mode == LOOKUP_FOR_REBUILD) + return result; + + if (result != UDS_SUCCESS) { + return uds_log_error_strerror(result, + "Reading chapter index page for chapter %u page %u", + chapter, index_page_number); + } + + uds_get_list_number_bounds(volume->index_page_map, chapter, index_page_number, + &lowest_list, &highest_list); + ci_virtual = chapter_index_page->virtual_chapter_number; + ci_chapter = uds_map_to_physical_chapter(geometry, ci_virtual); + if ((chapter == ci_chapter) && + (lowest_list == chapter_index_page->lowest_list_number) && + (highest_list == chapter_index_page->highest_list_number)) + return UDS_SUCCESS; + + uds_log_warning("Index page map updated to %llu", + (unsigned long long) volume->index_page_map->last_update); + uds_log_warning("Page map expects that chapter %u page %u has range %u to %u, but chapter index page has chapter %llu with range %u to %u", + chapter, index_page_number, lowest_list, highest_list, + (unsigned long long) ci_virtual, + chapter_index_page->lowest_list_number, + chapter_index_page->highest_list_number); + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "index page map mismatch with chapter index"); +} + +static int initialize_index_page(const struct volume *volume, u32 physical_page, + struct cached_page *page) +{ + u32 chapter = map_to_chapter_number(volume->geometry, physical_page); + u32 index_page_number = map_to_page_number(volume->geometry, physical_page); + + return init_chapter_index_page(volume, dm_bufio_get_block_data(page->buffer), + chapter, index_page_number, &page->index_page); +} + +static bool search_record_page(const u8 record_page[], + const struct uds_record_name *name, + const struct geometry *geometry, + struct uds_record_data *metadata) +{ + /* + * The array of records is sorted by name and stored as a binary tree in heap order, so the + * root of the tree is the first array element. + */ + u32 node = 0; + const struct uds_volume_record *records = (const struct uds_volume_record *) record_page; + + while (node < geometry->records_per_page) { + int result; + const struct uds_volume_record *record = &records[node]; + + result = memcmp(name, &record->name, UDS_RECORD_NAME_SIZE); + if (result == 0) { + if (metadata != NULL) + *metadata = record->data; + return true; + } + + /* The children of node N are at indexes 2N+1 and 2N+2. */ + node = ((2 * node) + ((result < 0) ? 1 : 2)); + } + + return false; +} + +/* + * If we've read in a record page, we're going to do an immediate search, to speed up processing by + * avoiding get_record_from_zone(), and to ensure that requests make progress even when queued. If + * we've read in an index page, we save the record page number so we don't have to resolve the + * index page again. We use the location, virtual_chapter, and old_metadata fields in the request + * to allow the index code to know where to begin processing the request again. + */ +static int search_page(struct cached_page *page, const struct volume *volume, + struct uds_request *request, u32 physical_page) +{ + int result; + enum uds_index_region location; + u16 record_page_number; + + if (is_record_page(volume->geometry, physical_page)) { + if (search_record_page(dm_bufio_get_block_data(page->buffer), + &request->record_name, volume->geometry, + &request->old_metadata)) + location = UDS_LOCATION_RECORD_PAGE_LOOKUP; + else + location = UDS_LOCATION_UNAVAILABLE; + } else { + result = uds_search_chapter_index_page(&page->index_page, + volume->geometry, + &request->record_name, + &record_page_number); + if (result != UDS_SUCCESS) + return result; + + if (record_page_number == NO_CHAPTER_INDEX_ENTRY) { + location = UDS_LOCATION_UNAVAILABLE; + } else { + location = UDS_LOCATION_INDEX_PAGE_LOOKUP; + *((u16 *) &request->old_metadata) = record_page_number; + } + } + + request->location = location; + request->found = false; + return UDS_SUCCESS; +} + +static int process_entry(struct volume *volume, struct queued_read *entry) +{ + u32 page_number = entry->physical_page; + struct uds_request *request; + struct cached_page *page = NULL; + u8 *page_data; + int result; + + if (entry->invalid) { + uds_log_debug("Requeuing requests for invalid page"); + return UDS_SUCCESS; + } + + page = select_victim_in_cache(&volume->page_cache); + + uds_unlock_mutex(&volume->read_threads_mutex); + page_data = dm_bufio_read(volume->client, page_number, &page->buffer); + if (IS_ERR(page_data)) { + result = -PTR_ERR(page_data); + uds_log_warning_strerror(result, + "error reading physical page %u from volume", + page_number); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return result; + } + uds_lock_mutex(&volume->read_threads_mutex); + + if (entry->invalid) { + uds_log_warning("Page %u invalidated after read", page_number); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return UDS_SUCCESS; + } + + if (!is_record_page(volume->geometry, page_number)) { + result = initialize_index_page(volume, page_number, page); + if (result != UDS_SUCCESS) { + uds_log_warning("Error initializing chapter index page"); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return result; + } + } + + result = put_page_in_cache(&volume->page_cache, page_number, page); + if (result != UDS_SUCCESS) { + uds_log_warning("Error putting page %u in cache", page_number); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return result; + } + + request = entry->first_request; + while ((request != NULL) && (result == UDS_SUCCESS)) { + result = search_page(page, volume, request, page_number); + request = request->next_request; + } + + return result; +} + +static void release_queued_requests(struct volume *volume, struct queued_read *entry, + int result) +{ + struct page_cache *cache = &volume->page_cache; + u16 next_read = cache->read_queue_next_read; + struct uds_request *request; + struct uds_request *next; + + for (request = entry->first_request; request != NULL; request = next) { + next = request->next_request; + request->status = result; + request->requeued = true; + uds_enqueue_request(request, STAGE_INDEX); + } + + entry->reserved = false; + + /* Move the read_queue_first pointer as far as we can. */ + while ((cache->read_queue_first != next_read) && + (!cache->read_queue[cache->read_queue_first].reserved)) + advance_queue_position(&cache->read_queue_first); + uds_broadcast_cond(&volume->read_threads_read_done_cond); +} + +static void read_thread_function(void *arg) +{ + struct volume *volume = arg; + + uds_log_debug("reader starting"); + uds_lock_mutex(&volume->read_threads_mutex); + while (true) { + struct queued_read *queue_entry; + int result; + + queue_entry = wait_to_reserve_read_queue_entry(volume); + if (volume->read_threads_exiting) + break; + + result = process_entry(volume, queue_entry); + release_queued_requests(volume, queue_entry, result); + } + uds_unlock_mutex(&volume->read_threads_mutex); + uds_log_debug("reader done"); +} + +static void get_page_and_index(struct page_cache *cache, u32 physical_page, + int *queue_index, struct cached_page **page_ptr) +{ + u16 index_value; + u16 index; + bool queued; + + /* + * ASSERTION: We are either a zone thread holding a search_pending_counter, or we are any + * thread holding the read_threads_mutex. + * + * Holding only a search_pending_counter is the most frequent case. + */ + /* + * It would be unlikely for the compiler to turn the usage of index_value into two reads of + * cache->index, but it would be possible and very bad if those reads did not return the + * same bits. + */ + index_value = READ_ONCE(cache->index[physical_page]); + queued = (index_value & VOLUME_CACHE_QUEUED_FLAG) != 0; + index = index_value & ~VOLUME_CACHE_QUEUED_FLAG; + + if (!queued && (index < cache->cache_slots)) { + *page_ptr = &cache->cache[index]; + /* + * We have acquired access to the cached page, but unless we hold the + * read_threads_mutex, we need a read memory barrier now. The corresponding write + * memory barrier is in put_page_in_cache(). + */ + smp_rmb(); + } else { + *page_ptr = NULL; + } + + *queue_index = queued ? index : -1; +} + +static void get_page_from_cache(struct page_cache *cache, u32 physical_page, + struct cached_page **page) +{ + /* + * ASSERTION: We are in a zone thread. + * ASSERTION: We holding a search_pending_counter or the read_threads_mutex. + */ + int queue_index = -1; + + get_page_and_index(cache, physical_page, &queue_index, page); +} + +static int read_page_locked(struct volume *volume, u32 physical_page, + struct cached_page **page_ptr) +{ + int result = UDS_SUCCESS; + struct cached_page *page = NULL; + u8 *page_data; + + page = select_victim_in_cache(&volume->page_cache); + page_data = dm_bufio_read(volume->client, physical_page, &page->buffer); + if (IS_ERR(page_data)) { + result = -PTR_ERR(page_data); + uds_log_warning_strerror(result, + "error reading physical page %u from volume", + physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + if (!is_record_page(volume->geometry, physical_page)) { + result = initialize_index_page(volume, physical_page, page); + if (result != UDS_SUCCESS) { + if (volume->lookup_mode != LOOKUP_FOR_REBUILD) + uds_log_warning("Corrupt index page %u", physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + } + + result = put_page_in_cache(&volume->page_cache, physical_page, page); + if (result != UDS_SUCCESS) { + uds_log_warning("Error putting page %u in cache", physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + *page_ptr = page; + return UDS_SUCCESS; +} + +/* Retrieve a page from the cache while holding the read threads mutex. */ +static int get_volume_page_locked(struct volume *volume, u32 physical_page, + struct cached_page **page_ptr) +{ + int result; + struct cached_page *page = NULL; + + get_page_from_cache(&volume->page_cache, physical_page, &page); + if (page == NULL) { + result = read_page_locked(volume, physical_page, &page); + if (result != UDS_SUCCESS) + return result; + } else { + make_page_most_recent(&volume->page_cache, page); + } + + *page_ptr = page; + return UDS_SUCCESS; +} + +/* Retrieve a page from the cache while holding a search_pending lock. */ +static int get_volume_page_protected(struct volume *volume, struct uds_request *request, + u32 physical_page, struct cached_page **page_ptr) +{ + struct cached_page *page; + + get_page_from_cache(&volume->page_cache, physical_page, &page); + if (page != NULL) { + if (request->zone_number == 0) { + /* Only one zone is allowed to update the LRU. */ + make_page_most_recent(&volume->page_cache, page); + } + + *page_ptr = page; + return UDS_SUCCESS; + } + + /* Prepare to enqueue a read for the page. */ + end_pending_search(&volume->page_cache, request->zone_number); + uds_lock_mutex(&volume->read_threads_mutex); + + /* + * Do the lookup again while holding the read mutex (no longer the fast case so this should + * be fine to repeat). We need to do this because a page may have been added to the cache + * by a reader thread between the time we searched above and the time we went to actually + * try to enqueue it below. This could result in us enqueuing another read for a page which + * is already in the cache, which would mean we end up with two entries in the cache for + * the same page. + */ + get_page_from_cache(&volume->page_cache, physical_page, &page); + if (page == NULL) { + enqueue_page_read(volume, request, physical_page); + /* + * The performance gain from unlocking first, while "search pending" mode is off, + * turns out to be significant in some cases. The page is not available yet so + * the order does not matter for correctness as it does below. + */ + uds_unlock_mutex(&volume->read_threads_mutex); + begin_pending_search(&volume->page_cache, physical_page, + request->zone_number); + return UDS_QUEUED; + } + + /* + * Now that the page is loaded, the volume needs to switch to "reader thread unlocked" and + * "search pending" state in careful order so no other thread can mess with the data before + * the caller gets to look at it. + */ + begin_pending_search(&volume->page_cache, physical_page, request->zone_number); + uds_unlock_mutex(&volume->read_threads_mutex); + *page_ptr = page; + return UDS_SUCCESS; +} + +static int get_volume_page(struct volume *volume, u32 chapter, u32 page_number, + struct cached_page **page_ptr) +{ + int result; + u32 physical_page = map_to_physical_page(volume->geometry, chapter, page_number); + + uds_lock_mutex(&volume->read_threads_mutex); + result = get_volume_page_locked(volume, physical_page, page_ptr); + uds_unlock_mutex(&volume->read_threads_mutex); + return result; +} + +int uds_get_volume_record_page(struct volume *volume, u32 chapter, u32 page_number, + u8 **data_ptr) +{ + int result; + struct cached_page *page = NULL; + + result = get_volume_page(volume, chapter, page_number, &page); + if (result == UDS_SUCCESS) + *data_ptr = dm_bufio_get_block_data(page->buffer); + return result; +} + +int uds_get_volume_index_page(struct volume *volume, u32 chapter, u32 page_number, + struct delta_index_page **index_page_ptr) +{ + int result; + struct cached_page *page = NULL; + + result = get_volume_page(volume, chapter, page_number, &page); + if (result == UDS_SUCCESS) + *index_page_ptr = &page->index_page; + return result; +} + +/* + * Find the record page associated with a name in a given index page. This will return UDS_QUEUED + * if the page in question must be read from storage. + */ +static int search_cached_index_page(struct volume *volume, struct uds_request *request, + u32 chapter, u32 index_page_number, + u16 *record_page_number) +{ + int result; + struct cached_page *page = NULL; + u32 physical_page = map_to_physical_page(volume->geometry, chapter, + index_page_number); + + /* + * Make sure the invalidate counter is updated before we try and read the mapping. This + * prevents this thread from reading a page in the cache which has already been marked for + * invalidation by the reader thread, before the reader thread has noticed that the + * invalidate_counter has been incremented. + */ + begin_pending_search(&volume->page_cache, physical_page, request->zone_number); + + result = get_volume_page_protected(volume, request, physical_page, &page); + if (result != UDS_SUCCESS) { + end_pending_search(&volume->page_cache, request->zone_number); + return result; + } + + result = uds_search_chapter_index_page(&page->index_page, volume->geometry, + &request->record_name, + record_page_number); + end_pending_search(&volume->page_cache, request->zone_number); + return result; +} + +/* + * Find the metadata associated with a name in a given record page. This will return UDS_QUEUED if + * the page in question must be read from storage. + */ +int uds_search_cached_record_page(struct volume *volume, struct uds_request *request, + u32 chapter, u16 record_page_number, bool *found) +{ + struct cached_page *record_page; + struct geometry *geometry = volume->geometry; + int result; + u32 physical_page, page_number; + + *found = false; + if (record_page_number == NO_CHAPTER_INDEX_ENTRY) + return UDS_SUCCESS; + + result = ASSERT(record_page_number < geometry->record_pages_per_chapter, + "0 <= %d < %u", record_page_number, + geometry->record_pages_per_chapter); + if (result != UDS_SUCCESS) + return result; + + page_number = geometry->index_pages_per_chapter + record_page_number; + + physical_page = map_to_physical_page(volume->geometry, chapter, page_number); + + /* + * Make sure the invalidate counter is updated before we try and read the mapping. This + * prevents this thread from reading a page in the cache which has already been marked for + * invalidation by the reader thread, before the reader thread has noticed that the + * invalidate_counter has been incremented. + */ + begin_pending_search(&volume->page_cache, physical_page, request->zone_number); + + result = get_volume_page_protected(volume, request, physical_page, &record_page); + if (result != UDS_SUCCESS) { + end_pending_search(&volume->page_cache, request->zone_number); + return result; + } + + if (search_record_page(dm_bufio_get_block_data(record_page->buffer), + &request->record_name, geometry, &request->old_metadata)) + *found = true; + + end_pending_search(&volume->page_cache, request->zone_number); + return UDS_SUCCESS; +} + +void uds_prefetch_volume_chapter(const struct volume *volume, u32 chapter) +{ + const struct geometry *geometry = volume->geometry; + u32 physical_page = map_to_physical_page(geometry, chapter, 0); + + dm_bufio_prefetch(volume->client, physical_page, geometry->pages_per_chapter); +} + +int uds_read_chapter_index_from_volume(const struct volume *volume, u64 virtual_chapter, + struct dm_buffer *volume_buffers[], + struct delta_index_page index_pages[]) +{ + int result; + u32 i; + const struct geometry *geometry = volume->geometry; + u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter); + u32 physical_page = map_to_physical_page(geometry, physical_chapter, 0); + + dm_bufio_prefetch(volume->client, physical_page, geometry->index_pages_per_chapter); + for (i = 0; i < geometry->index_pages_per_chapter; i++) { + u8 *index_page; + + index_page = dm_bufio_read(volume->client, physical_page + i, + &volume_buffers[i]); + if (IS_ERR(index_page)) { + result = -PTR_ERR(index_page); + uds_log_warning_strerror(result, + "error reading physical page %u", + physical_page); + return result; + } + + result = init_chapter_index_page(volume, index_page, physical_chapter, i, + &index_pages[i]); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +int uds_search_volume_page_cache(struct volume *volume, struct uds_request *request, + bool *found) +{ + int result; + u32 physical_chapter = + uds_map_to_physical_chapter(volume->geometry, request->virtual_chapter); + u32 index_page_number; + u16 record_page_number; + + index_page_number = uds_find_index_page_number(volume->index_page_map, + &request->record_name, + physical_chapter); + + if (request->location == UDS_LOCATION_INDEX_PAGE_LOOKUP) { + record_page_number = *((u16 *) &request->old_metadata); + } else { + result = search_cached_index_page(volume, request, physical_chapter, + index_page_number, + &record_page_number); + if (result != UDS_SUCCESS) + return result; + } + + return uds_search_cached_record_page(volume, request, physical_chapter, + record_page_number, found); +} + +int uds_search_volume_page_cache_for_rebuild(struct volume *volume, + const struct uds_record_name *name, + u64 virtual_chapter, bool *found) +{ + int result; + struct geometry *geometry = volume->geometry; + struct cached_page *page; + u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter); + u32 index_page_number; + u16 record_page_number; + u32 page_number; + + *found = false; + index_page_number = + uds_find_index_page_number(volume->index_page_map, name, + physical_chapter); + result = get_volume_page(volume, physical_chapter, index_page_number, &page); + if (result != UDS_SUCCESS) + return result; + + result = uds_search_chapter_index_page(&page->index_page, geometry, name, + &record_page_number); + if (result != UDS_SUCCESS) + return result; + + if (record_page_number == NO_CHAPTER_INDEX_ENTRY) + return UDS_SUCCESS; + + page_number = geometry->index_pages_per_chapter + record_page_number; + result = get_volume_page(volume, physical_chapter, page_number, &page); + if (result != UDS_SUCCESS) + return result; + + *found = search_record_page(dm_bufio_get_block_data(page->buffer), name, + geometry, NULL); + return UDS_SUCCESS; +} + +static void invalidate_page(struct page_cache *cache, u32 physical_page) +{ + struct cached_page *page; + int queue_index = -1; + + /* We hold the read_threads_mutex. */ + get_page_and_index(cache, physical_page, &queue_index, &page); + if (page != NULL) { + WRITE_ONCE(cache->index[page->physical_page], cache->cache_slots); + wait_for_pending_searches(cache, page->physical_page); + clear_cache_page(cache, page); + } else if (queue_index > -1) { + uds_log_debug("setting pending read to invalid"); + cache->read_queue[queue_index].invalid = true; + } +} + +void uds_forget_chapter(struct volume *volume, u64 virtual_chapter) +{ + u32 physical_chapter = + uds_map_to_physical_chapter(volume->geometry, virtual_chapter); + u32 first_page = map_to_physical_page(volume->geometry, physical_chapter, 0); + u32 i; + + uds_log_debug("forgetting chapter %llu", (unsigned long long) virtual_chapter); + uds_lock_mutex(&volume->read_threads_mutex); + for (i = 0; i < volume->geometry->pages_per_chapter; i++) + invalidate_page(&volume->page_cache, first_page + i); + uds_unlock_mutex(&volume->read_threads_mutex); +} + +/* + * Donate an index pages from a newly written chapter to the page cache since it is likely to be + * used again soon. The caller must already hold the reader thread mutex. + */ +static int donate_index_page_locked(struct volume *volume, u32 physical_chapter, + u32 index_page_number, struct dm_buffer *page_buffer) +{ + int result; + struct cached_page *page = NULL; + u32 physical_page = + map_to_physical_page(volume->geometry, physical_chapter, + index_page_number); + + page = select_victim_in_cache(&volume->page_cache); + page->buffer = page_buffer; + result = init_chapter_index_page(volume, dm_bufio_get_block_data(page_buffer), + physical_chapter, index_page_number, + &page->index_page); + if (result != UDS_SUCCESS) { + uds_log_warning("Error initialize chapter index page"); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + result = put_page_in_cache(&volume->page_cache, physical_page, page); + if (result != UDS_SUCCESS) { + uds_log_warning("Error putting page %u in cache", physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + return UDS_SUCCESS; +} + +static int write_index_pages(struct volume *volume, u32 physical_chapter_number, + struct open_chapter_index *chapter_index) +{ + struct geometry *geometry = volume->geometry; + struct dm_buffer *page_buffer; + u32 first_index_page = map_to_physical_page(geometry, physical_chapter_number, 0); + u32 delta_list_number = 0; + u32 index_page_number; + + for (index_page_number = 0; + index_page_number < geometry->index_pages_per_chapter; + index_page_number++) { + u8 *page_data; + u32 physical_page = first_index_page + index_page_number; + u32 lists_packed; + bool last_page; + int result; + + page_data = dm_bufio_new(volume->client, physical_page, &page_buffer); + if (IS_ERR(page_data)) { + return uds_log_warning_strerror(-PTR_ERR(page_data), + "failed to prepare index page"); + } + + last_page = ((index_page_number + 1) == geometry->index_pages_per_chapter); + result = uds_pack_open_chapter_index_page(chapter_index, page_data, + delta_list_number, last_page, + &lists_packed); + if (result != UDS_SUCCESS) { + dm_bufio_release(page_buffer); + return uds_log_warning_strerror(result, + "failed to pack index page"); + } + + dm_bufio_mark_buffer_dirty(page_buffer); + + if (lists_packed == 0) { + uds_log_debug("no delta lists packed on chapter %u page %u", + physical_chapter_number, index_page_number); + } else { + delta_list_number += lists_packed; + } + + uds_update_index_page_map(volume->index_page_map, + chapter_index->virtual_chapter_number, + physical_chapter_number, index_page_number, + delta_list_number - 1); + + uds_lock_mutex(&volume->read_threads_mutex); + result = donate_index_page_locked(volume, physical_chapter_number, + index_page_number, page_buffer); + uds_unlock_mutex(&volume->read_threads_mutex); + if (result != UDS_SUCCESS) { + dm_bufio_release(page_buffer); + return result; + } + } + + return UDS_SUCCESS; +} + +static u32 encode_tree(u8 record_page[], + const struct uds_volume_record *sorted_pointers[], + u32 next_record, u32 node, u32 node_count) +{ + if (node < node_count) { + u32 child = (2 * node) + 1; + + next_record = encode_tree(record_page, sorted_pointers, next_record, + child, node_count); + + /* + * In-order traversal: copy the contents of the next record into the page at the + * node offset. + */ + memcpy(&record_page[node * BYTES_PER_RECORD], + sorted_pointers[next_record++], BYTES_PER_RECORD); + + next_record = encode_tree(record_page, sorted_pointers, next_record, + child + 1, node_count); + } + + return next_record; +} + +static int encode_record_page(const struct volume *volume, + const struct uds_volume_record records[], u8 record_page[]) +{ + int result; + u32 i; + u32 records_per_page = volume->geometry->records_per_page; + const struct uds_volume_record **record_pointers = volume->record_pointers; + + for (i = 0; i < records_per_page; i++) + record_pointers[i] = &records[i]; + + /* + * Sort the record pointers by using just the names in the records, which is less work than + * sorting the entire record values. + */ + BUILD_BUG_ON(offsetof(struct uds_volume_record, name) != 0); + result = uds_radix_sort(volume->radix_sorter, (const u8 **) record_pointers, + records_per_page, UDS_RECORD_NAME_SIZE); + if (result != UDS_SUCCESS) + return result; + + encode_tree(record_page, record_pointers, 0, 0, records_per_page); + return UDS_SUCCESS; +} + +static int write_record_pages(struct volume *volume, u32 physical_chapter_number, + const struct uds_volume_record *records) +{ + u32 record_page_number; + struct geometry *geometry = volume->geometry; + struct dm_buffer *page_buffer; + const struct uds_volume_record *next_record = records; + u32 first_record_page = map_to_physical_page(geometry, physical_chapter_number, + geometry->index_pages_per_chapter); + + for (record_page_number = 0; + record_page_number < geometry->record_pages_per_chapter; + record_page_number++) { + u8 *page_data; + u32 physical_page = first_record_page + record_page_number; + int result; + + page_data = dm_bufio_new(volume->client, physical_page, &page_buffer); + if (IS_ERR(page_data)) { + return uds_log_warning_strerror(-PTR_ERR(page_data), + "failed to prepare record page"); + } + + result = encode_record_page(volume, next_record, page_data); + if (result != UDS_SUCCESS) { + dm_bufio_release(page_buffer); + return uds_log_warning_strerror(result, + "failed to encode record page %u", + record_page_number); + } + + next_record += geometry->records_per_page; + dm_bufio_mark_buffer_dirty(page_buffer); + dm_bufio_release(page_buffer); + } + + return UDS_SUCCESS; +} + +int uds_write_chapter(struct volume *volume, struct open_chapter_index *chapter_index, + const struct uds_volume_record *records) +{ + int result; + u32 physical_chapter_number = + uds_map_to_physical_chapter(volume->geometry, + chapter_index->virtual_chapter_number); + + result = write_index_pages(volume, physical_chapter_number, chapter_index); + if (result != UDS_SUCCESS) + return result; + + result = write_record_pages(volume, physical_chapter_number, records); + if (result != UDS_SUCCESS) + return result; + + result = -dm_bufio_write_dirty_buffers(volume->client); + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "cannot sync chapter to volume"); + + return result; +} + +static void probe_chapter(struct volume *volume, u32 chapter_number, + u64 *virtual_chapter_number) +{ + const struct geometry *geometry = volume->geometry; + u32 expected_list_number = 0; + u32 i; + u64 vcn = BAD_CHAPTER; + + *virtual_chapter_number = BAD_CHAPTER; + dm_bufio_prefetch(volume->client, + map_to_physical_page(geometry, chapter_number, 0), + geometry->index_pages_per_chapter); + + for (i = 0; i < geometry->index_pages_per_chapter; i++) { + struct delta_index_page *page; + int result; + + result = uds_get_volume_index_page(volume, chapter_number, i, &page); + if (result != UDS_SUCCESS) + return; + + if (page->virtual_chapter_number == BAD_CHAPTER) { + uds_log_error("corrupt index page in chapter %u", + chapter_number); + return; + } + + if (vcn == BAD_CHAPTER) { + vcn = page->virtual_chapter_number; + } else if (page->virtual_chapter_number != vcn) { + uds_log_error("inconsistent chapter %u index page %u: expected vcn %llu, got vcn %llu", + chapter_number, i, (unsigned long long) vcn, + (unsigned long long) page->virtual_chapter_number); + return; + } + + if (expected_list_number != page->lowest_list_number) { + uds_log_error("inconsistent chapter %u index page %u: expected list number %u, got list number %u", + chapter_number, i, expected_list_number, + page->lowest_list_number); + return; + } + expected_list_number = page->highest_list_number + 1; + + result = uds_validate_chapter_index_page(page, geometry); + if (result != UDS_SUCCESS) + return; + } + + if (chapter_number != uds_map_to_physical_chapter(geometry, vcn)) { + uds_log_error("chapter %u vcn %llu is out of phase (%u)", chapter_number, + (unsigned long long) vcn, geometry->chapters_per_volume); + return; + } + + *virtual_chapter_number = vcn; +} + +/* Find the last valid physical chapter in the volume. */ +static void find_real_end_of_volume(struct volume *volume, u32 limit, u32 *limit_ptr) +{ + u32 span = 1; + u32 tries = 0; + + while (limit > 0) { + u32 chapter = (span > limit) ? 0 : limit - span; + u64 vcn = 0; + + probe_chapter(volume, chapter, &vcn); + if (vcn == BAD_CHAPTER) { + limit = chapter; + if (++tries > 1) + span *= 2; + } else { + if (span == 1) + break; + span /= 2; + tries = 0; + } + } + + *limit_ptr = limit; +} + +static int find_chapter_limits(struct volume *volume, u32 chapter_limit, u64 *lowest_vcn, + u64 *highest_vcn) +{ + struct geometry *geometry = volume->geometry; + u64 zero_vcn; + u64 lowest = BAD_CHAPTER; + u64 highest = BAD_CHAPTER; + u64 moved_chapter = BAD_CHAPTER; + u32 left_chapter = 0; + u32 right_chapter = 0; + u32 bad_chapters = 0; + + /* + * This method assumes there is at most one run of contiguous bad chapters caused by + * unflushed writes. Either the bad spot is at the beginning and end, or somewhere in the + * middle. Wherever it is, the highest and lowest VCNs are adjacent to it. Otherwise the + * volume is cleanly saved and somewhere in the middle of it the highest VCN immediately + * precedes the lowest one. + */ + + /* It doesn't matter if this results in a bad spot (BAD_CHAPTER). */ + probe_chapter(volume, 0, &zero_vcn); + + /* + * Binary search for end of the discontinuity in the monotonically increasing virtual + * chapter numbers; bad spots are treated as a span of BAD_CHAPTER values. In effect we're + * searching for the index of the smallest value less than zero_vcn. In the case we go off + * the end it means that chapter 0 has the lowest vcn. + * + * If a virtual chapter is out-of-order, it will be the one moved by conversion. Always + * skip over the moved chapter when searching, adding it to the range at the end if + * necessary. + */ + if (geometry->remapped_physical > 0) { + u64 remapped_vcn; + + probe_chapter(volume, geometry->remapped_physical, &remapped_vcn); + if (remapped_vcn == geometry->remapped_virtual) + moved_chapter = geometry->remapped_physical; + } + + left_chapter = 0; + right_chapter = chapter_limit; + + while (left_chapter < right_chapter) { + u64 probe_vcn; + u32 chapter = (left_chapter + right_chapter) / 2; + + if (chapter == moved_chapter) + chapter--; + + probe_chapter(volume, chapter, &probe_vcn); + if (zero_vcn <= probe_vcn) { + left_chapter = chapter + 1; + if (left_chapter == moved_chapter) + left_chapter++; + } else { + right_chapter = chapter; + } + } + + /* If left_chapter goes off the end, chapter 0 has the lowest virtual chapter number.*/ + if (left_chapter >= chapter_limit) + left_chapter = 0; + + /* At this point, left_chapter is the chapter with the lowest virtual chapter number. */ + probe_chapter(volume, left_chapter, &lowest); + + /* The moved chapter might be the lowest in the range. */ + if ((moved_chapter != BAD_CHAPTER) && (lowest == geometry->remapped_virtual + 1)) + lowest = geometry->remapped_virtual; + + /* + * Circularly scan backwards, moving over any bad chapters until encountering a good one, + * which is the chapter with the highest vcn. + */ + while (highest == BAD_CHAPTER) { + right_chapter = (right_chapter + chapter_limit - 1) % chapter_limit; + if (right_chapter == moved_chapter) + continue; + + probe_chapter(volume, right_chapter, &highest); + if (bad_chapters++ >= MAX_BAD_CHAPTERS) { + uds_log_error("too many bad chapters in volume: %u", + bad_chapters); + return UDS_CORRUPT_DATA; + } + } + + *lowest_vcn = lowest; + *highest_vcn = highest; + return UDS_SUCCESS; +} + +/* + * Find the highest and lowest contiguous chapters present in the volume and determine their + * virtual chapter numbers. This is used by rebuild. + */ +int uds_find_volume_chapter_boundaries(struct volume *volume, u64 *lowest_vcn, + u64 *highest_vcn, bool *is_empty) +{ + u32 chapter_limit = volume->geometry->chapters_per_volume; + + find_real_end_of_volume(volume, chapter_limit, &chapter_limit); + if (chapter_limit == 0) { + *lowest_vcn = 0; + *highest_vcn = 0; + *is_empty = true; + return UDS_SUCCESS; + } + + *is_empty = false; + return find_chapter_limits(volume, chapter_limit, lowest_vcn, highest_vcn); +} + +int __must_check uds_replace_volume_storage(struct volume *volume, + struct index_layout *layout, + struct block_device *bdev) +{ + int result; + u32 i; + + result = uds_replace_index_layout_storage(layout, bdev); + if (result != UDS_SUCCESS) + return result; + + /* Release all outstanding dm_bufio objects */ + for (i = 0; i < volume->page_cache.indexable_pages; i++) + volume->page_cache.index[i] = volume->page_cache.cache_slots; + for (i = 0; i < volume->page_cache.cache_slots; i++) + clear_cache_page(&volume->page_cache, &volume->page_cache.cache[i]); + if (volume->sparse_cache != NULL) + uds_invalidate_sparse_cache(volume->sparse_cache); + if (volume->client != NULL) + dm_bufio_client_destroy(uds_forget(volume->client)); + + return uds_open_volume_bufio(layout, volume->geometry->bytes_per_page, + volume->reserved_buffers, &volume->client); +} + +static int __must_check initialize_page_cache(struct page_cache *cache, + const struct geometry *geometry, + u32 chapters_in_cache, + unsigned int zone_count) +{ + int result; + u32 i; + + cache->indexable_pages = geometry->pages_per_volume + 1; + cache->cache_slots = chapters_in_cache * geometry->record_pages_per_chapter; + cache->zone_count = zone_count; + atomic64_set(&cache->clock, 1); + + result = ASSERT((cache->cache_slots <= VOLUME_CACHE_MAX_ENTRIES), + "requested cache size, %u, within limit %u", + cache->cache_slots, VOLUME_CACHE_MAX_ENTRIES); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read, + "volume read queue", &cache->read_queue); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(cache->zone_count, struct search_pending_counter, + "Volume Cache Zones", &cache->search_pending_counters); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(cache->indexable_pages, u16, "page cache index", + &cache->index); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate(cache->cache_slots, struct cached_page, "page cache cache", + &cache->cache); + if (result != UDS_SUCCESS) + return result; + + /* Initialize index values to invalid values. */ + for (i = 0; i < cache->indexable_pages; i++) + cache->index[i] = cache->cache_slots; + + for (i = 0; i < cache->cache_slots; i++) + clear_cache_page(cache, &cache->cache[i]); + + return UDS_SUCCESS; +} + +int uds_make_volume(const struct configuration *config, struct index_layout *layout, + struct volume **new_volume) +{ + unsigned int i; + struct volume *volume = NULL; + struct geometry *geometry; + unsigned int reserved_buffers; + int result; + + result = uds_allocate(1, struct volume, "volume", &volume); + if (result != UDS_SUCCESS) + return result; + + volume->nonce = uds_get_volume_nonce(layout); + + result = uds_copy_geometry(config->geometry, &volume->geometry); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return uds_log_warning_strerror(result, + "failed to allocate geometry: error"); + } + geometry = volume->geometry; + + /* + * Reserve a buffer for each entry in the page cache, one for the chapter writer, and one + * for each entry in the sparse cache. + */ + reserved_buffers = config->cache_chapters * geometry->record_pages_per_chapter; + reserved_buffers += 1; + if (uds_is_sparse_geometry(geometry)) + reserved_buffers += (config->cache_chapters * geometry->index_pages_per_chapter); + volume->reserved_buffers = reserved_buffers; + result = uds_open_volume_bufio(layout, geometry->bytes_per_page, + volume->reserved_buffers, &volume->client); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_make_radix_sorter(geometry->records_per_page, + &volume->radix_sorter); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_allocate(geometry->records_per_page, + const struct uds_volume_record *, "record pointers", + &volume->record_pointers); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + if (uds_is_sparse_geometry(geometry)) { + size_t page_size = sizeof(struct delta_index_page) + geometry->bytes_per_page; + + result = uds_make_sparse_cache(geometry, config->cache_chapters, + config->zone_count, + &volume->sparse_cache); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + volume->cache_size = + page_size * geometry->index_pages_per_chapter * config->cache_chapters; + } + + result = initialize_page_cache(&volume->page_cache, geometry, + config->cache_chapters, config->zone_count); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + volume->cache_size += volume->page_cache.cache_slots * sizeof(struct delta_index_page); + result = uds_make_index_page_map(geometry, &volume->index_page_map); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_init_mutex(&volume->read_threads_mutex); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_init_cond(&volume->read_threads_read_done_cond); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_init_cond(&volume->read_threads_cond); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_allocate(config->read_threads, struct thread *, "reader threads", + &volume->reader_threads); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + for (i = 0; i < config->read_threads; i++) { + result = uds_create_thread(read_thread_function, (void *) volume, + "reader", &volume->reader_threads[i]); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + volume->read_thread_count = i + 1; + } + + *new_volume = volume; + return UDS_SUCCESS; +} + +static void uninitialize_page_cache(struct page_cache *cache) +{ + u16 i; + + if (cache->cache != NULL) { + for (i = 0; i < cache->cache_slots; i++) + release_page_buffer(&cache->cache[i]); + } + uds_free(cache->index); + uds_free(cache->cache); + uds_free(cache->search_pending_counters); + uds_free(cache->read_queue); +} + +void uds_free_volume(struct volume *volume) +{ + if (volume == NULL) + return; + + if (volume->reader_threads != NULL) { + unsigned int i; + + /* This works even if some threads weren't started. */ + uds_lock_mutex(&volume->read_threads_mutex); + volume->read_threads_exiting = true; + uds_broadcast_cond(&volume->read_threads_cond); + uds_unlock_mutex(&volume->read_threads_mutex); + for (i = 0; i < volume->read_thread_count; i++) + uds_join_threads(volume->reader_threads[i]); + uds_free(volume->reader_threads); + volume->reader_threads = NULL; + } + + /* Must destroy the client AFTER freeing the cached pages. */ + uninitialize_page_cache(&volume->page_cache); + uds_free_sparse_cache(volume->sparse_cache); + if (volume->client != NULL) + dm_bufio_client_destroy(uds_forget(volume->client)); + + uds_destroy_cond(&volume->read_threads_cond); + uds_destroy_cond(&volume->read_threads_read_done_cond); + uds_destroy_mutex(&volume->read_threads_mutex); + uds_free_index_page_map(volume->index_page_map); + uds_free_radix_sorter(volume->radix_sorter); + uds_free(volume->geometry); + uds_free(volume->record_pointers); + uds_free(volume); +} diff --git a/drivers/md/dm-vdo/volume.h b/drivers/md/dm-vdo/volume.h new file mode 100644 index 000000000000..0a96af964887 --- /dev/null +++ b/drivers/md/dm-vdo/volume.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_VOLUME_H +#define UDS_VOLUME_H + +#include +#include +#include +#include + +#include "chapter-index.h" +#include "config.h" +#include "geometry.h" +#include "index-layout.h" +#include "index-page-map.h" +#include "permassert.h" +#include "radix-sort.h" +#include "sparse-cache.h" +#include "uds.h" +#include "uds-threads.h" + +/* + * The volume manages deduplication records on permanent storage. The term "volume" can also refer + * to the region of permanent storage where the records (and the chapters containing them) are + * stored. The volume handles all I/O to this region by reading, caching, and writing chapter pages + * as necessary. + */ + +enum index_lookup_mode { + /* Always do lookups in all chapters normally */ + LOOKUP_NORMAL, + /* Only do a subset of lookups needed when rebuilding an index */ + LOOKUP_FOR_REBUILD, +}; + +struct queued_read { + bool invalid; + bool reserved; + u32 physical_page; + struct uds_request *first_request; + struct uds_request *last_request; +}; + +struct __aligned(L1_CACHE_BYTES) search_pending_counter { + u64 atomic_value; +}; + +struct cached_page { + /* Whether this page is currently being read asynchronously */ + bool read_pending; + /* The physical page stored in this cache entry */ + u32 physical_page; + /* The value of the volume clock when this page was last used */ + s64 last_used; + /* The cached page buffer */ + struct dm_buffer *buffer; + /* The chapter index page, meaningless for record pages */ + struct delta_index_page index_page; +}; + +struct page_cache { + /* The number of zones */ + unsigned int zone_count; + /* The number of volume pages that can be cached */ + u32 indexable_pages; + /* The maximum number of simultaneously cached pages */ + u16 cache_slots; + /* An index for each physical page noting where it is in the cache */ + u16 *index; + /* The array of cached pages */ + struct cached_page *cache; + /* A counter for each zone tracking if a search is occurring there */ + struct search_pending_counter *search_pending_counters; + /* The read queue entries as a circular array */ + struct queued_read *read_queue; + + /* All entries above this point are constant after initialization. */ + + /* + * These values are all indexes into the array of read queue entries. New entries in the + * read queue are enqueued at read_queue_last. To dequeue entries, a reader thread gets the + * lock and then claims the entry pointed to by read_queue_next_read and increments that + * value. After the read is completed, the reader thread calls release_read_queue_entry(), + * which increments read_queue_first until it points to a pending read, or is equal to + * read_queue_next_read. This means that if multiple reads are outstanding, + * read_queue_first might not advance until the last of the reads finishes. + */ + u16 read_queue_first; + u16 read_queue_next_read; + u16 read_queue_last; + + atomic64_t clock; +}; + +struct volume { + struct geometry *geometry; + struct dm_bufio_client *client; + u64 nonce; + size_t cache_size; + + /* A single page worth of records, for sorting */ + const struct uds_volume_record **record_pointers; + /* Sorter for sorting records within each page */ + struct radix_sorter *radix_sorter; + + struct sparse_cache *sparse_cache; + struct page_cache page_cache; + struct index_page_map *index_page_map; + + struct mutex read_threads_mutex; + struct cond_var read_threads_cond; + struct cond_var read_threads_read_done_cond; + struct thread **reader_threads; + unsigned int read_thread_count; + bool read_threads_exiting; + + enum index_lookup_mode lookup_mode; + unsigned int reserved_buffers; +}; + +int __must_check uds_make_volume(const struct configuration *config, + struct index_layout *layout, + struct volume **new_volume); + +void uds_free_volume(struct volume *volume); + +int __must_check uds_replace_volume_storage(struct volume *volume, + struct index_layout *layout, + struct block_device *bdev); + +int __must_check uds_find_volume_chapter_boundaries(struct volume *volume, + u64 *lowest_vcn, u64 *highest_vcn, + bool *is_empty); + +int __must_check uds_search_volume_page_cache(struct volume *volume, + struct uds_request *request, + bool *found); + +int __must_check uds_search_volume_page_cache_for_rebuild(struct volume *volume, + const struct uds_record_name *name, + u64 virtual_chapter, + bool *found); + +int __must_check uds_search_cached_record_page(struct volume *volume, + struct uds_request *request, u32 chapter, + u16 record_page_number, bool *found); + +void uds_forget_chapter(struct volume *volume, u64 chapter); + +int __must_check uds_write_chapter(struct volume *volume, + struct open_chapter_index *chapter_index, + const struct uds_volume_record records[]); + +void uds_prefetch_volume_chapter(const struct volume *volume, u32 chapter); + +int __must_check uds_read_chapter_index_from_volume(const struct volume *volume, + u64 virtual_chapter, + struct dm_buffer *volume_buffers[], + struct delta_index_page index_pages[]); + +int __must_check uds_get_volume_record_page(struct volume *volume, u32 chapter, + u32 page_number, u8 **data_ptr); + +int __must_check uds_get_volume_index_page(struct volume *volume, u32 chapter, + u32 page_number, + struct delta_index_page **page_ptr); + +#endif /* UDS_VOLUME_H */ From 766130e8a9f358652a43dd7ab3edfc6fca5b9b73 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:56:07 -0500 Subject: [PATCH 0563/1038] dm vdo: implement top-level deduplication index The top-level deduplication index brings all the earlier components together. The top-level index creates the separate zone structures that enable the index to handle several requests in parallel, handles dispatching requests to the right zones and components, and coordinates metadata to ensure that it remain consistent. It also coordinates recovery in the event of an unexpected index failure. If sparse caching is enabled, the top-level index also handles the coordination required by the sparse chapter index cache, which (unlike most index structures) is shared among all zones. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/index.c | 1400 ++++++++++++++++++++++++++++++ drivers/md/dm-vdo/index.h | 83 ++ drivers/md/dm-vdo/sparse-cache.c | 580 +++++++++++++ drivers/md/dm-vdo/sparse-cache.h | 46 + 4 files changed, 2109 insertions(+) create mode 100644 drivers/md/dm-vdo/index.c create mode 100644 drivers/md/dm-vdo/index.h create mode 100644 drivers/md/dm-vdo/sparse-cache.c create mode 100644 drivers/md/dm-vdo/sparse-cache.h diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c new file mode 100644 index 000000000000..28ae54efaced --- /dev/null +++ b/drivers/md/dm-vdo/index.c @@ -0,0 +1,1400 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + + +#include "index.h" + +#include "funnel-requestqueue.h" +#include "hash-utils.h" +#include "logger.h" +#include "memory-alloc.h" +#include "sparse-cache.h" + +static const u64 NO_LAST_SAVE = U64_MAX; + +/* + * When searching for deduplication records, the index first searches the volume index, and then + * searches the chapter index for the relevant chapter. If the chapter has been fully committed to + * storage, the chapter pages are loaded into the page cache. If the chapter has not yet been + * committed (either the open chapter or a recently closed one), the index searches the in-memory + * representation of the chapter. Finally, if the volume index does not find a record and the index + * is sparse, the index will search the sparse cache. + * + * The index send two kinds of messages to coordinate between zones: chapter close messages for the + * chapter writer, and sparse cache barrier messages for the sparse cache. + * + * The chapter writer is responsible for committing chapters of records to storage. Since zones can + * get different numbers of records, some zones may fall behind others. Each time a zone fills up + * its available space in a chapter, it informs the chapter writer that the chapter is complete, + * and also informs all other zones that it has closed the chapter. Each other zone will then close + * the chapter immediately, regardless of how full it is, in order to minimize skew between zones. + * Once every zone has closed the chapter, the chapter writer will commit that chapter to storage. + * + * The last zone to close the chapter also removes the oldest chapter from the volume index. + * Although that chapter is invalid for zones that have moved on, the existence of the open chapter + * means that those zones will never ask the volume index about it. No zone is allowed to get more + * than one chapter ahead of any other. If a zone is so far ahead that it tries to close another + * chapter before the previous one has been closed by all zones, it is forced to wait. + * + * The sparse cache relies on having the same set of chapter indexes available to all zones. When a + * request wants to add a chapter to the sparse cache, it sends a barrier message to each zone + * during the triage stage that acts as a rendezvous. Once every zone has reached the barrier and + * paused its operations, the cache membership is changed and each zone is then informed that it + * can proceed. More details can be found in the sparse cache documentation. + * + * If a sparse cache has only one zone, it will not create a triage queue, but it still needs the + * barrier message to change the sparse cache membership, so the index simulates the message by + * invoking the handler directly. + */ + +struct chapter_writer { + /* The index to which we belong */ + struct uds_index *index; + /* The thread to do the writing */ + struct thread *thread; + /* The lock protecting the following fields */ + struct mutex mutex; + /* The condition signalled on state changes */ + struct cond_var cond; + /* Set to true to stop the thread */ + bool stop; + /* The result from the most recent write */ + int result; + /* The number of bytes allocated by the chapter writer */ + size_t memory_size; + /* The number of zones which have submitted a chapter for writing */ + unsigned int zones_to_write; + /* Open chapter index used by uds_close_open_chapter() */ + struct open_chapter_index *open_chapter_index; + /* Collated records used by uds_close_open_chapter() */ + struct uds_volume_record *collated_records; + /* The chapters to write (one per zone) */ + struct open_chapter_zone *chapters[]; +}; + +static bool is_zone_chapter_sparse(const struct index_zone *zone, u64 virtual_chapter) +{ + return uds_is_chapter_sparse(zone->index->volume->geometry, + zone->oldest_virtual_chapter, + zone->newest_virtual_chapter, virtual_chapter); +} + +static int launch_zone_message(struct uds_zone_message message, unsigned int zone, + struct uds_index *index) +{ + int result; + struct uds_request *request; + + result = uds_allocate(1, struct uds_request, __func__, &request); + if (result != UDS_SUCCESS) + return result; + + request->index = index; + request->unbatched = true; + request->zone_number = zone; + request->zone_message = message; + + uds_enqueue_request(request, STAGE_MESSAGE); + return UDS_SUCCESS; +} + +static void enqueue_barrier_messages(struct uds_index *index, u64 virtual_chapter) +{ + struct uds_zone_message message = { + .type = UDS_MESSAGE_SPARSE_CACHE_BARRIER, + .virtual_chapter = virtual_chapter, + }; + unsigned int zone; + + for (zone = 0; zone < index->zone_count; zone++) { + int result = launch_zone_message(message, zone, index); + + ASSERT_LOG_ONLY((result == UDS_SUCCESS), "barrier message allocation"); + } +} + +/* + * Determine whether this request should trigger a sparse cache barrier message to change the + * membership of the sparse cache. If a change in membership is desired, the function returns the + * chapter number to add. + */ +static u64 triage_index_request(struct uds_index *index, struct uds_request *request) +{ + u64 virtual_chapter; + struct index_zone *zone; + + virtual_chapter = uds_lookup_volume_index_name(index->volume_index, + &request->record_name); + if (virtual_chapter == NO_CHAPTER) + return NO_CHAPTER; + + zone = index->zones[request->zone_number]; + if (!is_zone_chapter_sparse(zone, virtual_chapter)) + return NO_CHAPTER; + + /* + * FIXME: Optimize for a common case by remembering the chapter from the most recent + * barrier message and skipping this chapter if is it the same. + */ + + return virtual_chapter; +} + +/* + * Simulate a message to change the sparse cache membership for a single-zone sparse index. This + * allows us to forgo the complicated locking required by a multi-zone sparse index. Any other kind + * of index does nothing here. + */ +static int simulate_index_zone_barrier_message(struct index_zone *zone, + struct uds_request *request) +{ + u64 sparse_virtual_chapter; + + if ((zone->index->zone_count > 1) || + !uds_is_sparse_geometry(zone->index->volume->geometry)) + return UDS_SUCCESS; + + sparse_virtual_chapter = triage_index_request(zone->index, request); + if (sparse_virtual_chapter == NO_CHAPTER) + return UDS_SUCCESS; + + return uds_update_sparse_cache(zone, sparse_virtual_chapter); +} + +/* This is the request processing function for the triage queue. */ +static void triage_request(struct uds_request *request) +{ + struct uds_index *index = request->index; + u64 sparse_virtual_chapter = triage_index_request(index, request); + + if (sparse_virtual_chapter != NO_CHAPTER) + enqueue_barrier_messages(index, sparse_virtual_chapter); + + uds_enqueue_request(request, STAGE_INDEX); +} + +static int finish_previous_chapter(struct uds_index *index, u64 current_chapter_number) +{ + int result; + struct chapter_writer *writer = index->chapter_writer; + + uds_lock_mutex(&writer->mutex); + while (index->newest_virtual_chapter < current_chapter_number) + uds_wait_cond(&writer->cond, &writer->mutex); + result = writer->result; + uds_unlock_mutex(&writer->mutex); + + if (result != UDS_SUCCESS) + return uds_log_error_strerror(result, + "Writing of previous open chapter failed"); + + return UDS_SUCCESS; +} + +static int swap_open_chapter(struct index_zone *zone) +{ + int result; + struct open_chapter_zone *temporary_chapter; + + result = finish_previous_chapter(zone->index, zone->newest_virtual_chapter); + if (result != UDS_SUCCESS) + return result; + + temporary_chapter = zone->open_chapter; + zone->open_chapter = zone->writing_chapter; + zone->writing_chapter = temporary_chapter; + return UDS_SUCCESS; +} + +/* + * Inform the chapter writer that this zone is done with this chapter. The chapter won't start + * writing until all zones have closed it. + */ +static unsigned int start_closing_chapter(struct uds_index *index, + unsigned int zone_number, + struct open_chapter_zone *chapter) +{ + unsigned int finished_zones; + struct chapter_writer *writer = index->chapter_writer; + + uds_lock_mutex(&writer->mutex); + finished_zones = ++writer->zones_to_write; + writer->chapters[zone_number] = chapter; + uds_broadcast_cond(&writer->cond); + uds_unlock_mutex(&writer->mutex); + + return finished_zones; +} + +static int announce_chapter_closed(struct index_zone *zone, u64 closed_chapter) +{ + int result; + unsigned int i; + struct uds_zone_message zone_message = { + .type = UDS_MESSAGE_ANNOUNCE_CHAPTER_CLOSED, + .virtual_chapter = closed_chapter, + }; + + for (i = 0; i < zone->index->zone_count; i++) { + if (zone->id == i) + continue; + + result = launch_zone_message(zone_message, i, zone->index); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +static int open_next_chapter(struct index_zone *zone) +{ + int result; + u64 closed_chapter; + u64 expiring; + unsigned int finished_zones; + u32 expire_chapters; + + uds_log_debug("closing chapter %llu of zone %u after %u entries (%u short)", + (unsigned long long) zone->newest_virtual_chapter, zone->id, + zone->open_chapter->size, + zone->open_chapter->capacity - zone->open_chapter->size); + + result = swap_open_chapter(zone); + if (result != UDS_SUCCESS) + return result; + + closed_chapter = zone->newest_virtual_chapter++; + uds_set_volume_index_zone_open_chapter(zone->index->volume_index, zone->id, + zone->newest_virtual_chapter); + uds_reset_open_chapter(zone->open_chapter); + + finished_zones = start_closing_chapter(zone->index, zone->id, + zone->writing_chapter); + if ((finished_zones == 1) && (zone->index->zone_count > 1)) { + result = announce_chapter_closed(zone, closed_chapter); + if (result != UDS_SUCCESS) + return result; + } + + expiring = zone->oldest_virtual_chapter; + expire_chapters = uds_chapters_to_expire(zone->index->volume->geometry, + zone->newest_virtual_chapter); + zone->oldest_virtual_chapter += expire_chapters; + + if (finished_zones < zone->index->zone_count) + return UDS_SUCCESS; + + while (expire_chapters-- > 0) + uds_forget_chapter(zone->index->volume, expiring++); + + return UDS_SUCCESS; +} + +static int handle_chapter_closed(struct index_zone *zone, u64 virtual_chapter) +{ + if (zone->newest_virtual_chapter == virtual_chapter) + return open_next_chapter(zone); + + return UDS_SUCCESS; +} + +static int dispatch_index_zone_control_request(struct uds_request *request) +{ + struct uds_zone_message *message = &request->zone_message; + struct index_zone *zone = request->index->zones[request->zone_number]; + + switch (message->type) { + case UDS_MESSAGE_SPARSE_CACHE_BARRIER: + return uds_update_sparse_cache(zone, message->virtual_chapter); + + case UDS_MESSAGE_ANNOUNCE_CHAPTER_CLOSED: + return handle_chapter_closed(zone, message->virtual_chapter); + + default: + uds_log_error("invalid message type: %d", message->type); + return UDS_INVALID_ARGUMENT; + } +} + +static void set_request_location(struct uds_request *request, + enum uds_index_region new_location) +{ + request->location = new_location; + request->found = ((new_location == UDS_LOCATION_IN_OPEN_CHAPTER) || + (new_location == UDS_LOCATION_IN_DENSE) || + (new_location == UDS_LOCATION_IN_SPARSE)); +} + +static void set_chapter_location(struct uds_request *request, + const struct index_zone *zone, u64 virtual_chapter) +{ + request->found = true; + if (virtual_chapter == zone->newest_virtual_chapter) + request->location = UDS_LOCATION_IN_OPEN_CHAPTER; + else if (is_zone_chapter_sparse(zone, virtual_chapter)) + request->location = UDS_LOCATION_IN_SPARSE; + else + request->location = UDS_LOCATION_IN_DENSE; +} + +static int search_sparse_cache_in_zone(struct index_zone *zone, struct uds_request *request, + u64 virtual_chapter, bool *found) +{ + int result; + struct volume *volume; + u16 record_page_number; + u32 chapter; + + result = uds_search_sparse_cache(zone, &request->record_name, &virtual_chapter, + &record_page_number); + if ((result != UDS_SUCCESS) || (virtual_chapter == NO_CHAPTER)) + return result; + + request->virtual_chapter = virtual_chapter; + volume = zone->index->volume; + chapter = uds_map_to_physical_chapter(volume->geometry, virtual_chapter); + return uds_search_cached_record_page(volume, request, chapter, + record_page_number, found); +} + +static int get_record_from_zone(struct index_zone *zone, struct uds_request *request, + bool *found) +{ + struct volume *volume; + + if (request->location == UDS_LOCATION_RECORD_PAGE_LOOKUP) { + *found = true; + return UDS_SUCCESS; + } else if (request->location == UDS_LOCATION_UNAVAILABLE) { + *found = false; + return UDS_SUCCESS; + } + + if (request->virtual_chapter == zone->newest_virtual_chapter) { + uds_search_open_chapter(zone->open_chapter, &request->record_name, + &request->old_metadata, found); + return UDS_SUCCESS; + } + + if ((zone->newest_virtual_chapter > 0) && + (request->virtual_chapter == (zone->newest_virtual_chapter - 1)) && + (zone->writing_chapter->size > 0)) { + uds_search_open_chapter(zone->writing_chapter, &request->record_name, + &request->old_metadata, found); + return UDS_SUCCESS; + } + + volume = zone->index->volume; + if (is_zone_chapter_sparse(zone, request->virtual_chapter) && + uds_sparse_cache_contains(volume->sparse_cache, request->virtual_chapter, + request->zone_number)) + return search_sparse_cache_in_zone(zone, request, + request->virtual_chapter, found); + + return uds_search_volume_page_cache(volume, request, found); +} + +static int put_record_in_zone(struct index_zone *zone, struct uds_request *request, + const struct uds_record_data *metadata) +{ + unsigned int remaining; + + remaining = uds_put_open_chapter(zone->open_chapter, &request->record_name, + metadata); + if (remaining == 0) + return open_next_chapter(zone); + + return UDS_SUCCESS; +} + +static int search_index_zone(struct index_zone *zone, struct uds_request *request) +{ + int result; + struct volume_index_record record; + bool overflow_record, found = false; + struct uds_record_data *metadata; + u64 chapter; + + result = uds_get_volume_index_record(zone->index->volume_index, + &request->record_name, &record); + if (result != UDS_SUCCESS) + return result; + + if (record.is_found) { + if (request->requeued && request->virtual_chapter != record.virtual_chapter) + set_request_location(request, UDS_LOCATION_UNKNOWN); + + request->virtual_chapter = record.virtual_chapter; + result = get_record_from_zone(zone, request, &found); + if (result != UDS_SUCCESS) + return result; + } + + if (found) + set_chapter_location(request, zone, record.virtual_chapter); + + /* + * If a record has overflowed a chapter index in more than one chapter (or overflowed in + * one chapter and collided with an existing record), it will exist as a collision record + * in the volume index, but we won't find it in the volume. This case needs special + * handling. + */ + overflow_record = (record.is_found && record.is_collision && !found); + chapter = zone->newest_virtual_chapter; + if (found || overflow_record) { + if ((request->type == UDS_QUERY_NO_UPDATE) || + ((request->type == UDS_QUERY) && overflow_record)) { + /* There is nothing left to do. */ + return UDS_SUCCESS; + } + + if (record.virtual_chapter != chapter) { + /* + * Update the volume index to reference the new chapter for the block. If + * the record had been deleted or dropped from the chapter index, it will + * be back. + */ + result = uds_set_volume_index_record_chapter(&record, chapter); + } else if (request->type != UDS_UPDATE) { + /* The record is already in the open chapter. */ + return UDS_SUCCESS; + } + } else { + /* + * The record wasn't in the volume index, so check whether the + * name is in a cached sparse chapter. If we found the name on + * a previous search, use that result instead. + */ + if (request->location == UDS_LOCATION_RECORD_PAGE_LOOKUP) { + found = true; + } else if (request->location == UDS_LOCATION_UNAVAILABLE) { + found = false; + } else if (uds_is_sparse_geometry(zone->index->volume->geometry) && + !uds_is_volume_index_sample(zone->index->volume_index, + &request->record_name)) { + result = search_sparse_cache_in_zone(zone, request, NO_CHAPTER, + &found); + if (result != UDS_SUCCESS) + return result; + } + + if (found) + set_request_location(request, UDS_LOCATION_IN_SPARSE); + + if ((request->type == UDS_QUERY_NO_UPDATE) || + ((request->type == UDS_QUERY) && !found)) { + /* There is nothing left to do. */ + return UDS_SUCCESS; + } + + /* + * Add a new entry to the volume index referencing the open chapter. This needs to + * be done both for new records, and for records from cached sparse chapters. + */ + result = uds_put_volume_index_record(&record, chapter); + } + + if (result == UDS_OVERFLOW) { + /* + * The volume index encountered a delta list overflow. The condition was already + * logged. We will go on without adding the record to the open chapter. + */ + return UDS_SUCCESS; + } + + if (result != UDS_SUCCESS) + return result; + + if (!found || (request->type == UDS_UPDATE)) { + /* This is a new record or we're updating an existing record. */ + metadata = &request->new_metadata; + } else { + /* Move the existing record to the open chapter. */ + metadata = &request->old_metadata; + } + + return put_record_in_zone(zone, request, metadata); +} + +static int remove_from_index_zone(struct index_zone *zone, struct uds_request *request) +{ + int result; + struct volume_index_record record; + + result = uds_get_volume_index_record(zone->index->volume_index, + &request->record_name, &record); + if (result != UDS_SUCCESS) + return result; + + if (!record.is_found) + return UDS_SUCCESS; + + /* If the request was requeued, check whether the saved state is still valid. */ + + if (record.is_collision) { + set_chapter_location(request, zone, record.virtual_chapter); + } else { + /* Non-collision records are hints, so resolve the name in the chapter. */ + bool found; + + if (request->requeued && request->virtual_chapter != record.virtual_chapter) + set_request_location(request, UDS_LOCATION_UNKNOWN); + + request->virtual_chapter = record.virtual_chapter; + result = get_record_from_zone(zone, request, &found); + if (result != UDS_SUCCESS) + return result; + + if (!found) { + /* There is no record to remove. */ + return UDS_SUCCESS; + } + } + + set_chapter_location(request, zone, record.virtual_chapter); + + /* + * Delete the volume index entry for the named record only. Note that a later search might + * later return stale advice if there is a colliding name in the same chapter, but it's a + * very rare case (1 in 2^21). + */ + result = uds_remove_volume_index_record(&record); + if (result != UDS_SUCCESS) + return result; + + /* + * If the record is in the open chapter, we must remove it or mark it deleted to avoid + * trouble if the record is added again later. + */ + if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER) + uds_remove_from_open_chapter(zone->open_chapter, &request->record_name); + + return UDS_SUCCESS; +} + +static int dispatch_index_request(struct uds_index *index, struct uds_request *request) +{ + int result; + struct index_zone *zone = index->zones[request->zone_number]; + + if (!request->requeued) { + result = simulate_index_zone_barrier_message(zone, request); + if (result != UDS_SUCCESS) + return result; + } + + switch (request->type) { + case UDS_POST: + case UDS_UPDATE: + case UDS_QUERY: + case UDS_QUERY_NO_UPDATE: + result = search_index_zone(zone, request); + break; + + case UDS_DELETE: + result = remove_from_index_zone(zone, request); + break; + + default: + result = uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + "invalid request type: %d", + request->type); + break; + } + + return result; +} + +/* This is the request processing function invoked by each zone's thread. */ +static void execute_zone_request(struct uds_request *request) +{ + int result; + struct uds_index *index = request->index; + + if (request->zone_message.type != UDS_MESSAGE_NONE) { + result = dispatch_index_zone_control_request(request); + if (result != UDS_SUCCESS) { + uds_log_error_strerror(result, "error executing message: %d", + request->zone_message.type); + } + + /* Once the message is processed it can be freed. */ + uds_free(uds_forget(request)); + return; + } + + index->need_to_save = true; + if (request->requeued && (request->status != UDS_SUCCESS)) { + set_request_location(request, UDS_LOCATION_UNAVAILABLE); + index->callback(request); + return; + } + + result = dispatch_index_request(index, request); + if (result == UDS_QUEUED) { + /* The request has been requeued so don't let it complete. */ + return; + } + + if (!request->found) + set_request_location(request, UDS_LOCATION_UNAVAILABLE); + + request->status = result; + index->callback(request); +} + +static int initialize_index_queues(struct uds_index *index, + const struct geometry *geometry) +{ + int result; + unsigned int i; + + for (i = 0; i < index->zone_count; i++) { + result = uds_make_request_queue("indexW", &execute_zone_request, + &index->zone_queues[i]); + if (result != UDS_SUCCESS) + return result; + } + + /* The triage queue is only needed for sparse multi-zone indexes. */ + if ((index->zone_count > 1) && uds_is_sparse_geometry(geometry)) { + result = uds_make_request_queue("triageW", &triage_request, + &index->triage_queue); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +/* This is the driver function for the chapter writer thread. */ +static void close_chapters(void *arg) +{ + int result; + struct chapter_writer *writer = arg; + struct uds_index *index = writer->index; + + uds_log_debug("chapter writer starting"); + uds_lock_mutex(&writer->mutex); + for (;;) { + while (writer->zones_to_write < index->zone_count) { + if (writer->stop && (writer->zones_to_write == 0)) { + /* + * We've been told to stop, and all of the zones are in the same + * open chapter, so we can exit now. + */ + uds_unlock_mutex(&writer->mutex); + uds_log_debug("chapter writer stopping"); + return; + } + uds_wait_cond(&writer->cond, &writer->mutex); + } + + /* + * Release the lock while closing a chapter. We probably don't need to do this, but + * it seems safer in principle. It's OK to access the chapter and chapter_number + * fields without the lock since those aren't allowed to change until we're done. + */ + uds_unlock_mutex(&writer->mutex); + + if (index->has_saved_open_chapter) { + /* + * Remove the saved open chapter the first time we close an open chapter + * after loading from a clean shutdown, or after doing a clean save. The + * lack of the saved open chapter will indicate that a recovery is + * necessary. + */ + index->has_saved_open_chapter = false; + result = uds_discard_open_chapter(index->layout); + if (result == UDS_SUCCESS) + uds_log_debug("Discarding saved open chapter"); + } + + result = uds_close_open_chapter(writer->chapters, index->zone_count, + index->volume, + writer->open_chapter_index, + writer->collated_records, + index->newest_virtual_chapter); + + uds_lock_mutex(&writer->mutex); + index->newest_virtual_chapter++; + index->oldest_virtual_chapter += + uds_chapters_to_expire(index->volume->geometry, + index->newest_virtual_chapter); + writer->result = result; + writer->zones_to_write = 0; + uds_broadcast_cond(&writer->cond); + } +} + +static void stop_chapter_writer(struct chapter_writer *writer) +{ + struct thread *writer_thread = 0; + + uds_lock_mutex(&writer->mutex); + if (writer->thread != 0) { + writer_thread = writer->thread; + writer->thread = 0; + writer->stop = true; + uds_broadcast_cond(&writer->cond); + } + uds_unlock_mutex(&writer->mutex); + + if (writer_thread != 0) + uds_join_threads(writer_thread); +} + +static void free_chapter_writer(struct chapter_writer *writer) +{ + if (writer == NULL) + return; + + stop_chapter_writer(writer); + uds_destroy_mutex(&writer->mutex); + uds_destroy_cond(&writer->cond); + uds_free_open_chapter_index(writer->open_chapter_index); + uds_free(writer->collated_records); + uds_free(writer); +} + +static int make_chapter_writer(struct uds_index *index, + struct chapter_writer **writer_ptr) +{ + int result; + struct chapter_writer *writer; + size_t collated_records_size = + (sizeof(struct uds_volume_record) * index->volume->geometry->records_per_chapter); + + result = uds_allocate_extended(struct chapter_writer, index->zone_count, + struct open_chapter_zone *, "Chapter Writer", + &writer); + if (result != UDS_SUCCESS) + return result; + + writer->index = index; + result = uds_init_mutex(&writer->mutex); + if (result != UDS_SUCCESS) { + uds_free(writer); + return result; + } + + result = uds_init_cond(&writer->cond); + if (result != UDS_SUCCESS) { + uds_destroy_mutex(&writer->mutex); + uds_free(writer); + return result; + } + + result = uds_allocate_cache_aligned(collated_records_size, "collated records", + &writer->collated_records); + if (result != UDS_SUCCESS) { + free_chapter_writer(writer); + return result; + } + + result = uds_make_open_chapter_index(&writer->open_chapter_index, + index->volume->geometry, + index->volume->nonce); + if (result != UDS_SUCCESS) { + free_chapter_writer(writer); + return result; + } + + writer->memory_size = (sizeof(struct chapter_writer) + + index->zone_count * sizeof(struct open_chapter_zone *) + + collated_records_size + + writer->open_chapter_index->memory_size); + + result = uds_create_thread(close_chapters, writer, "writer", &writer->thread); + if (result != UDS_SUCCESS) { + free_chapter_writer(writer); + return result; + } + + *writer_ptr = writer; + return UDS_SUCCESS; +} + +static int load_index(struct uds_index *index) +{ + int result; + u64 last_save_chapter; + + result = uds_load_index_state(index->layout, index); + if (result != UDS_SUCCESS) + return UDS_INDEX_NOT_SAVED_CLEANLY; + + last_save_chapter = ((index->last_save != NO_LAST_SAVE) ? index->last_save : 0); + + uds_log_info("loaded index from chapter %llu through chapter %llu", + (unsigned long long) index->oldest_virtual_chapter, + (unsigned long long) last_save_chapter); + + return UDS_SUCCESS; +} + +static int rebuild_index_page_map(struct uds_index *index, u64 vcn) +{ + int result; + struct delta_index_page *chapter_index_page; + struct geometry *geometry = index->volume->geometry; + u32 chapter = uds_map_to_physical_chapter(geometry, vcn); + u32 expected_list_number = 0; + u32 index_page_number; + u32 lowest_delta_list; + u32 highest_delta_list; + + for (index_page_number = 0; + index_page_number < geometry->index_pages_per_chapter; + index_page_number++) { + result = uds_get_volume_index_page(index->volume, chapter, + index_page_number, + &chapter_index_page); + if (result != UDS_SUCCESS) { + return uds_log_error_strerror(result, + "failed to read index page %u in chapter %u", + index_page_number, chapter); + } + + lowest_delta_list = chapter_index_page->lowest_list_number; + highest_delta_list = chapter_index_page->highest_list_number; + if (lowest_delta_list != expected_list_number) { + return uds_log_error_strerror(UDS_CORRUPT_DATA, + "chapter %u index page %u is corrupt", + chapter, index_page_number); + } + + uds_update_index_page_map(index->volume->index_page_map, vcn, chapter, + index_page_number, highest_delta_list); + expected_list_number = highest_delta_list + 1; + } + + return UDS_SUCCESS; +} + +static int replay_record(struct uds_index *index, const struct uds_record_name *name, + u64 virtual_chapter, bool will_be_sparse_chapter) +{ + int result; + struct volume_index_record record; + bool update_record; + + if (will_be_sparse_chapter && + !uds_is_volume_index_sample(index->volume_index, name)) { + /* + * This entry will be in a sparse chapter after the rebuild completes, and it is + * not a sample, so just skip over it. + */ + return UDS_SUCCESS; + } + + result = uds_get_volume_index_record(index->volume_index, name, &record); + if (result != UDS_SUCCESS) + return result; + + if (record.is_found) { + if (record.is_collision) { + if (record.virtual_chapter == virtual_chapter) { + /* The record is already correct. */ + return UDS_SUCCESS; + } + + update_record = true; + } else if (record.virtual_chapter == virtual_chapter) { + /* + * There is a volume index entry pointing to the current chapter, but we + * don't know if it is for the same name as the one we are currently + * working on or not. For now, we're just going to assume that it isn't. + * This will create one extra collision record if there was a deleted + * record in the current chapter. + */ + update_record = false; + } else { + /* + * If we're rebuilding, we don't normally want to go to disk to see if the + * record exists, since we will likely have just read the record from disk + * (i.e. we know it's there). The exception to this is when we find an + * entry in the volume index that has a different chapter. In this case, we + * need to search that chapter to determine if the volume index entry was + * for the same record or a different one. + */ + result = uds_search_volume_page_cache_for_rebuild(index->volume, + name, + record.virtual_chapter, + &update_record); + if (result != UDS_SUCCESS) + return result; + } + } else { + update_record = false; + } + + if (update_record) { + /* + * Update the volume index to reference the new chapter for the block. If the + * record had been deleted or dropped from the chapter index, it will be back. + */ + result = uds_set_volume_index_record_chapter(&record, virtual_chapter); + } else { + /* + * Add a new entry to the volume index referencing the open chapter. This should be + * done regardless of whether we are a brand new record or a sparse record, i.e. + * one that doesn't exist in the index but does on disk, since for a sparse record, + * we would want to un-sparsify if it did exist. + */ + result = uds_put_volume_index_record(&record, virtual_chapter); + } + + if ((result == UDS_DUPLICATE_NAME) || (result == UDS_OVERFLOW)) { + /* The rebuilt index will lose these records. */ + return UDS_SUCCESS; + } + + return result; +} + +static bool check_for_suspend(struct uds_index *index) +{ + bool closing; + + if (index->load_context == NULL) + return false; + + uds_lock_mutex(&index->load_context->mutex); + if (index->load_context->status != INDEX_SUSPENDING) { + uds_unlock_mutex(&index->load_context->mutex); + return false; + } + + /* Notify that we are suspended and wait for the resume. */ + index->load_context->status = INDEX_SUSPENDED; + uds_broadcast_cond(&index->load_context->cond); + + while ((index->load_context->status != INDEX_OPENING) && + (index->load_context->status != INDEX_FREEING)) + uds_wait_cond(&index->load_context->cond, &index->load_context->mutex); + + closing = (index->load_context->status == INDEX_FREEING); + uds_unlock_mutex(&index->load_context->mutex); + return closing; +} + +static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse) +{ + int result; + u32 i; + u32 j; + const struct geometry *geometry; + u32 physical_chapter; + + if (check_for_suspend(index)) { + uds_log_info("Replay interrupted by index shutdown at chapter %llu", + (unsigned long long) virtual); + return -EBUSY; + } + + geometry = index->volume->geometry; + physical_chapter = uds_map_to_physical_chapter(geometry, virtual); + uds_prefetch_volume_chapter(index->volume, physical_chapter); + uds_set_volume_index_open_chapter(index->volume_index, virtual); + + result = rebuild_index_page_map(index, virtual); + if (result != UDS_SUCCESS) { + return uds_log_error_strerror(result, + "could not rebuild index page map for chapter %u", + physical_chapter); + } + + for (i = 0; i < geometry->record_pages_per_chapter; i++) { + u8 *record_page; + u32 record_page_number; + + record_page_number = geometry->index_pages_per_chapter + i; + result = uds_get_volume_record_page(index->volume, physical_chapter, + record_page_number, &record_page); + if (result != UDS_SUCCESS) { + return uds_log_error_strerror(result, "could not get page %d", + record_page_number); + } + + for (j = 0; j < geometry->records_per_page; j++) { + const u8 *name_bytes; + struct uds_record_name name; + + name_bytes = record_page + (j * BYTES_PER_RECORD); + memcpy(&name.name, name_bytes, UDS_RECORD_NAME_SIZE); + result = replay_record(index, &name, virtual, sparse); + if (result != UDS_SUCCESS) + return result; + } + } + + return UDS_SUCCESS; +} + +static int replay_volume(struct uds_index *index) +{ + int result; + u64 old_map_update; + u64 new_map_update; + u64 virtual; + u64 from_virtual = index->oldest_virtual_chapter; + u64 upto_virtual = index->newest_virtual_chapter; + bool will_be_sparse; + + uds_log_info("Replaying volume from chapter %llu through chapter %llu", + (unsigned long long) from_virtual, + (unsigned long long) upto_virtual); + + /* + * The index failed to load, so the volume index is empty. Add records to the volume index + * in order, skipping non-hooks in chapters which will be sparse to save time. + * + * Go through each record page of each chapter and add the records back to the volume + * index. This should not cause anything to be written to either the open chapter or the + * on-disk volume. Also skip the on-disk chapter corresponding to upto_virtual, as this + * would have already been purged from the volume index when the chapter was opened. + * + * Also, go through each index page for each chapter and rebuild the index page map. + */ + old_map_update = index->volume->index_page_map->last_update; + for (virtual = from_virtual; virtual < upto_virtual; virtual++) { + will_be_sparse = uds_is_chapter_sparse(index->volume->geometry, + from_virtual, upto_virtual, + virtual); + result = replay_chapter(index, virtual, will_be_sparse); + if (result != UDS_SUCCESS) + return result; + } + + /* Also reap the chapter being replaced by the open chapter. */ + uds_set_volume_index_open_chapter(index->volume_index, upto_virtual); + + new_map_update = index->volume->index_page_map->last_update; + if (new_map_update != old_map_update) { + uds_log_info("replay changed index page map update from %llu to %llu", + (unsigned long long) old_map_update, + (unsigned long long) new_map_update); + } + + return UDS_SUCCESS; +} + +static int rebuild_index(struct uds_index *index) +{ + int result; + u64 lowest; + u64 highest; + bool is_empty = false; + u32 chapters_per_volume = index->volume->geometry->chapters_per_volume; + + index->volume->lookup_mode = LOOKUP_FOR_REBUILD; + result = uds_find_volume_chapter_boundaries(index->volume, &lowest, &highest, + &is_empty); + if (result != UDS_SUCCESS) { + return uds_log_fatal_strerror(result, + "cannot rebuild index: unknown volume chapter boundaries"); + } + + if (is_empty) { + index->newest_virtual_chapter = 0; + index->oldest_virtual_chapter = 0; + index->volume->lookup_mode = LOOKUP_NORMAL; + return UDS_SUCCESS; + } + + index->newest_virtual_chapter = highest + 1; + index->oldest_virtual_chapter = lowest; + if (index->newest_virtual_chapter == + (index->oldest_virtual_chapter + chapters_per_volume)) { + /* Skip the chapter shadowed by the open chapter. */ + index->oldest_virtual_chapter++; + } + + result = replay_volume(index); + if (result != UDS_SUCCESS) + return result; + + index->volume->lookup_mode = LOOKUP_NORMAL; + return UDS_SUCCESS; +} + +static void free_index_zone(struct index_zone *zone) +{ + if (zone == NULL) + return; + + uds_free_open_chapter(zone->open_chapter); + uds_free_open_chapter(zone->writing_chapter); + uds_free(zone); +} + +static int make_index_zone(struct uds_index *index, unsigned int zone_number) +{ + int result; + struct index_zone *zone; + + result = uds_allocate(1, struct index_zone, "index zone", &zone); + if (result != UDS_SUCCESS) + return result; + + result = uds_make_open_chapter(index->volume->geometry, index->zone_count, + &zone->open_chapter); + if (result != UDS_SUCCESS) { + free_index_zone(zone); + return result; + } + + result = uds_make_open_chapter(index->volume->geometry, index->zone_count, + &zone->writing_chapter); + if (result != UDS_SUCCESS) { + free_index_zone(zone); + return result; + } + + zone->index = index; + zone->id = zone_number; + index->zones[zone_number] = zone; + + return UDS_SUCCESS; +} + +int uds_make_index(struct configuration *config, enum uds_open_index_type open_type, + struct index_load_context *load_context, index_callback_fn callback, + struct uds_index **new_index) +{ + int result; + bool loaded = false; + bool new = (open_type == UDS_CREATE); + struct uds_index *index = NULL; + struct index_zone *zone; + u64 nonce; + unsigned int z; + + result = uds_allocate_extended(struct uds_index, config->zone_count, + struct uds_request_queue *, "index", &index); + if (result != UDS_SUCCESS) + return result; + + index->zone_count = config->zone_count; + + result = uds_make_index_layout(config, new, &index->layout); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + result = uds_allocate(index->zone_count, struct index_zone *, "zones", + &index->zones); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + result = uds_make_volume(config, index->layout, &index->volume); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + index->volume->lookup_mode = LOOKUP_NORMAL; + for (z = 0; z < index->zone_count; z++) { + result = make_index_zone(index, z); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return uds_log_error_strerror(result, + "Could not create index zone"); + } + } + + nonce = uds_get_volume_nonce(index->layout); + result = uds_make_volume_index(config, nonce, &index->volume_index); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return uds_log_error_strerror(result, "could not make volume index"); + } + + index->load_context = load_context; + index->callback = callback; + + result = initialize_index_queues(index, config->geometry); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + result = make_chapter_writer(index, &index->chapter_writer); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + if (!new) { + result = load_index(index); + switch (result) { + case UDS_SUCCESS: + loaded = true; + break; + case -ENOMEM: + /* We should not try a rebuild for this error. */ + uds_log_error_strerror(result, "index could not be loaded"); + break; + default: + uds_log_error_strerror(result, "index could not be loaded"); + if (open_type == UDS_LOAD) { + result = rebuild_index(index); + if (result != UDS_SUCCESS) { + uds_log_error_strerror(result, + "index could not be rebuilt"); + } + } + break; + } + } + + if (result != UDS_SUCCESS) { + uds_free_index(index); + return uds_log_error_strerror(result, "fatal error in %s()", __func__); + } + + for (z = 0; z < index->zone_count; z++) { + zone = index->zones[z]; + zone->oldest_virtual_chapter = index->oldest_virtual_chapter; + zone->newest_virtual_chapter = index->newest_virtual_chapter; + } + + if (index->load_context != NULL) { + uds_lock_mutex(&index->load_context->mutex); + index->load_context->status = INDEX_READY; + /* + * If we get here, suspend is meaningless, but notify any thread trying to suspend + * us so it doesn't hang. + */ + uds_broadcast_cond(&index->load_context->cond); + uds_unlock_mutex(&index->load_context->mutex); + } + + index->has_saved_open_chapter = loaded; + index->need_to_save = !loaded; + *new_index = index; + return UDS_SUCCESS; +} + +void uds_free_index(struct uds_index *index) +{ + unsigned int i; + + if (index == NULL) + return; + + uds_request_queue_finish(index->triage_queue); + for (i = 0; i < index->zone_count; i++) + uds_request_queue_finish(index->zone_queues[i]); + + free_chapter_writer(index->chapter_writer); + + uds_free_volume_index(index->volume_index); + if (index->zones != NULL) { + for (i = 0; i < index->zone_count; i++) + free_index_zone(index->zones[i]); + uds_free(index->zones); + } + + uds_free_volume(index->volume); + uds_free_index_layout(uds_forget(index->layout)); + uds_free(index); +} + +/* Wait for the chapter writer to complete any outstanding writes. */ +void uds_wait_for_idle_index(struct uds_index *index) +{ + struct chapter_writer *writer = index->chapter_writer; + + uds_lock_mutex(&writer->mutex); + while (writer->zones_to_write > 0) + uds_wait_cond(&writer->cond, &writer->mutex); + uds_unlock_mutex(&writer->mutex); +} + +/* This function assumes that all requests have been drained. */ +int uds_save_index(struct uds_index *index) +{ + int result; + + if (!index->need_to_save) + return UDS_SUCCESS; + + uds_wait_for_idle_index(index); + index->prev_save = index->last_save; + index->last_save = ((index->newest_virtual_chapter == 0) ? + NO_LAST_SAVE : + index->newest_virtual_chapter - 1); + uds_log_info("beginning save (vcn %llu)", (unsigned long long) index->last_save); + + result = uds_save_index_state(index->layout, index); + if (result != UDS_SUCCESS) { + uds_log_info("save index failed"); + index->last_save = index->prev_save; + } else { + index->has_saved_open_chapter = true; + index->need_to_save = false; + uds_log_info("finished save (vcn %llu)", + (unsigned long long) index->last_save); + } + + return result; +} + +int uds_replace_index_storage(struct uds_index *index, struct block_device *bdev) +{ + return uds_replace_volume_storage(index->volume, index->layout, bdev); +} + +/* Accessing statistics should be safe from any thread. */ +void uds_get_index_stats(struct uds_index *index, struct uds_index_stats *counters) +{ + struct volume_index_stats stats; + + uds_get_volume_index_stats(index->volume_index, &stats); + counters->entries_indexed = stats.record_count; + counters->collisions = stats.collision_count; + counters->entries_discarded = stats.discard_count; + + counters->memory_used = (index->volume_index->memory_size + + index->volume->cache_size + + index->chapter_writer->memory_size); +} + +void uds_enqueue_request(struct uds_request *request, enum request_stage stage) +{ + struct uds_index *index = request->index; + struct uds_request_queue *queue; + + switch (stage) { + case STAGE_TRIAGE: + if (index->triage_queue != NULL) { + queue = index->triage_queue; + break; + } + + fallthrough; + + case STAGE_INDEX: + request->zone_number = + uds_get_volume_index_zone(index->volume_index, &request->record_name); + fallthrough; + + case STAGE_MESSAGE: + queue = index->zone_queues[request->zone_number]; + break; + + default: + ASSERT_LOG_ONLY(false, "invalid index stage: %d", stage); + return; + } + + uds_request_queue_enqueue(queue, request); +} diff --git a/drivers/md/dm-vdo/index.h b/drivers/md/dm-vdo/index.h new file mode 100644 index 000000000000..ace1835aecd5 --- /dev/null +++ b/drivers/md/dm-vdo/index.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_H +#define UDS_INDEX_H + +#include "index-layout.h" +#include "index-session.h" +#include "open-chapter.h" +#include "volume.h" +#include "volume-index.h" + +/* + * The index is a high-level structure which represents the totality of the UDS index. It manages + * the queues for incoming requests and dispatches them to the appropriate sub-components like the + * volume or the volume index. It also manages administrative tasks such as saving and loading the + * index. + * + * The index is divided into a number of independent zones and assigns each request to a zone based + * on its name. Most sub-components are similarly divided into zones as well so that requests in + * each zone usually operate without interference or coordination between zones. + */ + +typedef void (*index_callback_fn)(struct uds_request *request); + +struct index_zone { + struct uds_index *index; + struct open_chapter_zone *open_chapter; + struct open_chapter_zone *writing_chapter; + u64 oldest_virtual_chapter; + u64 newest_virtual_chapter; + unsigned int id; +}; + +struct uds_index { + bool has_saved_open_chapter; + bool need_to_save; + struct index_load_context *load_context; + struct index_layout *layout; + struct volume_index *volume_index; + struct volume *volume; + unsigned int zone_count; + struct index_zone **zones; + + u64 oldest_virtual_chapter; + u64 newest_virtual_chapter; + + u64 last_save; + u64 prev_save; + struct chapter_writer *chapter_writer; + + index_callback_fn callback; + struct uds_request_queue *triage_queue; + struct uds_request_queue *zone_queues[]; +}; + +enum request_stage { + STAGE_TRIAGE, + STAGE_INDEX, + STAGE_MESSAGE, +}; + +int __must_check uds_make_index(struct configuration *config, + enum uds_open_index_type open_type, + struct index_load_context *load_context, + index_callback_fn callback, struct uds_index **new_index); + +int __must_check uds_save_index(struct uds_index *index); + +void uds_free_index(struct uds_index *index); + +int __must_check uds_replace_index_storage(struct uds_index *index, + struct block_device *bdev); + +void uds_get_index_stats(struct uds_index *index, struct uds_index_stats *counters); + +void uds_enqueue_request(struct uds_request *request, enum request_stage stage); + +void uds_wait_for_idle_index(struct uds_index *index); + +#endif /* UDS_INDEX_H */ diff --git a/drivers/md/dm-vdo/sparse-cache.c b/drivers/md/dm-vdo/sparse-cache.c new file mode 100644 index 000000000000..f80cde3f93ee --- /dev/null +++ b/drivers/md/dm-vdo/sparse-cache.c @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "sparse-cache.h" + +#include +#include + +#include "chapter-index.h" +#include "config.h" +#include "index.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "uds-threads.h" + +/* + * Since the cache is small, it is implemented as a simple array of cache entries. Searching for a + * specific virtual chapter is implemented as a linear search. The cache replacement policy is + * least-recently-used (LRU). Again, the small size of the cache allows the LRU order to be + * maintained by shifting entries in an array list. + * + * Changing the contents of the cache requires the coordinated participation of all zone threads + * via the careful use of barrier messages sent to all the index zones by the triage queue worker + * thread. The critical invariant for coordination is that the cache membership must not change + * between updates, so that all calls to uds_sparse_cache_contains() from the zone threads must all + * receive the same results for every virtual chapter number. To ensure that critical invariant, + * state changes such as "that virtual chapter is no longer in the volume" and "skip searching that + * chapter because it has had too many cache misses" are represented separately from the cache + * membership information (the virtual chapter number). + * + * As a result of this invariant, we have the guarantee that every zone thread will call + * uds_update_sparse_cache() once and exactly once to request a chapter that is not in the cache, + * and the serialization of the barrier requests from the triage queue ensures they will all + * request the same chapter number. This means the only synchronization we need can be provided by + * a pair of thread barriers used only in the uds_update_sparse_cache() call, providing a critical + * section where a single zone thread can drive the cache update while all the other zone threads + * are known to be blocked, waiting in the second barrier. Outside that critical section, all the + * zone threads implicitly hold a shared lock. Inside it, the thread for zone zero holds an + * exclusive lock. No other threads may access or modify the cache entries. + * + * Chapter statistics must only be modified by a single thread, which is also the zone zero thread. + * All fields that might be frequently updated by that thread are kept in separate cache-aligned + * structures so they will not cause cache contention via "false sharing" with the fields that are + * frequently accessed by all of the zone threads. + * + * The LRU order is managed independently by each zone thread, and each zone uses its own list for + * searching and cache membership queries. The zone zero list is used to decide which chapter to + * evict when the cache is updated, and its search list is copied to the other threads at that + * time. + * + * The virtual chapter number field of the cache entry is the single field indicating whether a + * chapter is a member of the cache or not. The value NO_CHAPTER is used to represent a null or + * undefined chapter number. When present in the virtual chapter number field of a + * cached_chapter_index, it indicates that the cache entry is dead, and all the other fields of + * that entry (other than immutable pointers to cache memory) are undefined and irrelevant. Any + * cache entry that is not marked as dead is fully defined and a member of the cache, and + * uds_sparse_cache_contains() will always return true for any virtual chapter number that appears + * in any of the cache entries. + * + * A chapter index that is a member of the cache may be excluded from searches between calls to + * uds_update_sparse_cache() in two different ways. First, when a chapter falls off the end of the + * volume, its virtual chapter number will be less that the oldest virtual chapter number. Since + * that chapter is no longer part of the volume, there's no point in continuing to search that + * chapter index. Once invalidated, that virtual chapter will still be considered a member of the + * cache, but it will no longer be searched for matching names. + * + * The second mechanism is a heuristic based on keeping track of the number of consecutive search + * misses in a given chapter index. Once that count exceeds a threshold, the skip_search flag will + * be set to true, causing the chapter to be skipped when searching the entire cache, but still + * allowing it to be found when searching for a hook in that specific chapter. Finding a hook will + * clear the skip_search flag, once again allowing the non-hook searches to use that cache entry. + * Again, regardless of the state of the skip_search flag, the virtual chapter must still + * considered to be a member of the cache for uds_sparse_cache_contains(). + */ + +enum { + SKIP_SEARCH_THRESHOLD = 20000, + ZONE_ZERO = 0, +}; + +/* + * These counters are essentially fields of the struct cached_chapter_index, but are segregated + * into this structure because they are frequently modified. They are grouped and aligned to keep + * them on different cache lines from the chapter fields that are accessed far more often than they + * are updated. + */ +struct __aligned(L1_CACHE_BYTES) cached_index_counters { + u64 consecutive_misses; +}; + +struct __aligned(L1_CACHE_BYTES) cached_chapter_index { + /* + * The virtual chapter number of the cached chapter index. NO_CHAPTER means this cache + * entry is unused. This field must only be modified in the critical section in + * uds_update_sparse_cache(). + */ + u64 virtual_chapter; + + u32 index_pages_count; + + /* + * These pointers are immutable during the life of the cache. The contents of the arrays + * change when the cache entry is replaced. + */ + struct delta_index_page *index_pages; + struct dm_buffer **page_buffers; + + /* + * If set, skip the chapter when searching the entire cache. This flag is just a + * performance optimization. This flag is mutable between cache updates, but it rarely + * changes and is frequently accessed, so it groups with the immutable fields. + */ + bool skip_search; + + /* + * The cache-aligned counters change often and are placed at the end of the structure to + * prevent false sharing with the more stable fields above. + */ + struct cached_index_counters counters; +}; + +/* + * A search_list represents an ordering of the sparse chapter index cache entry array, from most + * recently accessed to least recently accessed, which is the order in which the indexes should be + * searched and the reverse order in which they should be evicted from the cache. + * + * Cache entries that are dead or empty are kept at the end of the list, avoiding the need to even + * iterate over them to search, and ensuring that dead entries are replaced before any live entries + * are evicted. + * + * The search list is instantiated for each zone thread, avoiding any need for synchronization. The + * structure is allocated on a cache boundary to avoid false sharing of memory cache lines between + * zone threads. + */ +struct search_list { + u8 capacity; + u8 first_dead_entry; + struct cached_chapter_index *entries[]; +}; + +struct sparse_cache { + const struct geometry *geometry; + unsigned int capacity; + unsigned int zone_count; + + unsigned int skip_threshold; + struct search_list *search_lists[MAX_ZONES]; + struct cached_chapter_index **scratch_entries; + + struct barrier begin_update_barrier; + struct barrier end_update_barrier; + + struct cached_chapter_index chapters[]; +}; + +static int __must_check initialize_cached_chapter_index(struct cached_chapter_index *chapter, + const struct geometry *geometry) +{ + int result; + + chapter->virtual_chapter = NO_CHAPTER; + chapter->index_pages_count = geometry->index_pages_per_chapter; + + result = uds_allocate(chapter->index_pages_count, struct delta_index_page, + __func__, &chapter->index_pages); + if (result != UDS_SUCCESS) + return result; + + return uds_allocate(chapter->index_pages_count, struct dm_buffer *, + "sparse index volume pages", &chapter->page_buffers); +} + +static int __must_check make_search_list(struct sparse_cache *cache, + struct search_list **list_ptr) +{ + struct search_list *list; + unsigned int bytes; + u8 i; + int result; + + bytes = (sizeof(struct search_list) + + (cache->capacity * sizeof(struct cached_chapter_index *))); + result = uds_allocate_cache_aligned(bytes, "search list", &list); + if (result != UDS_SUCCESS) + return result; + + list->capacity = cache->capacity; + list->first_dead_entry = 0; + + for (i = 0; i < list->capacity; i++) + list->entries[i] = &cache->chapters[i]; + + *list_ptr = list; + return UDS_SUCCESS; +} + +int uds_make_sparse_cache(const struct geometry *geometry, unsigned int capacity, + unsigned int zone_count, struct sparse_cache **cache_ptr) +{ + int result; + unsigned int i; + struct sparse_cache *cache; + unsigned int bytes; + + bytes = (sizeof(struct sparse_cache) + (capacity * sizeof(struct cached_chapter_index))); + result = uds_allocate_cache_aligned(bytes, "sparse cache", &cache); + if (result != UDS_SUCCESS) + return result; + + cache->geometry = geometry; + cache->capacity = capacity; + cache->zone_count = zone_count; + + /* + * Scale down the skip threshold since the cache only counts cache misses in zone zero, but + * requests are being handled in all zones. + */ + cache->skip_threshold = (SKIP_SEARCH_THRESHOLD / zone_count); + + result = uds_initialize_barrier(&cache->begin_update_barrier, zone_count); + if (result != UDS_SUCCESS) { + uds_free_sparse_cache(cache); + return result; + } + + result = uds_initialize_barrier(&cache->end_update_barrier, zone_count); + if (result != UDS_SUCCESS) { + uds_free_sparse_cache(cache); + return result; + } + + for (i = 0; i < capacity; i++) { + result = initialize_cached_chapter_index(&cache->chapters[i], geometry); + if (result != UDS_SUCCESS) { + uds_free_sparse_cache(cache); + return result; + } + } + + for (i = 0; i < zone_count; i++) { + result = make_search_list(cache, &cache->search_lists[i]); + if (result != UDS_SUCCESS) { + uds_free_sparse_cache(cache); + return result; + } + } + + /* purge_search_list() needs some temporary lists for sorting. */ + result = uds_allocate(capacity * 2, struct cached_chapter_index *, + "scratch entries", &cache->scratch_entries); + if (result != UDS_SUCCESS) { + uds_free_sparse_cache(cache); + return result; + } + + *cache_ptr = cache; + return UDS_SUCCESS; +} + +static inline void set_skip_search(struct cached_chapter_index *chapter, + bool skip_search) +{ + /* Check before setting to reduce cache line contention. */ + if (READ_ONCE(chapter->skip_search) != skip_search) + WRITE_ONCE(chapter->skip_search, skip_search); +} + +static void score_search_hit(struct cached_chapter_index *chapter) +{ + chapter->counters.consecutive_misses = 0; + set_skip_search(chapter, false); +} + +static void score_search_miss(struct sparse_cache *cache, + struct cached_chapter_index *chapter) +{ + chapter->counters.consecutive_misses++; + if (chapter->counters.consecutive_misses > cache->skip_threshold) + set_skip_search(chapter, true); +} + +static void release_cached_chapter_index(struct cached_chapter_index *chapter) +{ + unsigned int i; + + chapter->virtual_chapter = NO_CHAPTER; + if (chapter->page_buffers == NULL) + return; + + for (i = 0; i < chapter->index_pages_count; i++) { + if (chapter->page_buffers[i] != NULL) + dm_bufio_release(uds_forget(chapter->page_buffers[i])); + } +} + +void uds_free_sparse_cache(struct sparse_cache *cache) +{ + unsigned int i; + + if (cache == NULL) + return; + + uds_free(cache->scratch_entries); + + for (i = 0; i < cache->zone_count; i++) + uds_free(cache->search_lists[i]); + + for (i = 0; i < cache->capacity; i++) { + release_cached_chapter_index(&cache->chapters[i]); + uds_free(cache->chapters[i].index_pages); + uds_free(cache->chapters[i].page_buffers); + } + + uds_destroy_barrier(&cache->begin_update_barrier); + uds_destroy_barrier(&cache->end_update_barrier); + uds_free(cache); +} + +/* + * Take the indicated element of the search list and move it to the start, pushing the pointers + * previously before it back down the list. + */ +static inline void set_newest_entry(struct search_list *search_list, u8 index) +{ + struct cached_chapter_index *newest; + + if (index > 0) { + newest = search_list->entries[index]; + memmove(&search_list->entries[1], &search_list->entries[0], + index * sizeof(struct cached_chapter_index *)); + search_list->entries[0] = newest; + } + + /* + * This function may have moved a dead chapter to the front of the list for reuse, in which + * case the set of dead chapters becomes smaller. + */ + if (search_list->first_dead_entry <= index) + search_list->first_dead_entry++; +} + +bool uds_sparse_cache_contains(struct sparse_cache *cache, u64 virtual_chapter, + unsigned int zone_number) +{ + struct search_list *search_list; + struct cached_chapter_index *chapter; + u8 i; + + /* + * The correctness of the barriers depends on the invariant that between calls to + * uds_update_sparse_cache(), the answers this function returns must never vary: the result + * for a given chapter must be identical across zones. That invariant must be maintained + * even if the chapter falls off the end of the volume, or if searching it is disabled + * because of too many search misses. + */ + search_list = cache->search_lists[zone_number]; + for (i = 0; i < search_list->first_dead_entry; i++) { + chapter = search_list->entries[i]; + + if (virtual_chapter == chapter->virtual_chapter) { + if (zone_number == ZONE_ZERO) + score_search_hit(chapter); + + set_newest_entry(search_list, i); + return true; + } + } + + return false; +} + +/* + * Re-sort cache entries into three sets (active, skippable, and dead) while maintaining the LRU + * ordering that already existed. This operation must only be called during the critical section in + * uds_update_sparse_cache(). + */ +static void purge_search_list(struct search_list *search_list, + struct sparse_cache *cache, u64 oldest_virtual_chapter) +{ + struct cached_chapter_index **entries; + struct cached_chapter_index **skipped; + struct cached_chapter_index **dead; + struct cached_chapter_index *chapter; + unsigned int next_alive = 0; + unsigned int next_skipped = 0; + unsigned int next_dead = 0; + unsigned int i; + + entries = &search_list->entries[0]; + skipped = &cache->scratch_entries[0]; + dead = &cache->scratch_entries[search_list->capacity]; + + for (i = 0; i < search_list->first_dead_entry; i++) { + chapter = search_list->entries[i]; + if ((chapter->virtual_chapter < oldest_virtual_chapter) || + (chapter->virtual_chapter == NO_CHAPTER)) + dead[next_dead++] = chapter; + else if (chapter->skip_search) + skipped[next_skipped++] = chapter; + else + entries[next_alive++] = chapter; + } + + memcpy(&entries[next_alive], skipped, + next_skipped * sizeof(struct cached_chapter_index *)); + memcpy(&entries[next_alive + next_skipped], dead, + next_dead * sizeof(struct cached_chapter_index *)); + search_list->first_dead_entry = next_alive + next_skipped; +} + +static int __must_check cache_chapter_index(struct cached_chapter_index *chapter, + u64 virtual_chapter, + const struct volume *volume) +{ + int result; + + release_cached_chapter_index(chapter); + + result = uds_read_chapter_index_from_volume(volume, virtual_chapter, + chapter->page_buffers, + chapter->index_pages); + if (result != UDS_SUCCESS) + return result; + + chapter->counters.consecutive_misses = 0; + chapter->virtual_chapter = virtual_chapter; + chapter->skip_search = false; + + return UDS_SUCCESS; +} + +static inline void copy_search_list(const struct search_list *source, + struct search_list *target) +{ + *target = *source; + memcpy(target->entries, source->entries, + source->capacity * sizeof(struct cached_chapter_index *)); +} + +/* + * Update the sparse cache to contain a chapter index. This function must be called by all the zone + * threads with the same chapter number to correctly enter the thread barriers used to synchronize + * the cache updates. + */ +int uds_update_sparse_cache(struct index_zone *zone, u64 virtual_chapter) +{ + int result = UDS_SUCCESS; + const struct uds_index *index = zone->index; + struct sparse_cache *cache = index->volume->sparse_cache; + + if (uds_sparse_cache_contains(cache, virtual_chapter, zone->id)) + return UDS_SUCCESS; + + /* + * Wait for every zone thread to reach its corresponding barrier request and invoke this + * function before starting to modify the cache. + */ + uds_enter_barrier(&cache->begin_update_barrier); + + /* + * This is the start of the critical section: the zone zero thread is captain, effectively + * holding an exclusive lock on the sparse cache. All the other zone threads must do + * nothing between the two barriers. They will wait at the end_update_barrier again for the + * captain to finish the update. + */ + + if (zone->id == ZONE_ZERO) { + unsigned int z; + struct search_list *list = cache->search_lists[ZONE_ZERO]; + + purge_search_list(list, cache, zone->oldest_virtual_chapter); + + if (virtual_chapter >= index->oldest_virtual_chapter) { + set_newest_entry(list, list->capacity - 1); + result = cache_chapter_index(list->entries[0], virtual_chapter, + index->volume); + } + + for (z = 1; z < cache->zone_count; z++) + copy_search_list(list, cache->search_lists[z]); + } + + /* + * This is the end of the critical section. All cache invariants must have been restored. + */ + uds_enter_barrier(&cache->end_update_barrier); + return result; +} + +void uds_invalidate_sparse_cache(struct sparse_cache *cache) +{ + unsigned int i; + + for (i = 0; i < cache->capacity; i++) + release_cached_chapter_index(&cache->chapters[i]); +} + +static inline bool should_skip_chapter(struct cached_chapter_index *chapter, + u64 oldest_chapter, u64 requested_chapter) +{ + if ((chapter->virtual_chapter == NO_CHAPTER) || + (chapter->virtual_chapter < oldest_chapter)) + return true; + + if (requested_chapter != NO_CHAPTER) + return requested_chapter != chapter->virtual_chapter; + else + return READ_ONCE(chapter->skip_search); +} + +static int __must_check search_cached_chapter_index(struct cached_chapter_index *chapter, + const struct geometry *geometry, + const struct index_page_map *index_page_map, + const struct uds_record_name *name, + u16 *record_page_ptr) +{ + u32 physical_chapter = + uds_map_to_physical_chapter(geometry, chapter->virtual_chapter); + u32 index_page_number = + uds_find_index_page_number(index_page_map, name, physical_chapter); + struct delta_index_page *index_page = + &chapter->index_pages[index_page_number]; + + return uds_search_chapter_index_page(index_page, geometry, name, + record_page_ptr); +} + +int uds_search_sparse_cache(struct index_zone *zone, const struct uds_record_name *name, + u64 *virtual_chapter_ptr, u16 *record_page_ptr) +{ + int result; + struct volume *volume = zone->index->volume; + struct sparse_cache *cache = volume->sparse_cache; + struct cached_chapter_index *chapter; + struct search_list *search_list; + u8 i; + /* Search the entire cache unless a specific chapter was requested. */ + bool search_one = (*virtual_chapter_ptr != NO_CHAPTER); + + *record_page_ptr = NO_CHAPTER_INDEX_ENTRY; + search_list = cache->search_lists[zone->id]; + for (i = 0; i < search_list->first_dead_entry; i++) { + chapter = search_list->entries[i]; + + if (should_skip_chapter(chapter, zone->oldest_virtual_chapter, + *virtual_chapter_ptr)) + continue; + + result = search_cached_chapter_index(chapter, cache->geometry, + volume->index_page_map, name, + record_page_ptr); + if (result != UDS_SUCCESS) + return result; + + if (*record_page_ptr != NO_CHAPTER_INDEX_ENTRY) { + /* + * In theory, this might be a false match while a true match exists in + * another chapter, but that's a very rare case and not worth the extra + * search complexity. + */ + set_newest_entry(search_list, i); + if (zone->id == ZONE_ZERO) + score_search_hit(chapter); + + *virtual_chapter_ptr = chapter->virtual_chapter; + return UDS_SUCCESS; + } + + if (zone->id == ZONE_ZERO) + score_search_miss(cache, chapter); + + if (search_one) + break; + } + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/sparse-cache.h b/drivers/md/dm-vdo/sparse-cache.h new file mode 100644 index 000000000000..522027395cba --- /dev/null +++ b/drivers/md/dm-vdo/sparse-cache.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_SPARSE_CACHE_H +#define UDS_SPARSE_CACHE_H + +#include "geometry.h" +#include "uds.h" + +/* + * The sparse cache is a cache of entire chapter indexes from sparse chapters used for searching + * for names after all other search paths have failed. It contains only complete chapter indexes; + * record pages from sparse chapters and single index pages used for resolving hooks are kept in + * the regular page cache in the volume. + * + * The most important property of this cache is the absence of synchronization for read operations. + * Safe concurrent access to the cache by the zone threads is controlled by the triage queue and + * the barrier requests it issues to the zone queues. The set of cached chapters does not and must + * not change between the carefully coordinated calls to uds_update_sparse_cache() from the zone + * threads. Outside of updates, every zone will get the same result when calling + * uds_sparse_cache_contains() as every other zone. + */ + +struct index_zone; +struct sparse_cache; + +int __must_check uds_make_sparse_cache(const struct geometry *geometry, + unsigned int capacity, unsigned int zone_count, + struct sparse_cache **cache_ptr); + +void uds_free_sparse_cache(struct sparse_cache *cache); + +bool uds_sparse_cache_contains(struct sparse_cache *cache, u64 virtual_chapter, + unsigned int zone_number); + +int __must_check uds_update_sparse_cache(struct index_zone *zone, u64 virtual_chapter); + +void uds_invalidate_sparse_cache(struct sparse_cache *cache); + +int __must_check uds_search_sparse_cache(struct index_zone *zone, + const struct uds_record_name *name, + u64 *virtual_chapter_ptr, u16 *record_page_ptr); + +#endif /* UDS_SPARSE_CACHE_H */ From 8ce89dde122b4e3913552b68983e95e606710837 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:57:44 -0500 Subject: [PATCH 0564/1038] dm vdo: implement external deduplication index interface The deduplication index interface for index clients includes the deduplication request and index session structures. This is the interface that the rest of the vdo target uses to make requests, receive responses, and collect statistics. This patch also adds sysfs nodes for inspecting various index properties at runtime. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Thomas Jaskiewicz Signed-off-by: Thomas Jaskiewicz Co-developed-by: John Wiele Signed-off-by: John Wiele Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/index-session.c | 756 ++++++++++++++++++++++++++++++ drivers/md/dm-vdo/index-session.h | 84 ++++ drivers/md/dm-vdo/uds-sysfs.c | 186 ++++++++ drivers/md/dm-vdo/uds-sysfs.h | 12 + drivers/md/dm-vdo/uds.h | 329 +++++++++++++ 5 files changed, 1367 insertions(+) create mode 100644 drivers/md/dm-vdo/index-session.c create mode 100644 drivers/md/dm-vdo/index-session.h create mode 100644 drivers/md/dm-vdo/uds-sysfs.c create mode 100644 drivers/md/dm-vdo/uds-sysfs.h create mode 100644 drivers/md/dm-vdo/uds.h diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c new file mode 100644 index 000000000000..615c7b840d7b --- /dev/null +++ b/drivers/md/dm-vdo/index-session.c @@ -0,0 +1,756 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "index-session.h" + +#include + +#include "funnel-requestqueue.h" +#include "index.h" +#include "index-layout.h" +#include "logger.h" +#include "memory-alloc.h" +#include "time-utils.h" + +/* + * The index session contains a lock (the request_mutex) which ensures that only one thread can + * change the state of its index at a time. The state field indicates the current state of the + * index through a set of descriptive flags. The request_mutex must be notified whenever a + * non-transient state flag is cleared. The request_mutex is also used to count the number of + * requests currently in progress so that they can be drained when suspending or closing the index. + * + * If the index session is suspended shortly after opening an index, it may have to suspend during + * a rebuild. Depending on the size of the index, a rebuild may take a significant amount of time, + * so UDS allows the rebuild to be paused in order to suspend the session in a timely manner. When + * the index session is resumed, the rebuild can continue from where it left off. If the index + * session is shut down with a suspended rebuild, the rebuild progress is abandoned and the rebuild + * will start from the beginning the next time the index is loaded. The mutex and status fields in + * the index_load_context are used to record the state of any interrupted rebuild. + */ + +enum index_session_flag_bit { + IS_FLAG_BIT_START = 8, + /* The session has started loading an index but not completed it. */ + IS_FLAG_BIT_LOADING = IS_FLAG_BIT_START, + /* The session has loaded an index, which can handle requests. */ + IS_FLAG_BIT_LOADED, + /* The session's index has been permanently disabled. */ + IS_FLAG_BIT_DISABLED, + /* The session's index is suspended. */ + IS_FLAG_BIT_SUSPENDED, + /* The session is handling some index state change. */ + IS_FLAG_BIT_WAITING, + /* The session's index is closing and draining requests. */ + IS_FLAG_BIT_CLOSING, + /* The session is being destroyed and is draining requests. */ + IS_FLAG_BIT_DESTROYING, +}; + +enum index_session_flag { + IS_FLAG_LOADED = (1 << IS_FLAG_BIT_LOADED), + IS_FLAG_LOADING = (1 << IS_FLAG_BIT_LOADING), + IS_FLAG_DISABLED = (1 << IS_FLAG_BIT_DISABLED), + IS_FLAG_SUSPENDED = (1 << IS_FLAG_BIT_SUSPENDED), + IS_FLAG_WAITING = (1 << IS_FLAG_BIT_WAITING), + IS_FLAG_CLOSING = (1 << IS_FLAG_BIT_CLOSING), + IS_FLAG_DESTROYING = (1 << IS_FLAG_BIT_DESTROYING), +}; + +/* Release a reference to an index session. */ +static void release_index_session(struct uds_index_session *index_session) +{ + uds_lock_mutex(&index_session->request_mutex); + if (--index_session->request_count == 0) + uds_broadcast_cond(&index_session->request_cond); + uds_unlock_mutex(&index_session->request_mutex); +} + +/* + * Acquire a reference to the index session for an asynchronous index request. The reference must + * eventually be released with a corresponding call to release_index_session(). + */ +static int get_index_session(struct uds_index_session *index_session) +{ + unsigned int state; + int result = UDS_SUCCESS; + + uds_lock_mutex(&index_session->request_mutex); + index_session->request_count++; + state = index_session->state; + uds_unlock_mutex(&index_session->request_mutex); + + if (state == IS_FLAG_LOADED) { + return UDS_SUCCESS; + } else if (state & IS_FLAG_DISABLED) { + result = UDS_DISABLED; + } else if ((state & IS_FLAG_LOADING) || + (state & IS_FLAG_SUSPENDED) || + (state & IS_FLAG_WAITING)) { + result = -EBUSY; + } else { + result = UDS_NO_INDEX; + } + + release_index_session(index_session); + return result; +} + +int uds_launch_request(struct uds_request *request) +{ + size_t internal_size; + int result; + + if (request->callback == NULL) { + uds_log_error("missing required callback"); + return -EINVAL; + } + + switch (request->type) { + case UDS_DELETE: + case UDS_POST: + case UDS_QUERY: + case UDS_QUERY_NO_UPDATE: + case UDS_UPDATE: + break; + default: + uds_log_error("received invalid callback type"); + return -EINVAL; + } + + /* Reset all internal fields before processing. */ + internal_size = + sizeof(struct uds_request) - offsetof(struct uds_request, zone_number); + // FIXME should be using struct_group for this instead + memset((char *) request + sizeof(*request) - internal_size, 0, internal_size); + + result = get_index_session(request->session); + if (result != UDS_SUCCESS) + return result; + + request->found = false; + request->unbatched = false; + request->index = request->session->index; + + uds_enqueue_request(request, STAGE_TRIAGE); + return UDS_SUCCESS; +} + +static void enter_callback_stage(struct uds_request *request) +{ + if (request->status != UDS_SUCCESS) { + /* All request errors are considered unrecoverable */ + uds_lock_mutex(&request->session->request_mutex); + request->session->state |= IS_FLAG_DISABLED; + uds_unlock_mutex(&request->session->request_mutex); + } + + uds_request_queue_enqueue(request->session->callback_queue, request); +} + +static inline void count_once(u64 *count_ptr) +{ + WRITE_ONCE(*count_ptr, READ_ONCE(*count_ptr) + 1); +} + +static void update_session_stats(struct uds_request *request) +{ + struct session_stats *session_stats = &request->session->stats; + + count_once(&session_stats->requests); + + switch (request->type) { + case UDS_POST: + if (request->found) + count_once(&session_stats->posts_found); + else + count_once(&session_stats->posts_not_found); + + if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER) + count_once(&session_stats->posts_found_open_chapter); + else if (request->location == UDS_LOCATION_IN_DENSE) + count_once(&session_stats->posts_found_dense); + else if (request->location == UDS_LOCATION_IN_SPARSE) + count_once(&session_stats->posts_found_sparse); + break; + + case UDS_UPDATE: + if (request->found) + count_once(&session_stats->updates_found); + else + count_once(&session_stats->updates_not_found); + break; + + case UDS_DELETE: + if (request->found) + count_once(&session_stats->deletions_found); + else + count_once(&session_stats->deletions_not_found); + break; + + case UDS_QUERY: + case UDS_QUERY_NO_UPDATE: + if (request->found) + count_once(&session_stats->queries_found); + else + count_once(&session_stats->queries_not_found); + break; + + default: + request->status = ASSERT(false, "unknown request type: %d", + request->type); + } +} + +static void handle_callbacks(struct uds_request *request) +{ + struct uds_index_session *index_session = request->session; + + if (request->status == UDS_SUCCESS) + update_session_stats(request); + + request->status = uds_map_to_system_error(request->status); + request->callback(request); + release_index_session(index_session); +} + +static int __must_check make_empty_index_session(struct uds_index_session **index_session_ptr) +{ + int result; + struct uds_index_session *session; + + result = uds_allocate(1, struct uds_index_session, __func__, &session); + if (result != UDS_SUCCESS) + return result; + + result = uds_init_mutex(&session->request_mutex); + if (result != UDS_SUCCESS) { + uds_free(session); + return result; + } + + result = uds_init_cond(&session->request_cond); + if (result != UDS_SUCCESS) { + uds_destroy_mutex(&session->request_mutex); + uds_free(session); + return result; + } + + result = uds_init_mutex(&session->load_context.mutex); + if (result != UDS_SUCCESS) { + uds_destroy_cond(&session->request_cond); + uds_destroy_mutex(&session->request_mutex); + uds_free(session); + return result; + } + + result = uds_init_cond(&session->load_context.cond); + if (result != UDS_SUCCESS) { + uds_destroy_mutex(&session->load_context.mutex); + uds_destroy_cond(&session->request_cond); + uds_destroy_mutex(&session->request_mutex); + uds_free(session); + return result; + } + + result = uds_make_request_queue("callbackW", &handle_callbacks, + &session->callback_queue); + if (result != UDS_SUCCESS) { + uds_destroy_cond(&session->load_context.cond); + uds_destroy_mutex(&session->load_context.mutex); + uds_destroy_cond(&session->request_cond); + uds_destroy_mutex(&session->request_mutex); + uds_free(session); + return result; + } + + *index_session_ptr = session; + return UDS_SUCCESS; +} + +int uds_create_index_session(struct uds_index_session **session) +{ + if (session == NULL) { + uds_log_error("missing session pointer"); + return -EINVAL; + } + + return uds_map_to_system_error(make_empty_index_session(session)); +} + +static int __must_check start_loading_index_session(struct uds_index_session *index_session) +{ + int result; + + uds_lock_mutex(&index_session->request_mutex); + if (index_session->state & IS_FLAG_SUSPENDED) { + uds_log_info("Index session is suspended"); + result = -EBUSY; + } else if (index_session->state != 0) { + uds_log_info("Index is already loaded"); + result = -EBUSY; + } else { + index_session->state |= IS_FLAG_LOADING; + result = UDS_SUCCESS; + } + uds_unlock_mutex(&index_session->request_mutex); + return result; +} + +static void finish_loading_index_session(struct uds_index_session *index_session, + int result) +{ + uds_lock_mutex(&index_session->request_mutex); + index_session->state &= ~IS_FLAG_LOADING; + if (result == UDS_SUCCESS) + index_session->state |= IS_FLAG_LOADED; + + uds_broadcast_cond(&index_session->request_cond); + uds_unlock_mutex(&index_session->request_mutex); +} + +static int initialize_index_session(struct uds_index_session *index_session, + enum uds_open_index_type open_type) +{ + int result; + struct configuration *config; + + result = uds_make_configuration(&index_session->parameters, &config); + if (result != UDS_SUCCESS) { + uds_log_error_strerror(result, "Failed to allocate config"); + return result; + } + + memset(&index_session->stats, 0, sizeof(index_session->stats)); + result = uds_make_index(config, open_type, &index_session->load_context, + enter_callback_stage, &index_session->index); + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "Failed to make index"); + else + uds_log_configuration(config); + + uds_free_configuration(config); + return result; +} + +static const char *get_open_type_string(enum uds_open_index_type open_type) +{ + switch (open_type) { + case UDS_CREATE: + return "creating index"; + case UDS_LOAD: + return "loading or rebuilding index"; + case UDS_NO_REBUILD: + return "loading index"; + default: + return "unknown open method"; + } +} + +/* + * Open an index under the given session. This operation will fail if the + * index session is suspended, or if there is already an open index. + */ +int uds_open_index(enum uds_open_index_type open_type, + const struct uds_parameters *parameters, + struct uds_index_session *session) +{ + int result; + char name[BDEVNAME_SIZE]; + + if (parameters == NULL) { + uds_log_error("missing required parameters"); + return -EINVAL; + } + if (parameters->bdev == NULL) { + uds_log_error("missing required block device"); + return -EINVAL; + } + if (session == NULL) { + uds_log_error("missing required session pointer"); + return -EINVAL; + } + + result = start_loading_index_session(session); + if (result != UDS_SUCCESS) + return uds_map_to_system_error(result); + + session->parameters = *parameters; + format_dev_t(name, parameters->bdev->bd_dev); + uds_log_notice("%s: %s", get_open_type_string(open_type), name); + + result = initialize_index_session(session, open_type); + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "Failed %s", + get_open_type_string(open_type)); + + finish_loading_index_session(session, result); + return uds_map_to_system_error(result); +} + +static void wait_for_no_requests_in_progress(struct uds_index_session *index_session) +{ + uds_lock_mutex(&index_session->request_mutex); + while (index_session->request_count > 0) + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + uds_unlock_mutex(&index_session->request_mutex); +} + +static int __must_check save_index(struct uds_index_session *index_session) +{ + wait_for_no_requests_in_progress(index_session); + return uds_save_index(index_session->index); +} + +static void suspend_rebuild(struct uds_index_session *session) +{ + uds_lock_mutex(&session->load_context.mutex); + switch (session->load_context.status) { + case INDEX_OPENING: + session->load_context.status = INDEX_SUSPENDING; + + /* Wait until the index indicates that it is not replaying. */ + while ((session->load_context.status != INDEX_SUSPENDED) && + (session->load_context.status != INDEX_READY)) { + uds_wait_cond(&session->load_context.cond, + &session->load_context.mutex); + } + + break; + + case INDEX_READY: + /* Index load does not need to be suspended. */ + break; + + case INDEX_SUSPENDED: + case INDEX_SUSPENDING: + case INDEX_FREEING: + default: + /* These cases should not happen. */ + ASSERT_LOG_ONLY(false, "Bad load context state %u", + session->load_context.status); + break; + } + uds_unlock_mutex(&session->load_context.mutex); +} + +/* + * Suspend index operation, draining all current index requests and preventing new index requests + * from starting. Optionally saves all index data before returning. + */ +int uds_suspend_index_session(struct uds_index_session *session, bool save) +{ + int result = UDS_SUCCESS; + bool no_work = false; + bool rebuilding = false; + + /* Wait for any current index state change to complete. */ + uds_lock_mutex(&session->request_mutex); + while (session->state & IS_FLAG_CLOSING) + uds_wait_cond(&session->request_cond, &session->request_mutex); + + if ((session->state & IS_FLAG_WAITING) || (session->state & IS_FLAG_DESTROYING)) { + no_work = true; + uds_log_info("Index session is already changing state"); + result = -EBUSY; + } else if (session->state & IS_FLAG_SUSPENDED) { + no_work = true; + } else if (session->state & IS_FLAG_LOADING) { + session->state |= IS_FLAG_WAITING; + rebuilding = true; + } else if (session->state & IS_FLAG_LOADED) { + session->state |= IS_FLAG_WAITING; + } else { + no_work = true; + session->state |= IS_FLAG_SUSPENDED; + uds_broadcast_cond(&session->request_cond); + } + uds_unlock_mutex(&session->request_mutex); + + if (no_work) + return uds_map_to_system_error(result); + + if (rebuilding) + suspend_rebuild(session); + else if (save) + result = save_index(session); + else + result = uds_flush_index_session(session); + + uds_lock_mutex(&session->request_mutex); + session->state &= ~IS_FLAG_WAITING; + session->state |= IS_FLAG_SUSPENDED; + uds_broadcast_cond(&session->request_cond); + uds_unlock_mutex(&session->request_mutex); + return uds_map_to_system_error(result); +} + +static int replace_device(struct uds_index_session *session, struct block_device *bdev) +{ + int result; + + result = uds_replace_index_storage(session->index, bdev); + if (result != UDS_SUCCESS) + return result; + + session->parameters.bdev = bdev; + return UDS_SUCCESS; +} + +/* + * Resume index operation after being suspended. If the index is suspended and the supplied block + * device differs from the current backing store, the index will start using the new backing store. + */ +int uds_resume_index_session(struct uds_index_session *session, + struct block_device *bdev) +{ + int result = UDS_SUCCESS; + bool no_work = false; + bool resume_replay = false; + + uds_lock_mutex(&session->request_mutex); + if (session->state & IS_FLAG_WAITING) { + uds_log_info("Index session is already changing state"); + no_work = true; + result = -EBUSY; + } else if (!(session->state & IS_FLAG_SUSPENDED)) { + /* If not suspended, just succeed. */ + no_work = true; + result = UDS_SUCCESS; + } else { + session->state |= IS_FLAG_WAITING; + if (session->state & IS_FLAG_LOADING) + resume_replay = true; + } + uds_unlock_mutex(&session->request_mutex); + + if (no_work) + return result; + + if ((session->index != NULL) && (bdev != session->parameters.bdev)) { + result = replace_device(session, bdev); + if (result != UDS_SUCCESS) { + uds_lock_mutex(&session->request_mutex); + session->state &= ~IS_FLAG_WAITING; + uds_broadcast_cond(&session->request_cond); + uds_unlock_mutex(&session->request_mutex); + return uds_map_to_system_error(result); + } + } + + if (resume_replay) { + uds_lock_mutex(&session->load_context.mutex); + switch (session->load_context.status) { + case INDEX_SUSPENDED: + session->load_context.status = INDEX_OPENING; + /* Notify the index to start replaying again. */ + uds_broadcast_cond(&session->load_context.cond); + break; + + case INDEX_READY: + /* There is no index rebuild to resume. */ + break; + + case INDEX_OPENING: + case INDEX_SUSPENDING: + case INDEX_FREEING: + default: + /* These cases should not happen; do nothing. */ + ASSERT_LOG_ONLY(false, "Bad load context state %u", + session->load_context.status); + break; + } + uds_unlock_mutex(&session->load_context.mutex); + } + + uds_lock_mutex(&session->request_mutex); + session->state &= ~IS_FLAG_WAITING; + session->state &= ~IS_FLAG_SUSPENDED; + uds_broadcast_cond(&session->request_cond); + uds_unlock_mutex(&session->request_mutex); + return UDS_SUCCESS; +} + +static int save_and_free_index(struct uds_index_session *index_session) +{ + int result = UDS_SUCCESS; + bool suspended; + struct uds_index *index = index_session->index; + + if (index == NULL) + return UDS_SUCCESS; + + uds_lock_mutex(&index_session->request_mutex); + suspended = (index_session->state & IS_FLAG_SUSPENDED); + uds_unlock_mutex(&index_session->request_mutex); + + if (!suspended) { + result = uds_save_index(index); + if (result != UDS_SUCCESS) + uds_log_warning_strerror(result, + "ignoring error from save_index"); + } + uds_free_index(index); + index_session->index = NULL; + + /* + * Reset all index state that happens to be in the index + * session, so it doesn't affect any future index. + */ + uds_lock_mutex(&index_session->load_context.mutex); + index_session->load_context.status = INDEX_OPENING; + uds_unlock_mutex(&index_session->load_context.mutex); + + uds_lock_mutex(&index_session->request_mutex); + /* Only the suspend bit will remain relevant. */ + index_session->state &= IS_FLAG_SUSPENDED; + uds_unlock_mutex(&index_session->request_mutex); + + return result; +} + +/* Save and close the current index. */ +int uds_close_index(struct uds_index_session *index_session) +{ + int result = UDS_SUCCESS; + + /* Wait for any current index state change to complete. */ + uds_lock_mutex(&index_session->request_mutex); + while ((index_session->state & IS_FLAG_WAITING) || + (index_session->state & IS_FLAG_CLOSING)) + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + + if (index_session->state & IS_FLAG_SUSPENDED) { + uds_log_info("Index session is suspended"); + result = -EBUSY; + } else if ((index_session->state & IS_FLAG_DESTROYING) || + !(index_session->state & IS_FLAG_LOADED)) { + /* The index doesn't exist, hasn't finished loading, or is being destroyed. */ + result = UDS_NO_INDEX; + } else { + index_session->state |= IS_FLAG_CLOSING; + } + uds_unlock_mutex(&index_session->request_mutex); + if (result != UDS_SUCCESS) + return uds_map_to_system_error(result); + + uds_log_debug("Closing index"); + wait_for_no_requests_in_progress(index_session); + result = save_and_free_index(index_session); + uds_log_debug("Closed index"); + + uds_lock_mutex(&index_session->request_mutex); + index_session->state &= ~IS_FLAG_CLOSING; + uds_broadcast_cond(&index_session->request_cond); + uds_unlock_mutex(&index_session->request_mutex); + return uds_map_to_system_error(result); +} + +/* This will save and close an open index before destroying the session. */ +int uds_destroy_index_session(struct uds_index_session *index_session) +{ + int result; + bool load_pending = false; + + uds_log_debug("Destroying index session"); + + /* Wait for any current index state change to complete. */ + uds_lock_mutex(&index_session->request_mutex); + while ((index_session->state & IS_FLAG_WAITING) || + (index_session->state & IS_FLAG_CLOSING)) + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + + if (index_session->state & IS_FLAG_DESTROYING) { + uds_unlock_mutex(&index_session->request_mutex); + uds_log_info("Index session is already closing"); + return -EBUSY; + } + + index_session->state |= IS_FLAG_DESTROYING; + load_pending = ((index_session->state & IS_FLAG_LOADING) && + (index_session->state & IS_FLAG_SUSPENDED)); + uds_unlock_mutex(&index_session->request_mutex); + + if (load_pending) { + /* Tell the index to terminate the rebuild. */ + uds_lock_mutex(&index_session->load_context.mutex); + if (index_session->load_context.status == INDEX_SUSPENDED) { + index_session->load_context.status = INDEX_FREEING; + uds_broadcast_cond(&index_session->load_context.cond); + } + uds_unlock_mutex(&index_session->load_context.mutex); + + /* Wait until the load exits before proceeding. */ + uds_lock_mutex(&index_session->request_mutex); + while (index_session->state & IS_FLAG_LOADING) + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + uds_unlock_mutex(&index_session->request_mutex); + } + + wait_for_no_requests_in_progress(index_session); + result = save_and_free_index(index_session); + uds_request_queue_finish(index_session->callback_queue); + index_session->callback_queue = NULL; + uds_destroy_cond(&index_session->load_context.cond); + uds_destroy_mutex(&index_session->load_context.mutex); + uds_destroy_cond(&index_session->request_cond); + uds_destroy_mutex(&index_session->request_mutex); + uds_log_debug("Destroyed index session"); + uds_free(index_session); + return uds_map_to_system_error(result); +} + +/* Wait until all callbacks for index operations are complete. */ +int uds_flush_index_session(struct uds_index_session *index_session) +{ + wait_for_no_requests_in_progress(index_session); + uds_wait_for_idle_index(index_session->index); + return UDS_SUCCESS; +} + +/* Statistics collection is intended to be thread-safe. */ +static void collect_stats(const struct uds_index_session *index_session, + struct uds_index_stats *stats) +{ + const struct session_stats *session_stats = &index_session->stats; + + stats->current_time = ktime_to_seconds(current_time_ns(CLOCK_REALTIME)); + stats->posts_found = READ_ONCE(session_stats->posts_found); + stats->in_memory_posts_found = READ_ONCE(session_stats->posts_found_open_chapter); + stats->dense_posts_found = READ_ONCE(session_stats->posts_found_dense); + stats->sparse_posts_found = READ_ONCE(session_stats->posts_found_sparse); + stats->posts_not_found = READ_ONCE(session_stats->posts_not_found); + stats->updates_found = READ_ONCE(session_stats->updates_found); + stats->updates_not_found = READ_ONCE(session_stats->updates_not_found); + stats->deletions_found = READ_ONCE(session_stats->deletions_found); + stats->deletions_not_found = READ_ONCE(session_stats->deletions_not_found); + stats->queries_found = READ_ONCE(session_stats->queries_found); + stats->queries_not_found = READ_ONCE(session_stats->queries_not_found); + stats->requests = READ_ONCE(session_stats->requests); +} + +int uds_get_index_session_stats(struct uds_index_session *index_session, + struct uds_index_stats *stats) +{ + if (stats == NULL) { + uds_log_error("received a NULL index stats pointer"); + return -EINVAL; + } + + collect_stats(index_session, stats); + if (index_session->index != NULL) { + uds_get_index_stats(index_session->index, stats); + } else { + stats->entries_indexed = 0; + stats->memory_used = 0; + stats->collisions = 0; + stats->entries_discarded = 0; + } + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/index-session.h b/drivers/md/dm-vdo/index-session.h new file mode 100644 index 000000000000..c77ee021d510 --- /dev/null +++ b/drivers/md/dm-vdo/index-session.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_SESSION_H +#define UDS_INDEX_SESSION_H + +#include +#include + +#include "config.h" +#include "uds-threads.h" +#include "uds.h" + +/* + * The index session mediates all interactions with a UDS index. Once the index session is created, + * it can be used to open, close, suspend, or recreate an index. It implements the majority of the + * functions in the top-level UDS API. + * + * If any deduplication request fails due to an internal error, the index is marked disabled. It + * will not accept any further requests and can only be closed. Closing the index will clear the + * disabled flag, and the index can then be reopened and recovered using the same index session. + */ + +struct __aligned(L1_CACHE_BYTES) session_stats { + /* Post requests that found an entry */ + u64 posts_found; + /* Post requests found in the open chapter */ + u64 posts_found_open_chapter; + /* Post requests found in the dense index */ + u64 posts_found_dense; + /* Post requests found in the sparse index */ + u64 posts_found_sparse; + /* Post requests that did not find an entry */ + u64 posts_not_found; + /* Update requests that found an entry */ + u64 updates_found; + /* Update requests that did not find an entry */ + u64 updates_not_found; + /* Delete requests that found an entry */ + u64 deletions_found; + /* Delete requests that did not find an entry */ + u64 deletions_not_found; + /* Query requests that found an entry */ + u64 queries_found; + /* Query requests that did not find an entry */ + u64 queries_not_found; + /* Total number of requests */ + u64 requests; +}; + +enum index_suspend_status { + /* An index load has started but the index is not ready for use. */ + INDEX_OPENING = 0, + /* The index is able to handle requests. */ + INDEX_READY, + /* The index is attempting to suspend a rebuild. */ + INDEX_SUSPENDING, + /* An index rebuild has been suspended. */ + INDEX_SUSPENDED, + /* An index rebuild is being stopped in order to shut down. */ + INDEX_FREEING, +}; + +struct index_load_context { + struct mutex mutex; + struct cond_var cond; + enum index_suspend_status status; +}; + +struct uds_index_session { + unsigned int state; + struct uds_index *index; + struct uds_request_queue *callback_queue; + struct uds_parameters parameters; + struct index_load_context load_context; + struct mutex request_mutex; + struct cond_var request_cond; + int request_count; + struct session_stats stats; +}; + +#endif /* UDS_INDEX_SESSION_H */ diff --git a/drivers/md/dm-vdo/uds-sysfs.c b/drivers/md/dm-vdo/uds-sysfs.c new file mode 100644 index 000000000000..eee8a5b7d147 --- /dev/null +++ b/drivers/md/dm-vdo/uds-sysfs.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "uds-sysfs.h" + +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "string-utils.h" +#include "uds.h" + +#define UDS_SYSFS_NAME "uds" + +static struct { + /* /sys/uds */ + struct kobject kobj; + /* /sys/uds/parameter */ + struct kobject parameter_kobj; + + /* These flags are used to ensure a clean shutdown */ + + /* /sys/uds flag */ + bool flag; + /* /sys/uds/parameter flag */ + bool parameter_flag; +} object_root; + +static char *buffer_to_string(const char *buf, size_t length) +{ + char *string; + + if (uds_allocate(length + 1, char, __func__, &string) != UDS_SUCCESS) + return NULL; + + memcpy(string, buf, length); + string[length] = '\0'; + if (string[length - 1] == '\n') + string[length - 1] = '\0'; + + return string; +} + +/* + * This is the code for any directory in the /sys/ tree that contains no regular files + * (only subdirectories). + */ + +static void empty_release(struct kobject *kobj) +{ +} + +static ssize_t empty_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + return 0; +} + +static ssize_t empty_store(struct kobject *kobj, struct attribute *attr, const char *buf, + size_t length) +{ + return length; +} + +static const struct sysfs_ops empty_ops = { + .show = empty_show, + .store = empty_store, +}; + +static struct attribute *empty_attrs[] = { + NULL, +}; +ATTRIBUTE_GROUPS(empty); + +static const struct kobj_type empty_object_type = { + .release = empty_release, + .sysfs_ops = &empty_ops, + .default_groups = empty_groups, +}; + +/* + * This is the code for the /sys//parameter directory. + * /log_level UDS_LOG_LEVEL + */ + +struct parameter_attribute { + struct attribute attr; + const char *(*show_string)(void); + void (*store_string)(const char *string); +}; + +static ssize_t parameter_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct parameter_attribute *pa; + + pa = container_of(attr, struct parameter_attribute, attr); + if (pa->show_string != NULL) + return sprintf(buf, "%s\n", pa->show_string()); + else + return -EINVAL; +} + +static ssize_t parameter_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t length) +{ + char *string; + struct parameter_attribute *pa; + + pa = container_of(attr, struct parameter_attribute, attr); + if (pa->store_string == NULL) + return -EINVAL; + string = buffer_to_string(buf, length); + if (string == NULL) + return -ENOMEM; + + pa->store_string(string); + uds_free(string); + return length; +} + +static const char *parameter_show_log_level(void) +{ + return uds_log_priority_to_string(uds_get_log_level()); +} + +static void parameter_store_log_level(const char *string) +{ + uds_set_log_level(uds_log_string_to_priority(string)); +} + +static struct parameter_attribute log_level_attr = { + .attr = { .name = "log_level", .mode = 0600 }, + .show_string = parameter_show_log_level, + .store_string = parameter_store_log_level, +}; + +static struct attribute *parameter_attrs[] = { + &log_level_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(parameter); + +static const struct sysfs_ops parameter_ops = { + .show = parameter_show, + .store = parameter_store, +}; + +static const struct kobj_type parameter_object_type = { + .release = empty_release, + .sysfs_ops = ¶meter_ops, + .default_groups = parameter_groups, +}; + +int uds_init_sysfs(void) +{ + int result; + + memset(&object_root, 0, sizeof(object_root)); + kobject_init(&object_root.kobj, &empty_object_type); + result = kobject_add(&object_root.kobj, NULL, UDS_SYSFS_NAME); + if (result == 0) { + object_root.flag = true; + kobject_init(&object_root.parameter_kobj, ¶meter_object_type); + result = kobject_add(&object_root.parameter_kobj, &object_root.kobj, + "parameter"); + if (result == 0) + object_root.parameter_flag = true; + } + + if (result != 0) + uds_put_sysfs(); + + return result; +} + +void uds_put_sysfs(void) +{ + if (object_root.parameter_flag) + kobject_put(&object_root.parameter_kobj); + + if (object_root.flag) + kobject_put(&object_root.kobj); +} diff --git a/drivers/md/dm-vdo/uds-sysfs.h b/drivers/md/dm-vdo/uds-sysfs.h new file mode 100644 index 000000000000..c3d00a7187bd --- /dev/null +++ b/drivers/md/dm-vdo/uds-sysfs.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_SYSFS_H +#define UDS_SYSFS_H + +int uds_init_sysfs(void); +void uds_put_sysfs(void); + +#endif /* UDS_SYSFS_H */ diff --git a/drivers/md/dm-vdo/uds.h b/drivers/md/dm-vdo/uds.h new file mode 100644 index 000000000000..1264362f8372 --- /dev/null +++ b/drivers/md/dm-vdo/uds.h @@ -0,0 +1,329 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_H +#define UDS_H + +#include + +#include "funnel-queue.h" + +/* + * UDS public API + * + * The Universal Deduplication System (UDS) is an efficient name-value store. When used for + * deduplicating storage, the names are generally hashes of data blocks and the associated data is + * where that block is located on the underlying storage medium. The stored names are expected to + * be randomly distributed among the space of possible names. If this assumption is violated, the + * UDS index will store fewer names than normal but will otherwise continue to work. The data + * associated with each name can be any 16-byte value. + * + * A client must first create an index session to interact with an index. Once created, the session + * can be shared among multiple threads or users. When a session is destroyed, it will also close + * and save any associated index. + * + * To make a request, a client must allocate a uds_request structure and set the required fields + * before launching it. UDS will invoke the provided callback to complete the request. After the + * callback has been called, the uds_request structure can be freed or reused for a new request. + * There are five types of requests: + * + * A UDS_UPDATE request will associate the provided name with the provided data. Any previous data + * associated with that name will be discarded. + * + * A UDS_QUERY request will return the data associated with the provided name, if any. The entry + * for the name will also be marked as most recent, as if the data had been updated. + * + * A UDS_POST request is a combination of UDS_QUERY and UDS_UPDATE. If there is already data + * associated with the provided name, that data is returned. If there is no existing association, + * the name is associated with the newly provided data. This request is equivalent to a UDS_QUERY + * request followed by a UDS_UPDATE request if no data is found, but it is much more efficient. + * + * A UDS_QUERY_NO_UPDATE request will return the data associated with the provided name, but will + * not change the recency of the entry for the name. This request is primarily useful for testing, + * to determine whether an entry exists without changing the internal state of the index. + * + * A UDS_DELETE request removes any data associated with the provided name. This operation is + * generally not necessary, because the index will automatically discard its oldest entries once it + * becomes full. + */ + +/* General UDS constants and structures */ + +enum uds_request_type { + /* Create or update the mapping for a name, and make the name most recent. */ + UDS_UPDATE, + + /* Return any mapped data for a name, and make the name most recent. */ + UDS_QUERY, + + /* + * Return any mapped data for a name, or map the provided data to the name if there is no + * current data, and make the name most recent. + */ + UDS_POST, + + /* Return any mapped data for a name without updating its recency. */ + UDS_QUERY_NO_UPDATE, + + /* Remove any mapping for a name. */ + UDS_DELETE, + +}; + +enum uds_open_index_type { + /* Create a new index. */ + UDS_CREATE, + + /* Load an existing index and try to recover if necessary. */ + UDS_LOAD, + + /* Load an existing index, but only if it was saved cleanly. */ + UDS_NO_REBUILD, +}; + +enum { + /* The record name size in bytes */ + UDS_RECORD_NAME_SIZE = 16, + /* The maximum record data size in bytes */ + UDS_RECORD_DATA_SIZE = 16, +}; + +/* + * A type representing a UDS memory configuration which is either a positive integer number of + * gigabytes or one of the six special constants for configurations smaller than one gigabyte. + */ +typedef int uds_memory_config_size_t; + +enum { + /* The maximum configurable amount of memory */ + UDS_MEMORY_CONFIG_MAX = 1024, + /* Flag indicating that the index has one less chapter than usual */ + UDS_MEMORY_CONFIG_REDUCED = 0x1000, + UDS_MEMORY_CONFIG_REDUCED_MAX = 1024 + UDS_MEMORY_CONFIG_REDUCED, + /* Special values indicating sizes less than 1 GB */ + UDS_MEMORY_CONFIG_256MB = -256, + UDS_MEMORY_CONFIG_512MB = -512, + UDS_MEMORY_CONFIG_768MB = -768, + UDS_MEMORY_CONFIG_REDUCED_256MB = -1280, + UDS_MEMORY_CONFIG_REDUCED_512MB = -1536, + UDS_MEMORY_CONFIG_REDUCED_768MB = -1792, +}; + +struct uds_record_name { + unsigned char name[UDS_RECORD_NAME_SIZE]; +}; + +struct uds_record_data { + unsigned char data[UDS_RECORD_DATA_SIZE]; +}; + +struct uds_volume_record { + struct uds_record_name name; + struct uds_record_data data; +}; + +struct uds_parameters { + /* The block_device used for storage */ + struct block_device *bdev; + /* The maximum allowable size of the index on storage */ + size_t size; + /* The offset where the index should start */ + off_t offset; + /* The maximum memory allocation, in GB */ + uds_memory_config_size_t memory_size; + /* Whether the index should include sparse chapters */ + bool sparse; + /* A 64-bit nonce to validate the index */ + u64 nonce; + /* The number of threads used to process index requests */ + unsigned int zone_count; + /* The number of threads used to read volume pages */ + unsigned int read_threads; +}; + +/* + * These statistics capture characteristics of the current index, including resource usage and + * requests processed since the index was opened. + */ +struct uds_index_stats { + /* The total number of records stored in the index */ + u64 entries_indexed; + /* An estimate of the index's memory usage, in bytes */ + u64 memory_used; + /* The number of collisions recorded in the volume index */ + u64 collisions; + /* The number of entries discarded from the index since startup */ + u64 entries_discarded; + /* The time at which these statistics were fetched */ + s64 current_time; + /* The number of post calls that found an existing entry */ + u64 posts_found; + /* The number of post calls that added an entry */ + u64 posts_not_found; + /* + * The number of post calls that found an existing entry that is current enough to only + * exist in memory and not have been committed to disk yet + */ + u64 in_memory_posts_found; + /* + * The number of post calls that found an existing entry in the dense portion of the index + */ + u64 dense_posts_found; + /* + * The number of post calls that found an existing entry in the sparse portion of the index + */ + u64 sparse_posts_found; + /* The number of update calls that updated an existing entry */ + u64 updates_found; + /* The number of update calls that added a new entry */ + u64 updates_not_found; + /* The number of delete requests that deleted an existing entry */ + u64 deletions_found; + /* The number of delete requests that did nothing */ + u64 deletions_not_found; + /* The number of query calls that found existing entry */ + u64 queries_found; + /* The number of query calls that did not find an entry */ + u64 queries_not_found; + /* The total number of requests processed */ + u64 requests; +}; + +enum uds_index_region { + /* No location information has been determined */ + UDS_LOCATION_UNKNOWN = 0, + /* The index page entry has been found */ + UDS_LOCATION_INDEX_PAGE_LOOKUP, + /* The record page entry has been found */ + UDS_LOCATION_RECORD_PAGE_LOOKUP, + /* The record is not in the index */ + UDS_LOCATION_UNAVAILABLE, + /* The record was found in the open chapter */ + UDS_LOCATION_IN_OPEN_CHAPTER, + /* The record was found in the dense part of the index */ + UDS_LOCATION_IN_DENSE, + /* The record was found in the sparse part of the index */ + UDS_LOCATION_IN_SPARSE, +} __packed; + +/* Zone message requests are used to communicate between index zones. */ +enum uds_zone_message_type { + /* A standard request with no message */ + UDS_MESSAGE_NONE = 0, + /* Add a chapter to the sparse chapter index cache */ + UDS_MESSAGE_SPARSE_CACHE_BARRIER, + /* Close a chapter to keep the zone from falling behind */ + UDS_MESSAGE_ANNOUNCE_CHAPTER_CLOSED, +} __packed; + +struct uds_zone_message { + /* The type of message, determining how it will be processed */ + enum uds_zone_message_type type; + /* The virtual chapter number to which the message applies */ + u64 virtual_chapter; +}; + +struct uds_index_session; +struct uds_index; +struct uds_request; + +/* Once this callback has been invoked, the uds_request structure can be reused or freed. */ +typedef void (*uds_request_callback_fn)(struct uds_request *request); + +struct uds_request { + /* These input fields must be set before launching a request. */ + + /* The name of the record to look up or create */ + struct uds_record_name record_name; + /* New data to associate with the record name, if applicable */ + struct uds_record_data new_metadata; + /* A callback to invoke when the request is complete */ + uds_request_callback_fn callback; + /* The index session that will manage this request */ + struct uds_index_session *session; + /* The type of operation to perform, as describe above */ + enum uds_request_type type; + + /* These output fields are set when a request is complete. */ + + /* The existing data associated with the request name, if any */ + struct uds_record_data old_metadata; + /* Either UDS_SUCCESS or an error code for the request */ + int status; + /* True if the record name had an existing entry in the index */ + bool found; + + /* + * The remaining fields are used internally and should not be altered by clients. The index + * relies on zone_number being the first field in this section. + */ + + /* The number of the zone which will process this request*/ + unsigned int zone_number; + /* A link for adding a request to a lock-free queue */ + struct funnel_queue_entry queue_link; + /* A link for adding a request to a standard linked list */ + struct uds_request *next_request; + /* A pointer to the index processing this request */ + struct uds_index *index; + /* Control message for coordinating between zones */ + struct uds_zone_message zone_message; + /* If true, process request immediately by waking the worker thread */ + bool unbatched; + /* If true, continue this request before processing newer requests */ + bool requeued; + /* The virtual chapter containing the record name, if known */ + u64 virtual_chapter; + /* The region of the index containing the record name */ + enum uds_index_region location; +}; + +/* Compute the number of bytes needed to store an index. */ +int __must_check uds_compute_index_size(const struct uds_parameters *parameters, + u64 *index_size); + +/* A session is required for most index operations. */ +int __must_check uds_create_index_session(struct uds_index_session **session); + +/* Destroying an index session also closes and saves the associated index. */ +int uds_destroy_index_session(struct uds_index_session *session); + +/* + * Create or open an index with an existing session. This operation fails if the index session is + * suspended, or if there is already an open index. + */ +int __must_check uds_open_index(enum uds_open_index_type open_type, + const struct uds_parameters *parameters, + struct uds_index_session *session); + +/* + * Wait until all callbacks for index operations are complete, and prevent new index operations + * from starting. New index operations will fail with EBUSY until the session is resumed. Also + * optionally saves the index. + */ +int __must_check uds_suspend_index_session(struct uds_index_session *session, bool save); + +/* + * Allow new index operations for an index, whether it was suspended or not. If the index is + * suspended and the supplied block device differs from the current backing store, the index will + * start using the new backing store instead. + */ +int __must_check uds_resume_index_session(struct uds_index_session *session, + struct block_device *bdev); + +/* Wait until all outstanding index operations are complete. */ +int __must_check uds_flush_index_session(struct uds_index_session *session); + +/* Close an index. This operation fails if the index session is suspended. */ +int __must_check uds_close_index(struct uds_index_session *session); + +/* Get index statistics since the last time the index was opened. */ +int __must_check uds_get_index_session_stats(struct uds_index_session *session, + struct uds_index_stats *stats); + +/* This function will fail if any required field of the request is not set. */ +int __must_check uds_launch_request(struct uds_request *request); + +#endif /* UDS_H */ From 3f493fcea0fc7b93dc1296030445575e363ccb51 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:58:24 -0500 Subject: [PATCH 0565/1038] dm vdo: add administrative state and action manager This patch adds the admin_state structures which are used to track the states of individual vdo components for handling of operations like suspend and resume. It also adds the action manager which is used to schedule and manage cross-thread administrative and internal operations. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/action-manager.c | 388 ++++++++++++++++++++++ drivers/md/dm-vdo/action-manager.h | 110 +++++++ drivers/md/dm-vdo/admin-state.c | 506 +++++++++++++++++++++++++++++ drivers/md/dm-vdo/admin-state.h | 178 ++++++++++ 4 files changed, 1182 insertions(+) create mode 100644 drivers/md/dm-vdo/action-manager.c create mode 100644 drivers/md/dm-vdo/action-manager.h create mode 100644 drivers/md/dm-vdo/admin-state.c create mode 100644 drivers/md/dm-vdo/admin-state.h diff --git a/drivers/md/dm-vdo/action-manager.c b/drivers/md/dm-vdo/action-manager.c new file mode 100644 index 000000000000..973901fc3174 --- /dev/null +++ b/drivers/md/dm-vdo/action-manager.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "action-manager.h" + +#include "memory-alloc.h" +#include "permassert.h" + +#include "admin-state.h" +#include "completion.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" + +/** + * struct action - An action to be performed in each of a set of zones. + * @in_use: Whether this structure is in use. + * @operation: The admin operation associated with this action. + * @preamble: The method to run on the initiator thread before the action is applied to each zone. + * @zone_action: The action to be performed in each zone. + * @conclusion: The method to run on the initiator thread after the action is applied to each zone. + * @parent: The object to notify when the action is complete. + * @context: The action specific context. + * @next: The action to perform after this one. + */ +struct action { + bool in_use; + const struct admin_state_code *operation; + vdo_action_preamble_fn preamble; + vdo_zone_action_fn zone_action; + vdo_action_conclusion_fn conclusion; + struct vdo_completion *parent; + void *context; + struct action *next; +}; + +/** + * struct action_manager - Definition of an action manager. + * @completion: The completion for performing actions. + * @state: The state of this action manager. + * @actions: The two action slots. + * @current_action: The current action slot. + * @zones: The number of zones in which an action is to be applied. + * @Scheduler: A function to schedule a default next action. + * @get_zone_thread_id: A function to get the id of the thread on which to apply an action to a + * zone. + * @initiator_thread_id: The ID of the thread on which actions may be initiated. + * @context: Opaque data associated with this action manager. + * @acting_zone: The zone currently being acted upon. + */ +struct action_manager { + struct vdo_completion completion; + struct admin_state state; + struct action actions[2]; + struct action *current_action; + zone_count_t zones; + vdo_action_scheduler_fn scheduler; + vdo_zone_thread_getter_fn get_zone_thread_id; + thread_id_t initiator_thread_id; + void *context; + zone_count_t acting_zone; +}; + +static inline struct action_manager *as_action_manager(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_ACTION_COMPLETION); + return container_of(completion, struct action_manager, completion); +} + +/* Implements vdo_action_scheduler_fn. */ +static bool no_default_action(void *context __always_unused) +{ + return false; +} + +/* Implements vdo_action_preamble_fn. */ +static void no_preamble(void *context __always_unused, struct vdo_completion *completion) +{ + vdo_finish_completion(completion); +} + +/* Implements vdo_action_conclusion_fn. */ +static int no_conclusion(void *context __always_unused) +{ + return VDO_SUCCESS; +} + +/** + * vdo_make_action_manager() - Make an action manager. + * @zones: The number of zones to which actions will be applied. + * @get_zone_thread_id: A function to get the thread id associated with a zone. + * @initiator_thread_id: The thread on which actions may initiated. + * @context: The object which holds the per-zone context for the action. + * @scheduler: A function to schedule a next action after an action concludes if there is no + * pending action (may be NULL). + * @vdo: The vdo used to initialize completions. + * @manager_ptr: A pointer to hold the new action manager. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_action_manager(zone_count_t zones, + vdo_zone_thread_getter_fn get_zone_thread_id, + thread_id_t initiator_thread_id, void *context, + vdo_action_scheduler_fn scheduler, struct vdo *vdo, + struct action_manager **manager_ptr) +{ + struct action_manager *manager; + int result = uds_allocate(1, struct action_manager, __func__, &manager); + + if (result != VDO_SUCCESS) + return result; + + *manager = (struct action_manager) { + .zones = zones, + .scheduler = + ((scheduler == NULL) ? no_default_action : scheduler), + .get_zone_thread_id = get_zone_thread_id, + .initiator_thread_id = initiator_thread_id, + .context = context, + }; + + manager->actions[0].next = &manager->actions[1]; + manager->current_action = manager->actions[1].next = + &manager->actions[0]; + vdo_set_admin_state_code(&manager->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + vdo_initialize_completion(&manager->completion, vdo, VDO_ACTION_COMPLETION); + *manager_ptr = manager; + return VDO_SUCCESS; +} + +const struct admin_state_code *vdo_get_current_manager_operation(struct action_manager *manager) +{ + return vdo_get_admin_state_code(&manager->state); +} + +void *vdo_get_current_action_context(struct action_manager *manager) +{ + return manager->current_action->in_use ? manager->current_action->context : NULL; +} + +static void finish_action_callback(struct vdo_completion *completion); +static void apply_to_zone(struct vdo_completion *completion); + +static thread_id_t get_acting_zone_thread_id(struct action_manager *manager) +{ + return manager->get_zone_thread_id(manager->context, manager->acting_zone); +} + +static void preserve_error(struct vdo_completion *completion) +{ + if (completion->parent != NULL) + vdo_set_completion_result(completion->parent, completion->result); + + vdo_reset_completion(completion); + vdo_run_completion(completion); +} + +static void prepare_for_next_zone(struct action_manager *manager) +{ + vdo_prepare_completion_for_requeue(&manager->completion, apply_to_zone, + preserve_error, + get_acting_zone_thread_id(manager), + manager->current_action->parent); +} + +static void prepare_for_conclusion(struct action_manager *manager) +{ + vdo_prepare_completion_for_requeue(&manager->completion, finish_action_callback, + preserve_error, manager->initiator_thread_id, + manager->current_action->parent); +} + +static void apply_to_zone(struct vdo_completion *completion) +{ + zone_count_t zone; + struct action_manager *manager = as_action_manager(completion); + + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == get_acting_zone_thread_id(manager)), + "%s() called on acting zones's thread", __func__); + + zone = manager->acting_zone++; + if (manager->acting_zone == manager->zones) { + /* + * We are about to apply to the last zone. Once that is finished, we're done, so go + * back to the initiator thread and finish up. + */ + prepare_for_conclusion(manager); + } else { + /* Prepare to come back on the next zone */ + prepare_for_next_zone(manager); + } + + manager->current_action->zone_action(manager->context, zone, completion); +} + +static void handle_preamble_error(struct vdo_completion *completion) +{ + /* Skip the zone actions since the preamble failed. */ + completion->callback = finish_action_callback; + preserve_error(completion); +} + +static void launch_current_action(struct action_manager *manager) +{ + struct action *action = manager->current_action; + int result = vdo_start_operation(&manager->state, action->operation); + + if (result != VDO_SUCCESS) { + if (action->parent != NULL) + vdo_set_completion_result(action->parent, result); + + /* We aren't going to run the preamble, so don't run the conclusion */ + action->conclusion = no_conclusion; + finish_action_callback(&manager->completion); + return; + } + + if (action->zone_action == NULL) { + prepare_for_conclusion(manager); + } else { + manager->acting_zone = 0; + vdo_prepare_completion_for_requeue(&manager->completion, apply_to_zone, + handle_preamble_error, + get_acting_zone_thread_id(manager), + manager->current_action->parent); + } + + action->preamble(manager->context, &manager->completion); +} + +/** + * vdo_schedule_default_action() - Attempt to schedule the default action. + * @manager: The action manager. + * + * If the manager is not operating normally, the action will not be scheduled. + * + * Return: true if an action was scheduled. + */ +bool vdo_schedule_default_action(struct action_manager *manager) +{ + /* Don't schedule a default action if we are operating or not in normal operation. */ + const struct admin_state_code *code = vdo_get_current_manager_operation(manager); + + return ((code == VDO_ADMIN_STATE_NORMAL_OPERATION) && + manager->scheduler(manager->context)); +} + +static void finish_action_callback(struct vdo_completion *completion) +{ + bool has_next_action; + int result; + struct action_manager *manager = as_action_manager(completion); + struct action action = *(manager->current_action); + + manager->current_action->in_use = false; + manager->current_action = manager->current_action->next; + + /* + * We need to check this now to avoid use-after-free issues if running the conclusion or + * notifying the parent results in the manager being freed. + */ + has_next_action = + (manager->current_action->in_use || vdo_schedule_default_action(manager)); + result = action.conclusion(manager->context); + vdo_finish_operation(&manager->state, VDO_SUCCESS); + if (action.parent != NULL) + vdo_continue_completion(action.parent, result); + + if (has_next_action) + launch_current_action(manager); +} + +/** + * vdo_schedule_action() - Schedule an action to be applied to all zones. + * @manager: The action manager to schedule the action on. + * @preamble: A method to be invoked on the initiator thread once this action is started but before + * applying to each zone; may be NULL. + * @action: The action to apply to each zone; may be NULL. + * @conclusion: A method to be invoked back on the initiator thread once the action has been + * applied to all zones; may be NULL. + * @parent: The object to notify once the action is complete or if the action can not be scheduled; + * may be NULL. + * + * The action will be launched immediately if there is no current action, or as soon as the current + * action completes. If there is already a pending action, this action will not be scheduled, and, + * if it has a parent, that parent will be notified. At least one of the preamble, action, or + * conclusion must not be NULL. + * + * Return: true if the action was scheduled. + */ +bool vdo_schedule_action(struct action_manager *manager, vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent) +{ + return vdo_schedule_operation(manager, VDO_ADMIN_STATE_OPERATING, preamble, + action, conclusion, parent); +} + +/** + * vdo_schedule_operation() - Schedule an operation to be applied to all zones. + * @manager: The action manager to schedule the action on. + * @operation: The operation this action will perform + * @preamble: A method to be invoked on the initiator thread once this action is started but before + * applying to each zone; may be NULL. + * @action: The action to apply to each zone; may be NULL. + * @conclusion: A method to be invoked back on the initiator thread once the action has been + * applied to all zones; may be NULL. + * @parent: The object to notify once the action is complete or if the action can not be scheduled; + * may be NULL. + * + * The operation's action will be launched immediately if there is no current action, or as soon as + * the current action completes. If there is already a pending action, this operation will not be + * scheduled, and, if it has a parent, that parent will be notified. At least one of the preamble, + * action, or conclusion must not be NULL. + * + * Return: true if the action was scheduled. + */ +bool vdo_schedule_operation(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent) +{ + return vdo_schedule_operation_with_context(manager, operation, preamble, action, + conclusion, NULL, parent); +} + +/** + * vdo_schedule_operation_with_context() - Schedule an operation on all zones. + * @manager: The action manager to schedule the action on. + * @operation: The operation this action will perform. + * @preamble: A method to be invoked on the initiator thread once this action is started but before + * applying to each zone; may be NULL. + * @action: The action to apply to each zone; may be NULL. + * @conclusion: A method to be invoked back on the initiator thread once the action has been + * applied to all zones; may be NULL. + * @context: An action-specific context which may be retrieved via + * vdo_get_current_action_context(); may be NULL. + * @parent: The object to notify once the action is complete or if the action can not be scheduled; + * may be NULL. + * + * The operation's action will be launched immediately if there is no current action, or as soon as + * the current action completes. If there is already a pending action, this operation will not be + * scheduled, and, if it has a parent, that parent will be notified. At least one of the preamble, + * action, or conclusion must not be NULL. + * + * Return: true if the action was scheduled + */ +bool vdo_schedule_operation_with_context(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + void *context, struct vdo_completion *parent) +{ + struct action *current_action; + + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == manager->initiator_thread_id), + "action initiated from correct thread"); + if (!manager->current_action->in_use) { + current_action = manager->current_action; + } else if (!manager->current_action->next->in_use) { + current_action = manager->current_action->next; + } else { + if (parent != NULL) + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + + return false; + } + + *current_action = (struct action) { + .in_use = true, + .operation = operation, + .preamble = (preamble == NULL) ? no_preamble : preamble, + .zone_action = action, + .conclusion = (conclusion == NULL) ? no_conclusion : conclusion, + .context = context, + .parent = parent, + .next = current_action->next, + }; + + if (current_action == manager->current_action) + launch_current_action(manager); + + return true; +} diff --git a/drivers/md/dm-vdo/action-manager.h b/drivers/md/dm-vdo/action-manager.h new file mode 100644 index 000000000000..b0a8d3ddf3db --- /dev/null +++ b/drivers/md/dm-vdo/action-manager.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_ACTION_MANAGER_H +#define VDO_ACTION_MANAGER_H + +#include "admin-state.h" +#include "types.h" + +/* + * An action_manager provides a generic mechanism for applying actions to multi-zone entities (such + * as the block map or slab depot). Each action manager is tied to a specific context for which it + * manages actions. The manager ensures that only one action is active on that context at a time, + * and supports at most one pending action. Calls to schedule an action when there is already a + * pending action will result in VDO_COMPONENT_BUSY errors. Actions may only be submitted to the + * action manager from a single thread (which thread is determined when the action manager is + * constructed). + * + * A scheduled action consists of four components: + * + * preamble + * an optional method to be run on the initiator thread before applying the action to all zones + * zone_action + * an optional method to be applied to each of the zones + * conclusion + * an optional method to be run on the initiator thread once the per-zone method has been + * applied to all zones + * parent + * an optional completion to be finished once the conclusion is done + * + * At least one of the three methods must be provided. + */ + +/* + * A function which is to be applied asynchronously to a set of zones. + * @context: The object which holds the per-zone context for the action + * @zone_number: The number of zone to which the action is being applied + * @parent: The object to notify when the action is complete + */ +typedef void (*vdo_zone_action_fn)(void *context, zone_count_t zone_number, + struct vdo_completion *parent); + +/* + * A function which is to be applied asynchronously on an action manager's initiator thread as the + * preamble of an action. + * @context: The object which holds the per-zone context for the action + * @parent: The object to notify when the action is complete + */ +typedef void (*vdo_action_preamble_fn)(void *context, struct vdo_completion *parent); + +/* + * A function which will run on the action manager's initiator thread as the conclusion of an + * action. + * @context: The object which holds the per-zone context for the action + * + * Return: VDO_SUCCESS or an error + */ +typedef int (*vdo_action_conclusion_fn)(void *context); + +/* + * A function to schedule an action. + * @context: The object which holds the per-zone context for the action + * + * Return: true if an action was scheduled + */ +typedef bool (*vdo_action_scheduler_fn)(void *context); + +/* + * A function to get the id of the thread associated with a given zone. + * @context: The action context + * @zone_number: The number of the zone for which the thread ID is desired + */ +typedef thread_id_t (*vdo_zone_thread_getter_fn)(void *context, zone_count_t zone_number); + +struct action_manager; + +int __must_check vdo_make_action_manager(zone_count_t zones, + vdo_zone_thread_getter_fn get_zone_thread_id, + thread_id_t initiator_thread_id, void *context, + vdo_action_scheduler_fn scheduler, + struct vdo *vdo, + struct action_manager **manager_ptr); + +const struct admin_state_code *__must_check +vdo_get_current_manager_operation(struct action_manager *manager); + +void * __must_check vdo_get_current_action_context(struct action_manager *manager); + +bool vdo_schedule_default_action(struct action_manager *manager); + +bool vdo_schedule_action(struct action_manager *manager, vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent); + +bool vdo_schedule_operation(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent); + +bool vdo_schedule_operation_with_context(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + void *context, struct vdo_completion *parent); + +#endif /* VDO_ACTION_MANAGER_H */ diff --git a/drivers/md/dm-vdo/admin-state.c b/drivers/md/dm-vdo/admin-state.c new file mode 100644 index 000000000000..94533a802edb --- /dev/null +++ b/drivers/md/dm-vdo/admin-state.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "admin-state.h" + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "completion.h" +#include "types.h" + +static const struct admin_state_code VDO_CODE_NORMAL_OPERATION = { + .name = "VDO_ADMIN_STATE_NORMAL_OPERATION", + .normal = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_NORMAL_OPERATION = &VDO_CODE_NORMAL_OPERATION; +static const struct admin_state_code VDO_CODE_OPERATING = { + .name = "VDO_ADMIN_STATE_OPERATING", + .normal = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_OPERATING = &VDO_CODE_OPERATING; +static const struct admin_state_code VDO_CODE_FORMATTING = { + .name = "VDO_ADMIN_STATE_FORMATTING", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_FORMATTING = &VDO_CODE_FORMATTING; +static const struct admin_state_code VDO_CODE_PRE_LOADING = { + .name = "VDO_ADMIN_STATE_PRE_LOADING", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADING = &VDO_CODE_PRE_LOADING; +static const struct admin_state_code VDO_CODE_PRE_LOADED = { + .name = "VDO_ADMIN_STATE_PRE_LOADED", +}; +const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADED = &VDO_CODE_PRE_LOADED; +static const struct admin_state_code VDO_CODE_LOADING = { + .name = "VDO_ADMIN_STATE_LOADING", + .normal = true, + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_LOADING = &VDO_CODE_LOADING; +static const struct admin_state_code VDO_CODE_LOADING_FOR_RECOVERY = { + .name = "VDO_ADMIN_STATE_LOADING_FOR_RECOVERY", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_RECOVERY = + &VDO_CODE_LOADING_FOR_RECOVERY; +static const struct admin_state_code VDO_CODE_LOADING_FOR_REBUILD = { + .name = "VDO_ADMIN_STATE_LOADING_FOR_REBUILD", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_REBUILD = &VDO_CODE_LOADING_FOR_REBUILD; +static const struct admin_state_code VDO_CODE_WAITING_FOR_RECOVERY = { + .name = "VDO_ADMIN_STATE_WAITING_FOR_RECOVERY", + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_WAITING_FOR_RECOVERY = + &VDO_CODE_WAITING_FOR_RECOVERY; +static const struct admin_state_code VDO_CODE_NEW = { + .name = "VDO_ADMIN_STATE_NEW", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_NEW = &VDO_CODE_NEW; +static const struct admin_state_code VDO_CODE_INITIALIZED = { + .name = "VDO_ADMIN_STATE_INITIALIZED", +}; +const struct admin_state_code *VDO_ADMIN_STATE_INITIALIZED = &VDO_CODE_INITIALIZED; +static const struct admin_state_code VDO_CODE_RECOVERING = { + .name = "VDO_ADMIN_STATE_RECOVERING", + .draining = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_RECOVERING = &VDO_CODE_RECOVERING; +static const struct admin_state_code VDO_CODE_REBUILDING = { + .name = "VDO_ADMIN_STATE_REBUILDING", + .draining = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_REBUILDING = &VDO_CODE_REBUILDING; +static const struct admin_state_code VDO_CODE_SAVING = { + .name = "VDO_ADMIN_STATE_SAVING", + .draining = true, + .quiescing = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SAVING = &VDO_CODE_SAVING; +static const struct admin_state_code VDO_CODE_SAVED = { + .name = "VDO_ADMIN_STATE_SAVED", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SAVED = &VDO_CODE_SAVED; +static const struct admin_state_code VDO_CODE_SCRUBBING = { + .name = "VDO_ADMIN_STATE_SCRUBBING", + .draining = true, + .loading = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SCRUBBING = &VDO_CODE_SCRUBBING; +static const struct admin_state_code VDO_CODE_SAVE_FOR_SCRUBBING = { + .name = "VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING", + .draining = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING = &VDO_CODE_SAVE_FOR_SCRUBBING; +static const struct admin_state_code VDO_CODE_STOPPING = { + .name = "VDO_ADMIN_STATE_STOPPING", + .draining = true, + .quiescing = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_STOPPING = &VDO_CODE_STOPPING; +static const struct admin_state_code VDO_CODE_STOPPED = { + .name = "VDO_ADMIN_STATE_STOPPED", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_STOPPED = &VDO_CODE_STOPPED; +static const struct admin_state_code VDO_CODE_SUSPENDING = { + .name = "VDO_ADMIN_STATE_SUSPENDING", + .draining = true, + .quiescing = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDING = &VDO_CODE_SUSPENDING; +static const struct admin_state_code VDO_CODE_SUSPENDED = { + .name = "VDO_ADMIN_STATE_SUSPENDED", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED = &VDO_CODE_SUSPENDED; +static const struct admin_state_code VDO_CODE_SUSPENDED_OPERATION = { + .name = "VDO_ADMIN_STATE_SUSPENDED_OPERATION", + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED_OPERATION = &VDO_CODE_SUSPENDED_OPERATION; +static const struct admin_state_code VDO_CODE_RESUMING = { + .name = "VDO_ADMIN_STATE_RESUMING", + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_RESUMING = &VDO_CODE_RESUMING; + +/** + * get_next_state() - Determine the state which should be set after a given operation completes + * based on the operation and the current state. + * @operation The operation to be started. + * + * Return: The state to set when the operation completes or NULL if the operation can not be + * started in the current state. + */ +static const struct admin_state_code *get_next_state(const struct admin_state *state, + const struct admin_state_code *operation) +{ + const struct admin_state_code *code = vdo_get_admin_state_code(state); + + if (code->operating) + return NULL; + + if (operation == VDO_ADMIN_STATE_SAVING) + return (code == VDO_ADMIN_STATE_NORMAL_OPERATION ? VDO_ADMIN_STATE_SAVED : NULL); + + if (operation == VDO_ADMIN_STATE_SUSPENDING) { + return (code == VDO_ADMIN_STATE_NORMAL_OPERATION + ? VDO_ADMIN_STATE_SUSPENDED + : NULL); + } + + if (operation == VDO_ADMIN_STATE_STOPPING) + return (code == VDO_ADMIN_STATE_NORMAL_OPERATION ? VDO_ADMIN_STATE_STOPPED : NULL); + + if (operation == VDO_ADMIN_STATE_PRE_LOADING) + return (code == VDO_ADMIN_STATE_INITIALIZED ? VDO_ADMIN_STATE_PRE_LOADED : NULL); + + if (operation == VDO_ADMIN_STATE_SUSPENDED_OPERATION) { + return (((code == VDO_ADMIN_STATE_SUSPENDED) || + (code == VDO_ADMIN_STATE_SAVED)) ? code : NULL); + } + + return VDO_ADMIN_STATE_NORMAL_OPERATION; +} + +/** + * vdo_finish_operation() - Finish the current operation. + * + * Will notify the operation waiter if there is one. This method should be used for operations + * started with vdo_start_operation(). For operations which were started with vdo_start_draining(), + * use vdo_finish_draining() instead. + * + * Return: true if there was an operation to finish. + */ +bool vdo_finish_operation(struct admin_state *state, int result) +{ + if (!vdo_get_admin_state_code(state)->operating) + return false; + + state->complete = state->starting; + if (state->waiter != NULL) + vdo_set_completion_result(state->waiter, result); + + if (!state->starting) { + vdo_set_admin_state_code(state, state->next_state); + if (state->waiter != NULL) + vdo_launch_completion(uds_forget(state->waiter)); + } + + return true; +} + +/** + * begin_operation() - Begin an operation if it may be started given the current state. + * @waiter A completion to notify when the operation is complete; may be NULL. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check begin_operation(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator) +{ + int result; + const struct admin_state_code *next_state = get_next_state(state, operation); + + if (next_state == NULL) { + result = uds_log_error_strerror(VDO_INVALID_ADMIN_STATE, + "Can't start %s from %s", + operation->name, + vdo_get_admin_state_code(state)->name); + } else if (state->waiter != NULL) { + result = uds_log_error_strerror(VDO_COMPONENT_BUSY, + "Can't start %s with extant waiter", + operation->name); + } else { + state->waiter = waiter; + state->next_state = next_state; + vdo_set_admin_state_code(state, operation); + if (initiator != NULL) { + state->starting = true; + initiator(state); + state->starting = false; + if (state->complete) + vdo_finish_operation(state, VDO_SUCCESS); + } + + return VDO_SUCCESS; + } + + if (waiter != NULL) + vdo_continue_completion(waiter, result); + + return result; +} + +/** + * start_operation() - Start an operation if it may be started given the current state. + * @waiter A completion to notify when the operation is complete. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the operation was started. + */ +static inline bool __must_check start_operation(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator) +{ + return (begin_operation(state, operation, waiter, initiator) == VDO_SUCCESS); +} + +/** + * check_code() - Check the result of a state validation. + * @valid true if the code is of an appropriate type. + * @code The code which failed to be of the correct type. + * @what What the code failed to be, for logging. + * @waiter The completion to notify of the error; may be NULL. + * + * If the result failed, log an invalid state error and, if there is a waiter, notify it. + * + * Return: The result of the check. + */ +static bool check_code(bool valid, const struct admin_state_code *code, const char *what, + struct vdo_completion *waiter) +{ + int result; + + if (valid) + return true; + + result = uds_log_error_strerror(VDO_INVALID_ADMIN_STATE, + "%s is not a %s", code->name, what); + if (waiter != NULL) + vdo_continue_completion(waiter, result); + + return false; +} + +/** + * vdo_drain_operation() - Check that an operation is a drain. + * @waiter The completion to finish with an error if the operation is not a drain. + * + * Return: true if the specified operation is a drain. + */ +static bool __must_check assert_vdo_drain_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter) +{ + return check_code(operation->draining, operation, "drain operation", waiter); +} + +/** + * vdo_start_draining() - Initiate a drain operation if the current state permits it. + * @operation The type of drain to initiate. + * @waiter The completion to notify when the drain is complete. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the drain was initiated, if not the waiter will be notified. + */ +bool vdo_start_draining(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator) +{ + const struct admin_state_code *code = vdo_get_admin_state_code(state); + + if (!assert_vdo_drain_operation(operation, waiter)) + return false; + + if (code->quiescent) { + vdo_launch_completion(waiter); + return false; + } + + if (!code->normal) { + uds_log_error_strerror(VDO_INVALID_ADMIN_STATE, "can't start %s from %s", + operation->name, code->name); + vdo_continue_completion(waiter, VDO_INVALID_ADMIN_STATE); + return false; + } + + return start_operation(state, operation, waiter, initiator); +} + +/** + * vdo_finish_draining() - Finish a drain operation if one was in progress. + * + * Return: true if the state was draining; will notify the waiter if so. + */ +bool vdo_finish_draining(struct admin_state *state) +{ + return vdo_finish_draining_with_result(state, VDO_SUCCESS); +} + +/** + * vdo_finish_draining_with_result() - Finish a drain operation with a status code. + * + * Return: true if the state was draining; will notify the waiter if so. + */ +bool vdo_finish_draining_with_result(struct admin_state *state, int result) +{ + return (vdo_is_state_draining(state) && vdo_finish_operation(state, result)); +} + +/** + * vdo_assert_load_operation() - Check that an operation is a load. + * @waiter The completion to finish with an error if the operation is not a load. + * + * Return: true if the specified operation is a load. + */ +bool vdo_assert_load_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter) +{ + return check_code(operation->loading, operation, "load operation", waiter); +} + +/** + * vdo_start_loading() - Initiate a load operation if the current state permits it. + * @operation The type of load to initiate. + * @waiter The completion to notify when the load is complete (may be NULL). + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the load was initiated, if not the waiter will be notified. + */ +bool vdo_start_loading(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator) +{ + return (vdo_assert_load_operation(operation, waiter) && + start_operation(state, operation, waiter, initiator)); +} + +/** + * vdo_finish_loading() - Finish a load operation if one was in progress. + * + * Return: true if the state was loading; will notify the waiter if so. + */ +bool vdo_finish_loading(struct admin_state *state) +{ + return vdo_finish_loading_with_result(state, VDO_SUCCESS); +} + +/** + * vdo_finish_loading_with_result() - Finish a load operation with a status code. + * @result The result of the load operation. + * + * Return: true if the state was loading; will notify the waiter if so. + */ +bool vdo_finish_loading_with_result(struct admin_state *state, int result) +{ + return (vdo_is_state_loading(state) && vdo_finish_operation(state, result)); +} + +/** + * assert_vdo_resume_operation() - Check whether an admin_state_code is a resume operation. + * @waiter The completion to notify if the operation is not a resume operation; may be NULL. + * + * Return: true if the code is a resume operation. + */ +static bool __must_check assert_vdo_resume_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter) +{ + return check_code(operation == VDO_ADMIN_STATE_RESUMING, operation, + "resume operation", waiter); +} + +/** + * vdo_start_resuming() - Initiate a resume operation if the current state permits it. + * @operation The type of resume to start. + * @waiter The completion to notify when the resume is complete (may be NULL). + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the resume was initiated, if not the waiter will be notified. + */ +bool vdo_start_resuming(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator) +{ + return (assert_vdo_resume_operation(operation, waiter) && + start_operation(state, operation, waiter, initiator)); +} + +/** + * vdo_finish_resuming() - Finish a resume operation if one was in progress. + * + * Return: true if the state was resuming; will notify the waiter if so. + */ +bool vdo_finish_resuming(struct admin_state *state) +{ + return vdo_finish_resuming_with_result(state, VDO_SUCCESS); +} + +/** + * vdo_finish_resuming_with_result() - Finish a resume operation with a status code. + * @result The result of the resume operation. + * + * Return: true if the state was resuming; will notify the waiter if so. + */ +bool vdo_finish_resuming_with_result(struct admin_state *state, int result) +{ + return (vdo_is_state_resuming(state) && vdo_finish_operation(state, result)); +} + +/** + * vdo_resume_if_quiescent() - Change the state to normal operation if the current state is + * quiescent. + * + * Return: VDO_SUCCESS if the state resumed, VDO_INVALID_ADMIN_STATE otherwise. + */ +int vdo_resume_if_quiescent(struct admin_state *state) +{ + if (!vdo_is_state_quiescent(state)) + return VDO_INVALID_ADMIN_STATE; + + vdo_set_admin_state_code(state, VDO_ADMIN_STATE_NORMAL_OPERATION); + return VDO_SUCCESS; +} + +/** + * vdo_start_operation() - Attempt to start an operation. + * + * Return: VDO_SUCCESS if the operation was started, VDO_INVALID_ADMIN_STATE if not + */ +int vdo_start_operation(struct admin_state *state, + const struct admin_state_code *operation) +{ + return vdo_start_operation_with_waiter(state, operation, NULL, NULL); +} + +/** + * vdo_start_operation_with_waiter() - Attempt to start an operation. + * @waiter the completion to notify when the operation completes or fails to start; may be NULL. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: VDO_SUCCESS if the operation was started, VDO_INVALID_ADMIN_STATE if not + */ +int vdo_start_operation_with_waiter(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator) +{ + return (check_code(operation->operating, operation, "operation", waiter) ? + begin_operation(state, operation, waiter, initiator) : + VDO_INVALID_ADMIN_STATE); +} diff --git a/drivers/md/dm-vdo/admin-state.h b/drivers/md/dm-vdo/admin-state.h new file mode 100644 index 000000000000..a7d6ac2c30a6 --- /dev/null +++ b/drivers/md/dm-vdo/admin-state.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_ADMIN_STATE_H +#define VDO_ADMIN_STATE_H + +#include "completion.h" +#include "types.h" + +struct admin_state_code { + const char *name; + /* Normal operation, data_vios may be active */ + bool normal; + /* I/O is draining, new requests should not start */ + bool draining; + /* This is a startup time operation */ + bool loading; + /* The next state will be quiescent */ + bool quiescing; + /* The VDO is quiescent, there should be no I/O */ + bool quiescent; + /* Whether an operation is in progress and so no other operation may be started */ + bool operating; +}; + +extern const struct admin_state_code *VDO_ADMIN_STATE_NORMAL_OPERATION; +extern const struct admin_state_code *VDO_ADMIN_STATE_OPERATING; +extern const struct admin_state_code *VDO_ADMIN_STATE_FORMATTING; +extern const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADING; +extern const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADED; +extern const struct admin_state_code *VDO_ADMIN_STATE_LOADING; +extern const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_RECOVERY; +extern const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_REBUILD; +extern const struct admin_state_code *VDO_ADMIN_STATE_WAITING_FOR_RECOVERY; +extern const struct admin_state_code *VDO_ADMIN_STATE_NEW; +extern const struct admin_state_code *VDO_ADMIN_STATE_INITIALIZED; +extern const struct admin_state_code *VDO_ADMIN_STATE_RECOVERING; +extern const struct admin_state_code *VDO_ADMIN_STATE_REBUILDING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SAVING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SAVED; +extern const struct admin_state_code *VDO_ADMIN_STATE_SCRUBBING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING; +extern const struct admin_state_code *VDO_ADMIN_STATE_STOPPING; +extern const struct admin_state_code *VDO_ADMIN_STATE_STOPPED; +extern const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED; +extern const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED_OPERATION; +extern const struct admin_state_code *VDO_ADMIN_STATE_RESUMING; + +struct admin_state { + const struct admin_state_code *current_state; + /* The next administrative state (when the current operation finishes) */ + const struct admin_state_code *next_state; + /* A completion waiting on a state change */ + struct vdo_completion *waiter; + /* Whether an operation is being initiated */ + bool starting; + /* Whether an operation has completed in the initiator */ + bool complete; +}; + +/** + * typedef vdo_admin_initiator_fn - A method to be called once an admin operation may be initiated. + */ +typedef void (*vdo_admin_initiator_fn)(struct admin_state *state); + +static inline const struct admin_state_code * __must_check +vdo_get_admin_state_code(const struct admin_state *state) +{ + return READ_ONCE(state->current_state); +} + +/** + * vdo_set_admin_state_code() - Set the current admin state code. + * + * This function should be used primarily for initialization and by adminState internals. Most uses + * should go through the operation interfaces. + */ +static inline void vdo_set_admin_state_code(struct admin_state *state, + const struct admin_state_code *code) +{ + WRITE_ONCE(state->current_state, code); +} + +static inline bool __must_check vdo_is_state_normal(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->normal; +} + +static inline bool __must_check vdo_is_state_suspending(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_SUSPENDING); +} + +static inline bool __must_check vdo_is_state_saving(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_SAVING); +} + +static inline bool __must_check vdo_is_state_saved(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_SAVED); +} + +static inline bool __must_check vdo_is_state_draining(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->draining; +} + +static inline bool __must_check vdo_is_state_loading(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->loading; +} + +static inline bool __must_check vdo_is_state_resuming(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_RESUMING); +} + +static inline bool __must_check vdo_is_state_clean_load(const struct admin_state *state) +{ + const struct admin_state_code *code = vdo_get_admin_state_code(state); + + return ((code == VDO_ADMIN_STATE_FORMATTING) || (code == VDO_ADMIN_STATE_LOADING)); +} + +static inline bool __must_check vdo_is_state_quiescing(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->quiescing; +} + +static inline bool __must_check vdo_is_state_quiescent(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->quiescent; +} + +bool __must_check vdo_assert_load_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter); + +bool vdo_start_loading(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator); + +bool vdo_finish_loading(struct admin_state *state); + +bool vdo_finish_loading_with_result(struct admin_state *state, int result); + +bool vdo_start_resuming(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator); + +bool vdo_finish_resuming(struct admin_state *state); + +bool vdo_finish_resuming_with_result(struct admin_state *state, int result); + +int vdo_resume_if_quiescent(struct admin_state *state); + +bool vdo_start_draining(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator); + +bool vdo_finish_draining(struct admin_state *state); + +bool vdo_finish_draining_with_result(struct admin_state *state, int result); + +int vdo_start_operation(struct admin_state *state, + const struct admin_state_code *operation); + +int vdo_start_operation_with_waiter(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator); + +bool vdo_finish_operation(struct admin_state *state, int result); + +#endif /* VDO_ADMIN_STATE_H */ From 9533d804ae02f3908acccaa45031fd868d720ba8 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:58:59 -0500 Subject: [PATCH 0566/1038] dm vdo: add vio, the request object for vdo metadata Add the data and methods that implement the vio object that is basic unit of I/O in vdo. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/vio.c | 501 ++++++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/vio.h | 199 ++++++++++++++++ 2 files changed, 700 insertions(+) create mode 100644 drivers/md/dm-vdo/vio.c create mode 100644 drivers/md/dm-vdo/vio.h diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c new file mode 100644 index 000000000000..f83b56acc8e4 --- /dev/null +++ b/drivers/md/dm-vdo/vio.c @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "vio.h" + +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "constants.h" +#include "io-submitter.h" +#include "vdo.h" + +/* A vio_pool is a collection of preallocated vios. */ +struct vio_pool { + /** The number of objects managed by the pool */ + size_t size; + /** The list of objects which are available */ + struct list_head available; + /** The queue of requestors waiting for objects from the pool */ + struct wait_queue waiting; + /** The number of objects currently in use */ + size_t busy_count; + /** The list of objects which are in use */ + struct list_head busy; + /** The ID of the thread on which this pool may be used */ + thread_id_t thread_id; + /** The buffer backing the pool's vios */ + char *buffer; + /** The pool entries */ + struct pooled_vio vios[]; +}; + +physical_block_number_t pbn_from_vio_bio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + physical_block_number_t pbn = bio->bi_iter.bi_sector / VDO_SECTORS_PER_BLOCK; + + return ((pbn == VDO_GEOMETRY_BLOCK_LOCATION) ? pbn : pbn + vdo->geometry.bio_offset); +} + +static int create_multi_block_bio(block_count_t size, struct bio **bio_ptr) +{ + struct bio *bio = NULL; + int result; + + result = uds_allocate_extended(struct bio, size + 1, struct bio_vec, + "bio", &bio); + if (result != VDO_SUCCESS) + return result; + + *bio_ptr = bio; + return VDO_SUCCESS; +} + +int vdo_create_bio(struct bio **bio_ptr) +{ + return create_multi_block_bio(1, bio_ptr); +} + +void vdo_free_bio(struct bio *bio) +{ + if (bio == NULL) + return; + + bio_uninit(bio); + uds_free(uds_forget(bio)); +} + +int allocate_vio_components(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, void *parent, + unsigned int block_count, char *data, struct vio *vio) +{ + struct bio *bio; + int result; + + result = ASSERT(block_count <= MAX_BLOCKS_PER_VIO, + "block count %u does not exceed maximum %u", block_count, + MAX_BLOCKS_PER_VIO); + if (result != VDO_SUCCESS) + return result; + + result = ASSERT(((vio_type != VIO_TYPE_UNINITIALIZED) && (vio_type != VIO_TYPE_DATA)), + "%d is a metadata type", vio_type); + if (result != VDO_SUCCESS) + return result; + + result = create_multi_block_bio(block_count, &bio); + if (result != VDO_SUCCESS) + return result; + + initialize_vio(vio, bio, block_count, vio_type, priority, vdo); + vio->completion.parent = parent; + vio->data = data; + return VDO_SUCCESS; +} + +/** + * create_multi_block_metadata_vio() - Create a vio. + * @vdo: The vdo on which the vio will operate. + * @vio_type: The type of vio to create. + * @priority: The relative priority to assign to the vio. + * @parent: The parent of the vio. + * @block_count: The size of the vio in blocks. + * @data: The buffer. + * @vio_ptr: A pointer to hold the new vio. + * + * Return: VDO_SUCCESS or an error. + */ +int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, void *parent, + unsigned int block_count, char *data, + struct vio **vio_ptr) +{ + struct vio *vio; + int result; + + /* If struct vio grows past 256 bytes, we'll lose benefits of VDOSTORY-176. */ + BUILD_BUG_ON(sizeof(struct vio) > 256); + + /* + * Metadata vios should use direct allocation and not use the buffer pool, which is + * reserved for submissions from the linux block layer. + */ + result = uds_allocate(1, struct vio, __func__, &vio); + if (result != VDO_SUCCESS) { + uds_log_error("metadata vio allocation failure %d", result); + return result; + } + + result = allocate_vio_components(vdo, vio_type, priority, parent, block_count, + data, vio); + if (result != VDO_SUCCESS) { + uds_free(vio); + return result; + } + + *vio_ptr = vio; + return VDO_SUCCESS; +} + +/** + * free_vio_components() - Free the components of a vio embedded in a larger structure. + * @vio: The vio to destroy + */ +void free_vio_components(struct vio *vio) +{ + if (vio == NULL) + return; + + BUG_ON(is_data_vio(vio)); + vdo_free_bio(uds_forget(vio->bio)); +} + +/** + * free_vio() - Destroy a vio. + * @vio: The vio to destroy. + */ +void free_vio(struct vio *vio) +{ + free_vio_components(vio); + uds_free(vio); +} + +/* Set bio properties for a VDO read or write. */ +void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callback, + unsigned int bi_opf, physical_block_number_t pbn) +{ + struct vdo *vdo = vio->completion.vdo; + struct device_config *config = vdo->device_config; + + pbn -= vdo->geometry.bio_offset; + vio->bio_zone = ((pbn / config->thread_counts.bio_rotation_interval) % + config->thread_counts.bio_threads); + + bio->bi_private = vio; + bio->bi_end_io = callback; + bio->bi_opf = bi_opf; + bio->bi_iter.bi_sector = pbn * VDO_SECTORS_PER_BLOCK; +} + +/* + * Prepares the bio to perform IO with the specified buffer. May only be used on a VDO-allocated + * bio, as it assumes the bio wraps a 4k buffer that is 4k aligned, but there does not have to be a + * vio associated with the bio. + */ +int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, + unsigned int bi_opf, physical_block_number_t pbn) +{ + int bvec_count, offset, len, i; + struct bio *bio = vio->bio; + + bio_reset(bio, bio->bi_bdev, bi_opf); + vdo_set_bio_properties(bio, vio, callback, bi_opf, pbn); + if (data == NULL) + return VDO_SUCCESS; + + bio->bi_io_vec = bio->bi_inline_vecs; + bio->bi_max_vecs = vio->block_count + 1; + len = VDO_BLOCK_SIZE * vio->block_count; + offset = offset_in_page(data); + bvec_count = DIV_ROUND_UP(offset + len, PAGE_SIZE); + + /* + * If we knew that data was always on one page, or contiguous pages, we wouldn't need the + * loop. But if we're using vmalloc, it's not impossible that the data is in different + * pages that can't be merged in bio_add_page... + */ + for (i = 0; (i < bvec_count) && (len > 0); i++) { + struct page *page; + int bytes_added; + int bytes = PAGE_SIZE - offset; + + if (bytes > len) + bytes = len; + + page = is_vmalloc_addr(data) ? vmalloc_to_page(data) : virt_to_page(data); + bytes_added = bio_add_page(bio, page, bytes, offset); + + if (bytes_added != bytes) { + return uds_log_error_strerror(VDO_BIO_CREATION_FAILED, + "Could only add %i bytes to bio", + bytes_added); + } + + data += bytes; + len -= bytes; + offset = 0; + } + + return VDO_SUCCESS; +} + +/** + * update_vio_error_stats() - Update per-vio error stats and log the error. + * @vio: The vio which got an error. + * @format: The format of the message to log (a printf style format). + */ +void update_vio_error_stats(struct vio *vio, const char *format, ...) +{ + static DEFINE_RATELIMIT_STATE(error_limiter, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + va_list args; + int priority; + struct vdo *vdo = vio->completion.vdo; + + switch (vio->completion.result) { + case VDO_READ_ONLY: + atomic64_inc(&vdo->stats.read_only_error_count); + return; + + case VDO_NO_SPACE: + atomic64_inc(&vdo->stats.no_space_error_count); + priority = UDS_LOG_DEBUG; + break; + + default: + priority = UDS_LOG_ERR; + } + + if (!__ratelimit(&error_limiter)) + return; + + va_start(args, format); + uds_vlog_strerror(priority, vio->completion.result, UDS_LOGGING_MODULE_NAME, + format, args); + va_end(args); +} + +void vio_record_metadata_io_error(struct vio *vio) +{ + const char *description; + physical_block_number_t pbn = pbn_from_vio_bio(vio->bio); + + if (bio_op(vio->bio) == REQ_OP_READ) { + description = "read"; + } else if ((vio->bio->bi_opf & REQ_PREFLUSH) == REQ_PREFLUSH) { + description = (((vio->bio->bi_opf & REQ_FUA) == REQ_FUA) ? + "write+preflush+fua" : + "write+preflush"); + } else if ((vio->bio->bi_opf & REQ_FUA) == REQ_FUA) { + description = "write+fua"; + } else { + description = "write"; + } + + update_vio_error_stats(vio, + "Completing %s vio of type %u for physical block %llu with error", + description, vio->type, (unsigned long long) pbn); +} + +/** + * make_vio_pool() - Create a new vio pool. + * @vdo: The vdo. + * @pool_size: The number of vios in the pool. + * @thread_id: The ID of the thread using this pool. + * @vio_type: The type of vios in the pool. + * @priority: The priority with which vios from the pool should be enqueued. + * @context: The context that each entry will have. + * @pool_ptr: The resulting pool. + * + * Return: A success or error code. + */ +int make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t thread_id, + enum vio_type vio_type, enum vio_priority priority, void *context, + struct vio_pool **pool_ptr) +{ + struct vio_pool *pool; + char *ptr; + int result; + + result = uds_allocate_extended(struct vio_pool, pool_size, struct pooled_vio, + __func__, &pool); + if (result != VDO_SUCCESS) + return result; + + pool->thread_id = thread_id; + INIT_LIST_HEAD(&pool->available); + INIT_LIST_HEAD(&pool->busy); + + result = uds_allocate(pool_size * VDO_BLOCK_SIZE, char, + "VIO pool buffer", &pool->buffer); + if (result != VDO_SUCCESS) { + free_vio_pool(pool); + return result; + } + + ptr = pool->buffer; + for (pool->size = 0; pool->size < pool_size; pool->size++, ptr += VDO_BLOCK_SIZE) { + struct pooled_vio *pooled = &pool->vios[pool->size]; + + result = allocate_vio_components(vdo, vio_type, priority, NULL, 1, ptr, + &pooled->vio); + if (result != VDO_SUCCESS) { + free_vio_pool(pool); + return result; + } + + pooled->context = context; + list_add_tail(&pooled->pool_entry, &pool->available); + } + + *pool_ptr = pool; + return VDO_SUCCESS; +} + +/** + * free_vio_pool() - Destroy a vio pool. + * @pool: The pool to free. + */ +void free_vio_pool(struct vio_pool *pool) +{ + struct pooled_vio *pooled, *tmp; + + if (pool == NULL) + return; + + /* Remove all available vios from the object pool. */ + ASSERT_LOG_ONLY(!vdo_has_waiters(&pool->waiting), + "VIO pool must not have any waiters when being freed"); + ASSERT_LOG_ONLY((pool->busy_count == 0), + "VIO pool must not have %zu busy entries when being freed", + pool->busy_count); + ASSERT_LOG_ONLY(list_empty(&pool->busy), + "VIO pool must not have busy entries when being freed"); + + list_for_each_entry_safe(pooled, tmp, &pool->available, pool_entry) { + list_del(&pooled->pool_entry); + free_vio_components(&pooled->vio); + pool->size--; + } + + ASSERT_LOG_ONLY(pool->size == 0, + "VIO pool must not have missing entries when being freed"); + + uds_free(uds_forget(pool->buffer)); + uds_free(pool); +} + +/** + * is_vio_pool_busy() - Check whether an vio pool has outstanding entries. + * + * Return: true if the pool is busy. + */ +bool is_vio_pool_busy(struct vio_pool *pool) +{ + return (pool->busy_count != 0); +} + +/** + * acquire_vio_from_pool() - Acquire a vio and buffer from the pool (asynchronous). + * @pool: The vio pool. + * @waiter: Object that is requesting a vio. + */ +void acquire_vio_from_pool(struct vio_pool *pool, struct waiter *waiter) +{ + struct pooled_vio *pooled; + + ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), + "acquire from active vio_pool called from correct thread"); + + if (list_empty(&pool->available)) { + vdo_enqueue_waiter(&pool->waiting, waiter); + return; + } + + pooled = list_first_entry(&pool->available, struct pooled_vio, pool_entry); + pool->busy_count++; + list_move_tail(&pooled->pool_entry, &pool->busy); + (*waiter->callback)(waiter, pooled); +} + +/** + * return_vio_to_pool() - Return a vio to the pool + * @pool: The vio pool. + * @vio: The pooled vio to return. + */ +void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio) +{ + ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), + "vio pool entry returned on same thread as it was acquired"); + + vio->vio.completion.error_handler = NULL; + vio->vio.completion.parent = NULL; + if (vdo_has_waiters(&pool->waiting)) { + vdo_notify_next_waiter(&pool->waiting, NULL, vio); + return; + } + + list_move_tail(&vio->pool_entry, &pool->available); + --pool->busy_count; +} + +/* + * Various counting functions for statistics. + * These are used for bios coming into VDO, as well as bios generated by VDO. + */ +void vdo_count_bios(struct atomic_bio_stats *bio_stats, struct bio *bio) +{ + if (((bio->bi_opf & REQ_PREFLUSH) != 0) && (bio->bi_iter.bi_size == 0)) { + atomic64_inc(&bio_stats->empty_flush); + atomic64_inc(&bio_stats->flush); + return; + } + + switch (bio_op(bio)) { + case REQ_OP_WRITE: + atomic64_inc(&bio_stats->write); + break; + case REQ_OP_READ: + atomic64_inc(&bio_stats->read); + break; + case REQ_OP_DISCARD: + atomic64_inc(&bio_stats->discard); + break; + /* + * All other operations are filtered out in dmvdo.c, or not created by VDO, so + * shouldn't exist. + */ + default: + ASSERT_LOG_ONLY(0, "Bio operation %d not a write, read, discard, or empty flush", + bio_op(bio)); + } + + if ((bio->bi_opf & REQ_PREFLUSH) != 0) + atomic64_inc(&bio_stats->flush); + if (bio->bi_opf & REQ_FUA) + atomic64_inc(&bio_stats->fua); +} + +static void count_all_bios_completed(struct vio *vio, struct bio *bio) +{ + struct atomic_statistics *stats = &vio->completion.vdo->stats; + + if (is_data_vio(vio)) { + vdo_count_bios(&stats->bios_out_completed, bio); + return; + } + + vdo_count_bios(&stats->bios_meta_completed, bio); + if (vio->type == VIO_TYPE_RECOVERY_JOURNAL) + vdo_count_bios(&stats->bios_journal_completed, bio); + else if (vio->type == VIO_TYPE_BLOCK_MAP) + vdo_count_bios(&stats->bios_page_cache_completed, bio); +} + +void vdo_count_completed_bios(struct bio *bio) +{ + struct vio *vio = (struct vio *) bio->bi_private; + + atomic64_inc(&vio->completion.vdo->stats.bios_completed); + count_all_bios_completed(vio, bio); +} diff --git a/drivers/md/dm-vdo/vio.h b/drivers/md/dm-vdo/vio.h new file mode 100644 index 000000000000..3c72fded69b0 --- /dev/null +++ b/drivers/md/dm-vdo/vio.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VIO_H +#define VIO_H + +#include +#include +#include +#include +#include + +#include "completion.h" +#include "constants.h" +#include "types.h" +#include "vdo.h" + +enum { + MAX_BLOCKS_PER_VIO = (BIO_MAX_VECS << PAGE_SHIFT) / VDO_BLOCK_SIZE, +}; + +struct pooled_vio { + /* The underlying vio */ + struct vio vio; + /* The list entry for chaining pooled vios together */ + struct list_head list_entry; + /* The context set by the pool */ + void *context; + /* The list entry used by the pool */ + struct list_head pool_entry; +}; + +/** + * as_vio() - Convert a generic vdo_completion to a vio. + * @completion: The completion to convert. + * + * Return: The completion as a vio. + */ +static inline struct vio *as_vio(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VIO_COMPLETION); + return container_of(completion, struct vio, completion); +} + +/** + * get_vio_bio_zone_thread_id() - Get the thread id of the bio zone in which a vio should submit + * its I/O. + * @vio: The vio. + * + * Return: The id of the bio zone thread the vio should use. + */ +static inline thread_id_t __must_check get_vio_bio_zone_thread_id(struct vio *vio) +{ + return vio->completion.vdo->thread_config.bio_threads[vio->bio_zone]; +} + +physical_block_number_t __must_check pbn_from_vio_bio(struct bio *bio); + +/** + * assert_vio_in_bio_zone() - Check that a vio is running on the correct thread for its bio zone. + * @vio: The vio to check. + */ +static inline void assert_vio_in_bio_zone(struct vio *vio) +{ + thread_id_t expected = get_vio_bio_zone_thread_id(vio); + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((expected == thread_id), + "vio I/O for physical block %llu on thread %u, should be on bio zone thread %u", + (unsigned long long) pbn_from_vio_bio(vio->bio), thread_id, + expected); +} + +int vdo_create_bio(struct bio **bio_ptr); +void vdo_free_bio(struct bio *bio); +int allocate_vio_components(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, void *parent, + unsigned int block_count, char *data, struct vio *vio); +int __must_check create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, + void *parent, unsigned int block_count, + char *data, struct vio **vio_ptr); + +static inline int __must_check create_metadata_vio(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, + void *parent, char *data, + struct vio **vio_ptr) +{ + return create_multi_block_metadata_vio(vdo, vio_type, priority, parent, 1, data, + vio_ptr); +} + +void free_vio_components(struct vio *vio); +void free_vio(struct vio *vio); + +/** + * initialize_vio() - Initialize a vio. + * @vio: The vio to initialize. + * @bio: The bio this vio should use for its I/O. + * @block_count: The size of this vio in vdo blocks. + * @vio_type: The vio type. + * @priority: The relative priority of the vio. + * @vdo: The vdo for this vio. + */ +static inline void initialize_vio(struct vio *vio, struct bio *bio, + unsigned int block_count, enum vio_type vio_type, + enum vio_priority priority, struct vdo *vdo) +{ + /* data_vio's may not span multiple blocks */ + BUG_ON((vio_type == VIO_TYPE_DATA) && (block_count != 1)); + + vio->bio = bio; + vio->block_count = block_count; + vio->type = vio_type; + vio->priority = priority; + vdo_initialize_completion(&vio->completion, vdo, VIO_COMPLETION); +} + +void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callback, + unsigned int bi_opf, physical_block_number_t pbn); + +int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, + unsigned int bi_opf, physical_block_number_t pbn); + +void update_vio_error_stats(struct vio *vio, const char *format, ...) + __printf(2, 3); + +/** + * is_data_vio() - Check whether a vio is servicing an external data request. + * @vio: The vio to check. + */ +static inline bool is_data_vio(struct vio *vio) +{ + return (vio->type == VIO_TYPE_DATA); +} + +/** + * get_metadata_priority() - Convert a vio's priority to a work item priority. + * @vio: The vio. + * + * Return: The priority with which to submit the vio's bio. + */ +static inline enum vdo_completion_priority get_metadata_priority(struct vio *vio) +{ + return ((vio->priority == VIO_PRIORITY_HIGH) ? + BIO_Q_HIGH_PRIORITY : + BIO_Q_METADATA_PRIORITY); +} + +/** + * continue_vio() - Enqueue a vio to run its next callback. + * @vio: The vio to continue. + * + * Return: The result of the current operation. + */ +static inline void continue_vio(struct vio *vio, int result) +{ + if (unlikely(result != VDO_SUCCESS)) + vdo_set_completion_result(&vio->completion, result); + + vdo_enqueue_completion(&vio->completion, VDO_WORK_Q_DEFAULT_PRIORITY); +} + +void vdo_count_bios(struct atomic_bio_stats *bio_stats, struct bio *bio); +void vdo_count_completed_bios(struct bio *bio); + +/** + * continue_vio_after_io() - Continue a vio now that its I/O has returned. + */ +static inline void continue_vio_after_io(struct vio *vio, vdo_action_fn callback, + thread_id_t thread) +{ + vdo_count_completed_bios(vio->bio); + vdo_set_completion_callback(&vio->completion, callback, thread); + continue_vio(vio, blk_status_to_errno(vio->bio->bi_status)); +} + +void vio_record_metadata_io_error(struct vio *vio); + +/* A vio_pool is a collection of preallocated vios used to write arbitrary metadata blocks. */ + +static inline struct pooled_vio *vio_as_pooled_vio(struct vio *vio) +{ + return container_of(vio, struct pooled_vio, vio); +} + +struct vio_pool; + +int __must_check make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t thread_id, + enum vio_type vio_type, enum vio_priority priority, + void *context, struct vio_pool **pool_ptr); +void free_vio_pool(struct vio_pool *pool); +bool __must_check is_vio_pool_busy(struct vio_pool *pool); +void acquire_vio_from_pool(struct vio_pool *pool, struct waiter *waiter); +void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio); + +#endif /* VIO_H */ From 79535a7881c0cbe95063a2670d840cc950ae9282 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 19:59:39 -0500 Subject: [PATCH 0567/1038] dm vdo: add data_vio, the request object which services incoming bios Add the data and methods that implement the data_vio object that handles user data bios as they are processed. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/data-vio.c | 2057 ++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/data-vio.h | 669 +++++++++++ 2 files changed, 2726 insertions(+) create mode 100644 drivers/md/dm-vdo/data-vio.c create mode 100644 drivers/md/dm-vdo/data-vio.h diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c new file mode 100644 index 000000000000..09a99a28da4f --- /dev/null +++ b/drivers/md/dm-vdo/data-vio.c @@ -0,0 +1,2057 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "data-vio.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "murmurhash3.h" +#include "permassert.h" + +#include "block-map.h" +#include "dump.h" +#include "encodings.h" +#include "int-map.h" +#include "io-submitter.h" +#include "logical-zone.h" +#include "packer.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +/** + * DOC: Bio flags. + * + * For certain flags set on user bios, if the user bio has not yet been acknowledged, setting those + * flags on our own bio(s) for that request may help underlying layers better fulfill the user + * bio's needs. This constant contains the aggregate of those flags; VDO strips all the other + * flags, as they convey incorrect information. + * + * These flags are always irrelevant if we have already finished the user bio as they are only + * hints on IO importance. If VDO has finished the user bio, any remaining IO done doesn't care how + * important finishing the finished bio was. + * + * Note that bio.c contains the complete list of flags we believe may be set; the following list + * explains the action taken with each of those flags VDO could receive: + * + * * REQ_SYNC: Passed down if the user bio is not yet completed, since it indicates the user bio + * completion is required for further work to be done by the issuer. + * * REQ_META: Passed down if the user bio is not yet completed, since it may mean the lower layer + * treats it as more urgent, similar to REQ_SYNC. + * * REQ_PRIO: Passed down if the user bio is not yet completed, since it indicates the user bio is + * important. + * * REQ_NOMERGE: Set only if the incoming bio was split; irrelevant to VDO IO. + * * REQ_IDLE: Set if the incoming bio had more IO quickly following; VDO's IO pattern doesn't + * match incoming IO, so this flag is incorrect for it. + * * REQ_FUA: Handled separately, and irrelevant to VDO IO otherwise. + * * REQ_RAHEAD: Passed down, as, for reads, it indicates trivial importance. + * * REQ_BACKGROUND: Not passed down, as VIOs are a limited resource and VDO needs them recycled + * ASAP to service heavy load, which is the only place where REQ_BACKGROUND might aid in load + * prioritization. + */ +static unsigned int PASSTHROUGH_FLAGS = (REQ_PRIO | REQ_META | REQ_SYNC | REQ_RAHEAD); + +/** + * DOC: + * + * The data_vio_pool maintains the pool of data_vios which a vdo uses to service incoming bios. For + * correctness, and in order to avoid potentially expensive or blocking memory allocations during + * normal operation, the number of concurrently active data_vios is capped. Furthermore, in order + * to avoid starvation of reads and writes, at most 75% of the data_vios may be used for + * discards. The data_vio_pool is responsible for enforcing these limits. Threads submitting bios + * for which a data_vio or discard permit are not available will block until the necessary + * resources are available. The pool is also responsible for distributing resources to blocked + * threads and waking them. Finally, the pool attempts to batch the work of recycling data_vios by + * performing the work of actually assigning resources to blocked threads or placing data_vios back + * into the pool on a single cpu at a time. + * + * The pool contains two "limiters", one for tracking data_vios and one for tracking discard + * permits. The limiters also provide safe cross-thread access to pool statistics without the need + * to take the pool's lock. When a thread submits a bio to a vdo device, it will first attempt to + * get a discard permit if it is a discard, and then to get a data_vio. If the necessary resources + * are available, the incoming bio will be assigned to the acquired data_vio, and it will be + * launched. However, if either of these are unavailable, the arrival time of the bio is recorded + * in the bio's bi_private field, the bio and its submitter are both queued on the appropriate + * limiter and the submitting thread will then put itself to sleep. (note that this mechanism will + * break if jiffies are only 32 bits.) + * + * Whenever a data_vio has completed processing for the bio it was servicing, release_data_vio() + * will be called on it. This function will add the data_vio to a funnel queue, and then check the + * state of the pool. If the pool is not currently processing released data_vios, the pool's + * completion will be enqueued on a cpu queue. This obviates the need for the releasing threads to + * hold the pool's lock, and also batches release work while avoiding starvation of the cpu + * threads. + * + * Whenever the pool's completion is run on a cpu thread, it calls process_release_callback() which + * processes a batch of returned data_vios (currently at most 32) from the pool's funnel queue. For + * each data_vio, it first checks whether that data_vio was processing a discard. If so, and there + * is a blocked bio waiting for a discard permit, that permit is notionally transferred to the + * eldest discard waiter, and that waiter is moved to the end of the list of discard bios waiting + * for a data_vio. If there are no discard waiters, the discard permit is returned to the pool. + * Next, the data_vio is assigned to the oldest blocked bio which either has a discard permit, or + * doesn't need one and relaunched. If neither of these exist, the data_vio is returned to the + * pool. Finally, if any waiting bios were launched, the threads which blocked trying to submit + * them are awakened. + */ + +enum { + DATA_VIO_RELEASE_BATCH_SIZE = 128, +}; + +static const unsigned int VDO_SECTORS_PER_BLOCK_MASK = VDO_SECTORS_PER_BLOCK - 1; +static const u32 COMPRESSION_STATUS_MASK = 0xff; +static const u32 MAY_NOT_COMPRESS_MASK = 0x80000000; + +struct limiter; +typedef void (*assigner_fn)(struct limiter *limiter); + +/* Bookkeeping structure for a single type of resource. */ +struct limiter { + /* The data_vio_pool to which this limiter belongs */ + struct data_vio_pool *pool; + /* The maximum number of data_vios available */ + data_vio_count_t limit; + /* The number of resources in use */ + data_vio_count_t busy; + /* The maximum number of resources ever simultaneously in use */ + data_vio_count_t max_busy; + /* The number of resources to release */ + data_vio_count_t release_count; + /* The number of waiters to wake */ + data_vio_count_t wake_count; + /* The list of waiting bios which are known to process_release_callback() */ + struct bio_list waiters; + /* The list of waiting bios which are not yet known to process_release_callback() */ + struct bio_list new_waiters; + /* The list of waiters which have their permits */ + struct bio_list *permitted_waiters; + /* The function for assigning a resource to a waiter */ + assigner_fn assigner; + /* The queue of blocked threads */ + wait_queue_head_t blocked_threads; + /* The arrival time of the eldest waiter */ + u64 arrival; +}; + +/* + * A data_vio_pool is a collection of preallocated data_vios which may be acquired from any thread, + * and are released in batches. + */ +struct data_vio_pool { + /* Completion for scheduling releases */ + struct vdo_completion completion; + /* The administrative state of the pool */ + struct admin_state state; + /* Lock protecting the pool */ + spinlock_t lock; + /* The main limiter controlling the total data_vios in the pool. */ + struct limiter limiter; + /* The limiter controlling data_vios for discard */ + struct limiter discard_limiter; + /* The list of bios which have discard permits but still need a data_vio */ + struct bio_list permitted_discards; + /* The list of available data_vios */ + struct list_head available; + /* The queue of data_vios waiting to be returned to the pool */ + struct funnel_queue *queue; + /* Whether the pool is processing, or scheduled to process releases */ + atomic_t processing; + /* The data vios in the pool */ + struct data_vio data_vios[]; +}; + +static const char * const ASYNC_OPERATION_NAMES[] = { + "launch", + "acknowledge_write", + "acquire_hash_lock", + "attempt_logical_block_lock", + "lock_duplicate_pbn", + "check_for_duplication", + "cleanup", + "compress_data_vio", + "find_block_map_slot", + "get_mapped_block_for_read", + "get_mapped_block_for_write", + "hash_data_vio", + "journal_remapping", + "vdo_attempt_packing", + "put_mapped_block", + "read_data_vio", + "update_dedupe_index", + "update_reference_counts", + "verify_duplication", + "write_data_vio", +}; + +/* The steps taken cleaning up a VIO, in the order they are performed. */ +enum data_vio_cleanup_stage { + VIO_CLEANUP_START, + VIO_RELEASE_HASH_LOCK = VIO_CLEANUP_START, + VIO_RELEASE_ALLOCATED, + VIO_RELEASE_RECOVERY_LOCKS, + VIO_RELEASE_LOGICAL, + VIO_CLEANUP_DONE +}; + +static inline struct data_vio_pool * __must_check +as_data_vio_pool(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_DATA_VIO_POOL_COMPLETION); + return container_of(completion, struct data_vio_pool, completion); +} + +static inline u64 get_arrival_time(struct bio *bio) +{ + return (u64) bio->bi_private; +} + +/** + * check_for_drain_complete_locked() - Check whether a data_vio_pool has no outstanding data_vios + * or waiters while holding the pool's lock. + */ +static bool check_for_drain_complete_locked(struct data_vio_pool *pool) +{ + if (pool->limiter.busy > 0) + return false; + + ASSERT_LOG_ONLY((pool->discard_limiter.busy == 0), + "no outstanding discard permits"); + + return (bio_list_empty(&pool->limiter.new_waiters) && + bio_list_empty(&pool->discard_limiter.new_waiters)); +} + +static void initialize_lbn_lock(struct data_vio *data_vio, logical_block_number_t lbn) +{ + struct vdo *vdo = vdo_from_data_vio(data_vio); + zone_count_t zone_number; + struct lbn_lock *lock = &data_vio->logical; + + lock->lbn = lbn; + lock->locked = false; + vdo_initialize_wait_queue(&lock->waiters); + zone_number = vdo_compute_logical_zone(data_vio); + lock->zone = &vdo->logical_zones->zones[zone_number]; +} + +static void launch_locked_request(struct data_vio *data_vio) +{ + data_vio->logical.locked = true; + if (data_vio->write) { + struct vdo *vdo = vdo_from_data_vio(data_vio); + + if (vdo_is_read_only(vdo)) { + continue_data_vio_with_error(data_vio, VDO_READ_ONLY); + return; + } + } + + data_vio->last_async_operation = VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT; + vdo_find_block_map_slot(data_vio); +} + +static void acknowledge_data_vio(struct data_vio *data_vio) +{ + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct bio *bio = data_vio->user_bio; + int error = vdo_map_to_system_error(data_vio->vio.completion.result); + + if (bio == NULL) + return; + + ASSERT_LOG_ONLY((data_vio->remaining_discard <= + (u32) (VDO_BLOCK_SIZE - data_vio->offset)), + "data_vio to acknowledge is not an incomplete discard"); + + data_vio->user_bio = NULL; + vdo_count_bios(&vdo->stats.bios_acknowledged, bio); + if (data_vio->is_partial) + vdo_count_bios(&vdo->stats.bios_acknowledged_partial, bio); + + bio->bi_status = errno_to_blk_status(error); + bio_endio(bio); +} + +static void copy_to_bio(struct bio *bio, char *data_ptr) +{ + struct bio_vec biovec; + struct bvec_iter iter; + + bio_for_each_segment(biovec, bio, iter) { + memcpy_to_bvec(&biovec, data_ptr); + data_ptr += biovec.bv_len; + } +} + +struct data_vio_compression_status get_data_vio_compression_status(struct data_vio *data_vio) +{ + u32 packed = atomic_read(&data_vio->compression.status); + + /* pairs with cmpxchg in set_data_vio_compression_status */ + smp_rmb(); + return (struct data_vio_compression_status) { + .stage = packed & COMPRESSION_STATUS_MASK, + .may_not_compress = ((packed & MAY_NOT_COMPRESS_MASK) != 0), + }; +} + +/** + * pack_status() - Convert a data_vio_compression_status into a u32 which may be stored + * atomically. + * @status: The state to convert. + * + * Return: The compression state packed into a u32. + */ +static u32 __must_check pack_status(struct data_vio_compression_status status) +{ + return status.stage | (status.may_not_compress ? MAY_NOT_COMPRESS_MASK : 0); +} + +/** + * set_data_vio_compression_status() - Set the compression status of a data_vio. + * @state: The expected current status of the data_vio. + * @new_state: The status to set. + * + * Return: true if the new status was set, false if the data_vio's compression status did not + * match the expected state, and so was left unchanged. + */ +static bool __must_check +set_data_vio_compression_status(struct data_vio *data_vio, + struct data_vio_compression_status status, + struct data_vio_compression_status new_status) +{ + u32 actual; + u32 expected = pack_status(status); + u32 replacement = pack_status(new_status); + + /* + * Extra barriers because this was original developed using a CAS operation that implicitly + * had them. + */ + smp_mb__before_atomic(); + actual = atomic_cmpxchg(&data_vio->compression.status, expected, replacement); + /* same as before_atomic */ + smp_mb__after_atomic(); + return (expected == actual); +} + +struct data_vio_compression_status advance_data_vio_compression_stage(struct data_vio *data_vio) +{ + for (;;) { + struct data_vio_compression_status status = + get_data_vio_compression_status(data_vio); + struct data_vio_compression_status new_status = status; + + if (status.stage == DATA_VIO_POST_PACKER) { + /* We're already in the last stage. */ + return status; + } + + if (status.may_not_compress) { + /* + * Compression has been dis-allowed for this VIO, so skip the rest of the + * path and go to the end. + */ + new_status.stage = DATA_VIO_POST_PACKER; + } else { + /* Go to the next state. */ + new_status.stage++; + } + + if (set_data_vio_compression_status(data_vio, status, new_status)) + return new_status; + + /* Another thread changed the status out from under us so try again. */ + } +} + +/** + * cancel_data_vio_compression() - Prevent this data_vio from being compressed or packed. + * + * Return: true if the data_vio is in the packer and the caller was the first caller to cancel it. + */ +bool cancel_data_vio_compression(struct data_vio *data_vio) +{ + struct data_vio_compression_status status, new_status; + + for (;;) { + status = get_data_vio_compression_status(data_vio); + if (status.may_not_compress || (status.stage == DATA_VIO_POST_PACKER)) { + /* This data_vio is already set up to not block in the packer. */ + break; + } + + new_status.stage = status.stage; + new_status.may_not_compress = true; + + if (set_data_vio_compression_status(data_vio, status, new_status)) + break; + } + + return ((status.stage == DATA_VIO_PACKING) && !status.may_not_compress); +} + +/** + * attempt_logical_block_lock() - Attempt to acquire the lock on a logical block. + * @completion: The data_vio for an external data request as a completion. + * + * This is the start of the path for all external requests. It is registered in launch_data_vio(). + */ +static void attempt_logical_block_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct lbn_lock *lock = &data_vio->logical; + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct data_vio *lock_holder; + int result; + + assert_data_vio_in_logical_zone(data_vio); + + if (data_vio->logical.lbn >= vdo->states.vdo.config.logical_blocks) { + continue_data_vio_with_error(data_vio, VDO_OUT_OF_RANGE); + return; + } + + result = vdo_int_map_put(lock->zone->lbn_operations, lock->lbn, + data_vio, false, (void **) &lock_holder); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + if (lock_holder == NULL) { + /* We got the lock */ + launch_locked_request(data_vio); + return; + } + + result = ASSERT(lock_holder->logical.locked, "logical block lock held"); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + /* + * If the new request is a pure read request (not read-modify-write) and the lock_holder is + * writing and has received an allocation (VDO-2683), service the read request immediately + * by copying data from the lock_holder to avoid having to flush the write out of the + * packer just to prevent the read from waiting indefinitely. If the lock_holder does not + * yet have an allocation, prevent it from blocking in the packer and wait on it. + */ + if (!data_vio->write && READ_ONCE(lock_holder->allocation_succeeded)) { + copy_to_bio(data_vio->user_bio, lock_holder->vio.data + data_vio->offset); + acknowledge_data_vio(data_vio); + complete_data_vio(completion); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK; + vdo_enqueue_waiter(&lock_holder->logical.waiters, &data_vio->waiter); + + /* + * Prevent writes and read-modify-writes from blocking indefinitely on lock holders in the + * packer. + */ + if (lock_holder->write && cancel_data_vio_compression(lock_holder)) { + data_vio->compression.lock_holder = lock_holder; + launch_data_vio_packer_callback(data_vio, + vdo_remove_lock_holder_from_packer); + } +} + +/** + * launch_data_vio() - (Re)initialize a data_vio to have a new logical block number, keeping the + * same parent and other state and send it on its way. + */ +static void launch_data_vio(struct data_vio *data_vio, logical_block_number_t lbn) +{ + struct vdo_completion *completion = &data_vio->vio.completion; + + /* + * Clearing the tree lock must happen before initializing the LBN lock, which also adds + * information to the tree lock. + */ + memset(&data_vio->tree_lock, 0, sizeof(data_vio->tree_lock)); + initialize_lbn_lock(data_vio, lbn); + INIT_LIST_HEAD(&data_vio->hash_lock_entry); + INIT_LIST_HEAD(&data_vio->write_entry); + + memset(&data_vio->allocation, 0, sizeof(data_vio->allocation)); + + data_vio->is_duplicate = false; + + memset(&data_vio->record_name, 0, sizeof(data_vio->record_name)); + memset(&data_vio->duplicate, 0, sizeof(data_vio->duplicate)); + vdo_reset_completion(completion); + completion->error_handler = handle_data_vio_error; + set_data_vio_logical_callback(data_vio, attempt_logical_block_lock); + vdo_enqueue_completion(completion, VDO_DEFAULT_Q_MAP_BIO_PRIORITY); +} + +static bool is_zero_block(char *block) +{ + int i; + + for (i = 0; i < VDO_BLOCK_SIZE; i += sizeof(u64)) { + if (*((u64 *) &block[i])) + return false; + } + + return true; +} + +static void copy_from_bio(struct bio *bio, char *data_ptr) +{ + struct bio_vec biovec; + struct bvec_iter iter; + + bio_for_each_segment(biovec, bio, iter) { + memcpy_from_bvec(data_ptr, &biovec); + data_ptr += biovec.bv_len; + } +} + +static void launch_bio(struct vdo *vdo, struct data_vio *data_vio, struct bio *bio) +{ + logical_block_number_t lbn; + /* + * Zero out the fields which don't need to be preserved (i.e. which are not pointers to + * separately allocated objects). + */ + memset(data_vio, 0, offsetof(struct data_vio, vio)); + memset(&data_vio->compression, 0, offsetof(struct compression_state, block)); + + data_vio->user_bio = bio; + data_vio->offset = to_bytes(bio->bi_iter.bi_sector & VDO_SECTORS_PER_BLOCK_MASK); + data_vio->is_partial = (bio->bi_iter.bi_size < VDO_BLOCK_SIZE) || (data_vio->offset != 0); + + /* + * Discards behave very differently than other requests when coming in from device-mapper. + * We have to be able to handle any size discards and various sector offsets within a + * block. + */ + if (bio_op(bio) == REQ_OP_DISCARD) { + data_vio->remaining_discard = bio->bi_iter.bi_size; + data_vio->write = true; + data_vio->is_trim = true; + if (data_vio->is_partial) { + vdo_count_bios(&vdo->stats.bios_in_partial, bio); + data_vio->read = true; + } + } else if (data_vio->is_partial) { + vdo_count_bios(&vdo->stats.bios_in_partial, bio); + data_vio->read = true; + if (bio_data_dir(bio) == WRITE) + data_vio->write = true; + } else if (bio_data_dir(bio) == READ) { + data_vio->read = true; + } else { + /* + * Copy the bio data to a char array so that we can continue to use the data after + * we acknowledge the bio. + */ + copy_from_bio(bio, data_vio->vio.data); + data_vio->is_zero = is_zero_block(data_vio->vio.data); + data_vio->write = true; + } + + if (data_vio->user_bio->bi_opf & REQ_FUA) + data_vio->fua = true; + + lbn = (bio->bi_iter.bi_sector - vdo->starting_sector_offset) / VDO_SECTORS_PER_BLOCK; + launch_data_vio(data_vio, lbn); +} + +static void assign_data_vio(struct limiter *limiter, struct data_vio *data_vio) +{ + struct bio *bio = bio_list_pop(limiter->permitted_waiters); + + launch_bio(limiter->pool->completion.vdo, data_vio, bio); + limiter->wake_count++; + + bio = bio_list_peek(limiter->permitted_waiters); + limiter->arrival = ((bio == NULL) ? U64_MAX : get_arrival_time(bio)); +} + +static void assign_discard_permit(struct limiter *limiter) +{ + struct bio *bio = bio_list_pop(&limiter->waiters); + + if (limiter->arrival == U64_MAX) + limiter->arrival = get_arrival_time(bio); + + bio_list_add(limiter->permitted_waiters, bio); +} + +static void get_waiters(struct limiter *limiter) +{ + bio_list_merge(&limiter->waiters, &limiter->new_waiters); + bio_list_init(&limiter->new_waiters); +} + +static inline struct data_vio *get_available_data_vio(struct data_vio_pool *pool) +{ + struct data_vio *data_vio = + list_first_entry(&pool->available, struct data_vio, pool_entry); + + list_del_init(&data_vio->pool_entry); + return data_vio; +} + +static void assign_data_vio_to_waiter(struct limiter *limiter) +{ + assign_data_vio(limiter, get_available_data_vio(limiter->pool)); +} + +static void update_limiter(struct limiter *limiter) +{ + struct bio_list *waiters = &limiter->waiters; + data_vio_count_t available = limiter->limit - limiter->busy; + + ASSERT_LOG_ONLY((limiter->release_count <= limiter->busy), + "Release count %u is not more than busy count %u", + limiter->release_count, limiter->busy); + + get_waiters(limiter); + for (; (limiter->release_count > 0) && !bio_list_empty(waiters); limiter->release_count--) + limiter->assigner(limiter); + + if (limiter->release_count > 0) { + WRITE_ONCE(limiter->busy, limiter->busy - limiter->release_count); + limiter->release_count = 0; + return; + } + + for (; (available > 0) && !bio_list_empty(waiters); available--) + limiter->assigner(limiter); + + WRITE_ONCE(limiter->busy, limiter->limit - available); + if (limiter->max_busy < limiter->busy) + WRITE_ONCE(limiter->max_busy, limiter->busy); +} + +/** + * schedule_releases() - Ensure that release processing is scheduled. + * + * If this call switches the state to processing, enqueue. Otherwise, some other thread has already + * done so. + */ +static void schedule_releases(struct data_vio_pool *pool) +{ + /* Pairs with the barrier in process_release_callback(). */ + smp_mb__before_atomic(); + if (atomic_cmpxchg(&pool->processing, false, true)) + return; + + pool->completion.requeue = true; + vdo_launch_completion_with_priority(&pool->completion, + CPU_Q_COMPLETE_VIO_PRIORITY); +} + +static void reuse_or_release_resources(struct data_vio_pool *pool, + struct data_vio *data_vio, + struct list_head *returned) +{ + if (data_vio->remaining_discard > 0) { + if (bio_list_empty(&pool->discard_limiter.waiters)) { + /* Return the data_vio's discard permit. */ + pool->discard_limiter.release_count++; + } else { + assign_discard_permit(&pool->discard_limiter); + } + } + + if (pool->limiter.arrival < pool->discard_limiter.arrival) { + assign_data_vio(&pool->limiter, data_vio); + } else if (pool->discard_limiter.arrival < U64_MAX) { + assign_data_vio(&pool->discard_limiter, data_vio); + } else { + list_add(&data_vio->pool_entry, returned); + pool->limiter.release_count++; + } +} + +/** + * process_release_callback() - Process a batch of data_vio releases. + * @completion: The pool with data_vios to release. + */ +static void process_release_callback(struct vdo_completion *completion) +{ + struct data_vio_pool *pool = as_data_vio_pool(completion); + bool reschedule; + bool drained; + data_vio_count_t processed; + data_vio_count_t to_wake; + data_vio_count_t discards_to_wake; + LIST_HEAD(returned); + + spin_lock(&pool->lock); + get_waiters(&pool->discard_limiter); + get_waiters(&pool->limiter); + spin_unlock(&pool->lock); + + if (pool->limiter.arrival == U64_MAX) { + struct bio *bio = bio_list_peek(&pool->limiter.waiters); + + if (bio != NULL) + pool->limiter.arrival = get_arrival_time(bio); + } + + for (processed = 0; processed < DATA_VIO_RELEASE_BATCH_SIZE; processed++) { + struct data_vio *data_vio; + struct funnel_queue_entry *entry = uds_funnel_queue_poll(pool->queue); + + if (entry == NULL) + break; + + data_vio = as_data_vio(container_of(entry, struct vdo_completion, + work_queue_entry_link)); + acknowledge_data_vio(data_vio); + reuse_or_release_resources(pool, data_vio, &returned); + } + + spin_lock(&pool->lock); + /* + * There is a race where waiters could be added while we are in the unlocked section above. + * Those waiters could not see the resources we are now about to release, so we assign + * those resources now as we have no guarantee of being rescheduled. This is handled in + * update_limiter(). + */ + update_limiter(&pool->discard_limiter); + list_splice(&returned, &pool->available); + update_limiter(&pool->limiter); + to_wake = pool->limiter.wake_count; + pool->limiter.wake_count = 0; + discards_to_wake = pool->discard_limiter.wake_count; + pool->discard_limiter.wake_count = 0; + + atomic_set(&pool->processing, false); + /* Pairs with the barrier in schedule_releases(). */ + smp_mb(); + + reschedule = !uds_is_funnel_queue_empty(pool->queue); + drained = (!reschedule && + vdo_is_state_draining(&pool->state) && + check_for_drain_complete_locked(pool)); + spin_unlock(&pool->lock); + + if (to_wake > 0) + wake_up_nr(&pool->limiter.blocked_threads, to_wake); + + if (discards_to_wake > 0) + wake_up_nr(&pool->discard_limiter.blocked_threads, discards_to_wake); + + if (reschedule) + schedule_releases(pool); + else if (drained) + vdo_finish_draining(&pool->state); +} + +static void initialize_limiter(struct limiter *limiter, struct data_vio_pool *pool, + assigner_fn assigner, data_vio_count_t limit) +{ + limiter->pool = pool; + limiter->assigner = assigner; + limiter->limit = limit; + limiter->arrival = U64_MAX; + init_waitqueue_head(&limiter->blocked_threads); +} + +/** + * initialize_data_vio() - Allocate the components of a data_vio. + * + * The caller is responsible for cleaning up the data_vio on error. + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_data_vio(struct data_vio *data_vio, struct vdo *vdo) +{ + struct bio *bio; + int result; + + BUILD_BUG_ON(VDO_BLOCK_SIZE > PAGE_SIZE); + result = uds_allocate_memory(VDO_BLOCK_SIZE, 0, "data_vio data", + &data_vio->vio.data); + if (result != VDO_SUCCESS) + return uds_log_error_strerror(result, + "data_vio data allocation failure"); + + result = uds_allocate_memory(VDO_BLOCK_SIZE, 0, "compressed block", + &data_vio->compression.block); + if (result != VDO_SUCCESS) { + return uds_log_error_strerror(result, + "data_vio compressed block allocation failure"); + } + + result = uds_allocate_memory(VDO_BLOCK_SIZE, 0, "vio scratch", + &data_vio->scratch_block); + if (result != VDO_SUCCESS) + return uds_log_error_strerror(result, + "data_vio scratch allocation failure"); + + result = vdo_create_bio(&bio); + if (result != VDO_SUCCESS) + return uds_log_error_strerror(result, + "data_vio data bio allocation failure"); + + vdo_initialize_completion(&data_vio->decrement_completion, vdo, + VDO_DECREMENT_COMPLETION); + initialize_vio(&data_vio->vio, bio, 1, VIO_TYPE_DATA, VIO_PRIORITY_DATA, vdo); + + return VDO_SUCCESS; +} + +static void destroy_data_vio(struct data_vio *data_vio) +{ + if (data_vio == NULL) + return; + + vdo_free_bio(uds_forget(data_vio->vio.bio)); + uds_free(uds_forget(data_vio->vio.data)); + uds_free(uds_forget(data_vio->compression.block)); + uds_free(uds_forget(data_vio->scratch_block)); +} + +/** + * make_data_vio_pool() - Initialize a data_vio pool. + * @vdo: The vdo to which the pool will belong. + * @pool_size: The number of data_vios in the pool. + * @discard_limit: The maximum number of data_vios which may be used for discards. + * @pool: A pointer to hold the newly allocated pool. + */ +int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, + data_vio_count_t discard_limit, struct data_vio_pool **pool_ptr) +{ + int result; + struct data_vio_pool *pool; + data_vio_count_t i; + + result = uds_allocate_extended(struct data_vio_pool, pool_size, struct data_vio, + __func__, &pool); + if (result != UDS_SUCCESS) + return result; + + ASSERT_LOG_ONLY((discard_limit <= pool_size), + "discard limit does not exceed pool size"); + initialize_limiter(&pool->discard_limiter, pool, assign_discard_permit, + discard_limit); + pool->discard_limiter.permitted_waiters = &pool->permitted_discards; + initialize_limiter(&pool->limiter, pool, assign_data_vio_to_waiter, pool_size); + pool->limiter.permitted_waiters = &pool->limiter.waiters; + INIT_LIST_HEAD(&pool->available); + spin_lock_init(&pool->lock); + vdo_set_admin_state_code(&pool->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + vdo_initialize_completion(&pool->completion, vdo, VDO_DATA_VIO_POOL_COMPLETION); + vdo_prepare_completion(&pool->completion, process_release_callback, + process_release_callback, vdo->thread_config.cpu_thread, + NULL); + + result = uds_make_funnel_queue(&pool->queue); + if (result != UDS_SUCCESS) { + free_data_vio_pool(uds_forget(pool)); + return result; + } + + for (i = 0; i < pool_size; i++) { + struct data_vio *data_vio = &pool->data_vios[i]; + + result = initialize_data_vio(data_vio, vdo); + if (result != VDO_SUCCESS) { + destroy_data_vio(data_vio); + free_data_vio_pool(pool); + return result; + } + + list_add(&data_vio->pool_entry, &pool->available); + } + + *pool_ptr = pool; + return VDO_SUCCESS; +} + +/** + * free_data_vio_pool() - Free a data_vio_pool and the data_vios in it. + * + * All data_vios must be returned to the pool before calling this function. + */ +void free_data_vio_pool(struct data_vio_pool *pool) +{ + struct data_vio *data_vio, *tmp; + + if (pool == NULL) + return; + + /* + * Pairs with the barrier in process_release_callback(). Possibly not needed since it + * caters to an enqueue vs. free race. + */ + smp_mb(); + BUG_ON(atomic_read(&pool->processing)); + + spin_lock(&pool->lock); + ASSERT_LOG_ONLY((pool->limiter.busy == 0), + "data_vio pool must not have %u busy entries when being freed", + pool->limiter.busy); + ASSERT_LOG_ONLY((bio_list_empty(&pool->limiter.waiters) && + bio_list_empty(&pool->limiter.new_waiters)), + "data_vio pool must not have threads waiting to read or write when being freed"); + ASSERT_LOG_ONLY((bio_list_empty(&pool->discard_limiter.waiters) && + bio_list_empty(&pool->discard_limiter.new_waiters)), + "data_vio pool must not have threads waiting to discard when being freed"); + spin_unlock(&pool->lock); + + list_for_each_entry_safe(data_vio, tmp, &pool->available, pool_entry) { + list_del_init(&data_vio->pool_entry); + destroy_data_vio(data_vio); + } + + uds_free_funnel_queue(uds_forget(pool->queue)); + uds_free(pool); +} + +static bool acquire_permit(struct limiter *limiter, struct bio *bio) +{ + if (limiter->busy >= limiter->limit) { + DEFINE_WAIT(wait); + + bio_list_add(&limiter->new_waiters, bio); + prepare_to_wait_exclusive(&limiter->blocked_threads, &wait, + TASK_UNINTERRUPTIBLE); + spin_unlock(&limiter->pool->lock); + io_schedule(); + finish_wait(&limiter->blocked_threads, &wait); + return false; + } + + WRITE_ONCE(limiter->busy, limiter->busy + 1); + if (limiter->max_busy < limiter->busy) + WRITE_ONCE(limiter->max_busy, limiter->busy); + + return true; +} + +/** + * vdo_launch_bio() - Acquire a data_vio from the pool, assign the bio to it, and launch it. + * + * This will block if data_vios or discard permits are not available. + */ +void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio) +{ + struct data_vio *data_vio; + + ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&pool->state), + "data_vio_pool not quiescent on acquire"); + + bio->bi_private = (void *) jiffies; + spin_lock(&pool->lock); + if ((bio_op(bio) == REQ_OP_DISCARD) && + !acquire_permit(&pool->discard_limiter, bio)) + return; + + if (!acquire_permit(&pool->limiter, bio)) + return; + + data_vio = get_available_data_vio(pool); + spin_unlock(&pool->lock); + launch_bio(pool->completion.vdo, data_vio, bio); +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_drain(struct admin_state *state) +{ + bool drained; + struct data_vio_pool *pool = container_of(state, struct data_vio_pool, state); + + spin_lock(&pool->lock); + drained = check_for_drain_complete_locked(pool); + spin_unlock(&pool->lock); + + if (drained) + vdo_finish_draining(state); +} + +static void assert_on_vdo_cpu_thread(const struct vdo *vdo, const char *name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.cpu_thread), + "%s called on cpu thread", name); +} + +/** + * drain_data_vio_pool() - Wait asynchronously for all data_vios to be returned to the pool. + * @completion: The completion to notify when the pool has drained. + */ +void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion) +{ + assert_on_vdo_cpu_thread(completion->vdo, __func__); + vdo_start_draining(&pool->state, VDO_ADMIN_STATE_SUSPENDING, completion, + initiate_drain); +} + +/** + * resume_data_vio_pool() - Resume a data_vio pool. + * @completion: The completion to notify when the pool has resumed. + */ +void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion) +{ + assert_on_vdo_cpu_thread(completion->vdo, __func__); + vdo_continue_completion(completion, vdo_resume_if_quiescent(&pool->state)); +} + +static void dump_limiter(const char *name, struct limiter *limiter) +{ + uds_log_info("%s: %u of %u busy (max %u), %s", name, limiter->busy, + limiter->limit, limiter->max_busy, + ((bio_list_empty(&limiter->waiters) && + bio_list_empty(&limiter->new_waiters)) ? + "no waiters" : "has waiters")); +} + +/** + * dump_data_vio_pool() - Dump a data_vio pool to the log. + * @dump_vios: Whether to dump the details of each busy data_vio as well. + */ +void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios) +{ + /* + * In order that syslog can empty its buffer, sleep after 35 elements for 4ms (till the + * second clock tick). These numbers were picked based on experiments with lab machines. + */ + enum { ELEMENTS_PER_BATCH = 35 }; + enum { SLEEP_FOR_SYSLOG = 4000 }; + + if (pool == NULL) + return; + + spin_lock(&pool->lock); + dump_limiter("data_vios", &pool->limiter); + dump_limiter("discard permits", &pool->discard_limiter); + if (dump_vios) { + int i; + int dumped = 0; + + for (i = 0; i < pool->limiter.limit; i++) { + struct data_vio *data_vio = &pool->data_vios[i]; + + if (!list_empty(&data_vio->pool_entry)) + continue; + + dump_data_vio(data_vio); + if (++dumped >= ELEMENTS_PER_BATCH) { + spin_unlock(&pool->lock); + dumped = 0; + fsleep(SLEEP_FOR_SYSLOG); + spin_lock(&pool->lock); + } + } + } + + spin_unlock(&pool->lock); +} + +data_vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->discard_limiter.busy); +} + +data_vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->discard_limiter.limit); +} + +data_vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->discard_limiter.max_busy); +} + +int set_data_vio_pool_discard_limit(struct data_vio_pool *pool, data_vio_count_t limit) +{ + if (get_data_vio_pool_request_limit(pool) < limit) { + // The discard limit may not be higher than the data_vio limit. + return -EINVAL; + } + + spin_lock(&pool->lock); + pool->discard_limiter.limit = limit; + spin_unlock(&pool->lock); + + return VDO_SUCCESS; +} + +data_vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->limiter.busy); +} + +data_vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->limiter.limit); +} + +data_vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->limiter.max_busy); +} + +static void update_data_vio_error_stats(struct data_vio *data_vio) +{ + u8 index = 0; + static const char * const operations[] = { + [0] = "empty", + [1] = "read", + [2] = "write", + [3] = "read-modify-write", + [5] = "read+fua", + [6] = "write+fua", + [7] = "read-modify-write+fua", + }; + + if (data_vio->read) + index = 1; + + if (data_vio->write) + index += 2; + + if (data_vio->fua) + index += 4; + + update_vio_error_stats(&data_vio->vio, + "Completing %s vio for LBN %llu with error after %s", + operations[index], + (unsigned long long) data_vio->logical.lbn, + get_data_vio_operation_name(data_vio)); +} + +static void perform_cleanup_stage(struct data_vio *data_vio, + enum data_vio_cleanup_stage stage); + +/** + * release_allocated_lock() - Release the PBN lock and/or the reference on the allocated block at + * the end of processing a data_vio. + */ +static void release_allocated_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + release_data_vio_allocation_lock(data_vio, false); + perform_cleanup_stage(data_vio, VIO_RELEASE_RECOVERY_LOCKS); +} + +/** release_lock() - Release an uncontended LBN lock. */ +static void release_lock(struct data_vio *data_vio, struct lbn_lock *lock) +{ + struct int_map *lock_map = lock->zone->lbn_operations; + struct data_vio *lock_holder; + + if (!lock->locked) { + /* The lock is not locked, so it had better not be registered in the lock map. */ + struct data_vio *lock_holder = vdo_int_map_get(lock_map, lock->lbn); + + ASSERT_LOG_ONLY((data_vio != lock_holder), + "no logical block lock held for block %llu", + (unsigned long long) lock->lbn); + return; + } + + /* Release the lock by removing the lock from the map. */ + lock_holder = vdo_int_map_remove(lock_map, lock->lbn); + ASSERT_LOG_ONLY((data_vio == lock_holder), + "logical block lock mismatch for block %llu", + (unsigned long long) lock->lbn); + lock->locked = false; +} + +/** transfer_lock() - Transfer a contended LBN lock to the eldest waiter. */ +static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) +{ + struct data_vio *lock_holder, *next_lock_holder; + int result; + + ASSERT_LOG_ONLY(lock->locked, "lbn_lock with waiters is not locked"); + + /* Another data_vio is waiting for the lock, transfer it in a single lock map operation. */ + next_lock_holder = + waiter_as_data_vio(vdo_dequeue_next_waiter(&lock->waiters)); + + /* Transfer the remaining lock waiters to the next lock holder. */ + vdo_transfer_all_waiters(&lock->waiters, + &next_lock_holder->logical.waiters); + + result = vdo_int_map_put(lock->zone->lbn_operations, lock->lbn, + next_lock_holder, true, (void **) &lock_holder); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(next_lock_holder, result); + return; + } + + ASSERT_LOG_ONLY((lock_holder == data_vio), + "logical block lock mismatch for block %llu", + (unsigned long long) lock->lbn); + lock->locked = false; + + /* + * If there are still waiters, other data_vios must be trying to get the lock we just + * transferred. We must ensure that the new lock holder doesn't block in the packer. + */ + if (vdo_has_waiters(&next_lock_holder->logical.waiters)) + cancel_data_vio_compression(next_lock_holder); + + /* + * Avoid stack overflow on lock transfer. + * FIXME: this is only an issue in the 1 thread config. + */ + next_lock_holder->vio.completion.requeue = true; + launch_locked_request(next_lock_holder); +} + +/** + * release_logical_lock() - Release the logical block lock and flush generation lock at the end of + * processing a data_vio. + */ +static void release_logical_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct lbn_lock *lock = &data_vio->logical; + + assert_data_vio_in_logical_zone(data_vio); + + if (vdo_has_waiters(&lock->waiters)) + transfer_lock(data_vio, lock); + else + release_lock(data_vio, lock); + + vdo_release_flush_generation_lock(data_vio); + perform_cleanup_stage(data_vio, VIO_CLEANUP_DONE); +} + +/** clean_hash_lock() - Release the hash lock at the end of processing a data_vio. */ +static void clean_hash_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_hash_zone(data_vio); + if (completion->result != VDO_SUCCESS) { + vdo_clean_failed_hash_lock(data_vio); + return; + } + + vdo_release_hash_lock(data_vio); + perform_cleanup_stage(data_vio, VIO_RELEASE_LOGICAL); +} + +/** + * finish_cleanup() - Make some assertions about a data_vio which has finished cleaning up. + * + * If it is part of a multi-block discard, starts on the next block, otherwise, returns it to the + * pool. + */ +static void finish_cleanup(struct data_vio *data_vio) +{ + struct vdo_completion *completion = &data_vio->vio.completion; + + ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL, + "complete data_vio has no allocation lock"); + ASSERT_LOG_ONLY(data_vio->hash_lock == NULL, + "complete data_vio has no hash lock"); + if ((data_vio->remaining_discard <= VDO_BLOCK_SIZE) || + (completion->result != VDO_SUCCESS)) { + struct data_vio_pool *pool = completion->vdo->data_vio_pool; + + uds_funnel_queue_put(pool->queue, &completion->work_queue_entry_link); + schedule_releases(pool); + return; + } + + data_vio->remaining_discard -= min_t(u32, data_vio->remaining_discard, + VDO_BLOCK_SIZE - data_vio->offset); + data_vio->is_partial = (data_vio->remaining_discard < VDO_BLOCK_SIZE); + data_vio->read = data_vio->is_partial; + data_vio->offset = 0; + completion->requeue = true; + launch_data_vio(data_vio, data_vio->logical.lbn + 1); +} + +/** perform_cleanup_stage() - Perform the next step in the process of cleaning up a data_vio. */ +static void perform_cleanup_stage(struct data_vio *data_vio, + enum data_vio_cleanup_stage stage) +{ + struct vdo *vdo = vdo_from_data_vio(data_vio); + + switch (stage) { + case VIO_RELEASE_HASH_LOCK: + if (data_vio->hash_lock != NULL) { + launch_data_vio_hash_zone_callback(data_vio, clean_hash_lock); + return; + } + fallthrough; + + case VIO_RELEASE_ALLOCATED: + if (data_vio_has_allocation(data_vio)) { + launch_data_vio_allocated_zone_callback(data_vio, + release_allocated_lock); + return; + } + fallthrough; + + case VIO_RELEASE_RECOVERY_LOCKS: + if ((data_vio->recovery_sequence_number > 0) && + (READ_ONCE(vdo->read_only_notifier.read_only_error) == VDO_SUCCESS) && + (data_vio->vio.completion.result != VDO_READ_ONLY)) + uds_log_warning("VDO not read-only when cleaning data_vio with RJ lock"); + fallthrough; + + case VIO_RELEASE_LOGICAL: + launch_data_vio_logical_callback(data_vio, release_logical_lock); + return; + + default: + finish_cleanup(data_vio); + } +} + +void complete_data_vio(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + completion->error_handler = NULL; + data_vio->last_async_operation = VIO_ASYNC_OP_CLEANUP; + perform_cleanup_stage(data_vio, + (data_vio->write ? VIO_CLEANUP_START : VIO_RELEASE_LOGICAL)); +} + +static void enter_read_only_mode(struct vdo_completion *completion) +{ + if (vdo_is_read_only(completion->vdo)) + return; + + if (completion->result != VDO_READ_ONLY) { + struct data_vio *data_vio = as_data_vio(completion); + + uds_log_error_strerror(completion->result, + "Preparing to enter read-only mode: data_vio for LBN %llu (becoming mapped to %llu, previously mapped to %llu, allocated %llu) is completing with a fatal error after operation %s", + (unsigned long long) data_vio->logical.lbn, + (unsigned long long) data_vio->new_mapped.pbn, + (unsigned long long) data_vio->mapped.pbn, + (unsigned long long) data_vio->allocation.pbn, + get_data_vio_operation_name(data_vio)); + } + + vdo_enter_read_only_mode(completion->vdo, completion->result); +} + +void handle_data_vio_error(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + if ((completion->result == VDO_READ_ONLY) || (data_vio->user_bio == NULL)) + enter_read_only_mode(completion); + + update_data_vio_error_stats(data_vio); + complete_data_vio(completion); +} + +/** + * get_data_vio_operation_name() - Get the name of the last asynchronous operation performed on a + * data_vio. + */ +const char *get_data_vio_operation_name(struct data_vio *data_vio) +{ + BUILD_BUG_ON((MAX_VIO_ASYNC_OPERATION_NUMBER - MIN_VIO_ASYNC_OPERATION_NUMBER) != + ARRAY_SIZE(ASYNC_OPERATION_NAMES)); + + return ((data_vio->last_async_operation < MAX_VIO_ASYNC_OPERATION_NUMBER) ? + ASYNC_OPERATION_NAMES[data_vio->last_async_operation] : + "unknown async operation"); +} + +/** + * data_vio_allocate_data_block() - Allocate a data block. + * + * @write_lock_type: The type of write lock to obtain on the block. + * @callback: The callback which will attempt an allocation in the current zone and continue if it + * succeeds. + * @error_handler: The handler for errors while allocating. + */ +void data_vio_allocate_data_block(struct data_vio *data_vio, + enum pbn_lock_type write_lock_type, + vdo_action_fn callback, vdo_action_fn error_handler) +{ + struct allocation *allocation = &data_vio->allocation; + + ASSERT_LOG_ONLY((allocation->pbn == VDO_ZERO_BLOCK), + "data_vio does not have an allocation"); + allocation->write_lock_type = write_lock_type; + allocation->zone = vdo_get_next_allocation_zone(data_vio->logical.zone); + allocation->first_allocation_zone = allocation->zone->zone_number; + + data_vio->vio.completion.error_handler = error_handler; + launch_data_vio_allocated_zone_callback(data_vio, callback); +} + +/** + * release_data_vio_allocation_lock() - Release the PBN lock on a data_vio's allocated block. + * @reset: If true, the allocation will be reset (i.e. any allocated pbn will be forgotten). + * + * If the reference to the locked block is still provisional, it will be released as well. + */ +void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset) +{ + struct allocation *allocation = &data_vio->allocation; + physical_block_number_t locked_pbn = allocation->pbn; + + assert_data_vio_in_allocated_zone(data_vio); + + if (reset || vdo_pbn_lock_has_provisional_reference(allocation->lock)) + allocation->pbn = VDO_ZERO_BLOCK; + + vdo_release_physical_zone_pbn_lock(allocation->zone, locked_pbn, + uds_forget(allocation->lock)); +} + +/** + * uncompress_data_vio() - Uncompress the data a data_vio has just read. + * @mapping_state: The mapping state indicating which fragment to decompress. + * @buffer: The buffer to receive the uncompressed data. + */ +int uncompress_data_vio(struct data_vio *data_vio, + enum block_mapping_state mapping_state, char *buffer) +{ + int size; + u16 fragment_offset, fragment_size; + struct compressed_block *block = data_vio->compression.block; + int result = vdo_get_compressed_block_fragment(mapping_state, block, + &fragment_offset, &fragment_size); + + if (result != VDO_SUCCESS) { + uds_log_debug("%s: compressed fragment error %d", __func__, result); + return result; + } + + size = LZ4_decompress_safe((block->data + fragment_offset), buffer, + fragment_size, VDO_BLOCK_SIZE); + if (size != VDO_BLOCK_SIZE) { + uds_log_debug("%s: lz4 error", __func__); + return VDO_INVALID_FRAGMENT; + } + + return VDO_SUCCESS; +} + +/** + * modify_for_partial_write() - Do the modify-write part of a read-modify-write cycle. + * @completion: The data_vio which has just finished its read. + * + * This callback is registered in read_block(). + */ +static void modify_for_partial_write(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + char *data = data_vio->vio.data; + struct bio *bio = data_vio->user_bio; + + assert_data_vio_on_cpu_thread(data_vio); + + if (bio_op(bio) == REQ_OP_DISCARD) { + memset(data + data_vio->offset, '\0', min_t(u32, + data_vio->remaining_discard, + VDO_BLOCK_SIZE - data_vio->offset)); + } else { + copy_from_bio(bio, data + data_vio->offset); + } + + data_vio->is_zero = is_zero_block(data); + data_vio->read = false; + launch_data_vio_logical_callback(data_vio, + continue_data_vio_with_block_map_slot); +} + +static void complete_read(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + char *data = data_vio->vio.data; + bool compressed = vdo_is_state_compressed(data_vio->mapped.state); + + assert_data_vio_on_cpu_thread(data_vio); + + if (compressed) { + int result = uncompress_data_vio(data_vio, data_vio->mapped.state, data); + + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + } + + if (data_vio->write) { + modify_for_partial_write(completion); + return; + } + + if (compressed || data_vio->is_partial) + copy_to_bio(data_vio->user_bio, data + data_vio->offset); + + acknowledge_data_vio(data_vio); + complete_data_vio(completion); +} + +static void read_endio(struct bio *bio) +{ + struct data_vio *data_vio = vio_as_data_vio(bio->bi_private); + int result = blk_status_to_errno(bio->bi_status); + + vdo_count_completed_bios(bio); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + launch_data_vio_cpu_callback(data_vio, complete_read, + CPU_Q_COMPLETE_READ_PRIORITY); +} + +static void complete_zero_read(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_on_cpu_thread(data_vio); + + if (data_vio->is_partial) { + memset(data_vio->vio.data, 0, VDO_BLOCK_SIZE); + if (data_vio->write) { + modify_for_partial_write(completion); + return; + } + } else { + zero_fill_bio(data_vio->user_bio); + } + + complete_read(completion); +} + +/** + * read_block() - Read a block asynchronously. + * + * This is the callback registered in read_block_mapping(). + */ +static void read_block(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct vio *vio = as_vio(completion); + int result = VDO_SUCCESS; + + if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) { + launch_data_vio_cpu_callback(data_vio, complete_zero_read, + CPU_Q_COMPLETE_VIO_PRIORITY); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_READ_DATA_VIO; + if (vdo_is_state_compressed(data_vio->mapped.state)) { + result = vio_reset_bio(vio, (char *) data_vio->compression.block, + read_endio, REQ_OP_READ, data_vio->mapped.pbn); + } else { + int opf = ((data_vio->user_bio->bi_opf & PASSTHROUGH_FLAGS) | REQ_OP_READ); + + if (data_vio->is_partial) { + result = vio_reset_bio(vio, vio->data, read_endio, opf, + data_vio->mapped.pbn); + } else { + /* A full 4k read. Use the incoming bio to avoid having to copy the data */ + bio_reset(vio->bio, vio->bio->bi_bdev, opf); + bio_init_clone(data_vio->user_bio->bi_bdev, vio->bio, + data_vio->user_bio, GFP_KERNEL); + + /* Copy over the original bio iovec and opflags. */ + vdo_set_bio_properties(vio->bio, vio, read_endio, opf, + data_vio->mapped.pbn); + } + } + + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + submit_data_vio_io(data_vio); +} + +static inline struct data_vio * +reference_count_update_completion_as_data_vio(struct vdo_completion *completion) +{ + if (completion->type == VIO_COMPLETION) + return as_data_vio(completion); + + return container_of(completion, struct data_vio, decrement_completion); +} + +/** + * update_block_map() - Rendezvous of the data_vio and decrement completions after each has + * made its reference updates. Handle any error from either, or proceed + * to updating the block map. + * @completion: The completion of the write in progress. + */ +static void update_block_map(struct vdo_completion *completion) +{ + struct data_vio *data_vio = reference_count_update_completion_as_data_vio(completion); + + assert_data_vio_in_logical_zone(data_vio); + + if (!data_vio->first_reference_operation_complete) { + /* Rendezvous, we're first */ + data_vio->first_reference_operation_complete = true; + return; + } + + completion = &data_vio->vio.completion; + vdo_set_completion_result(completion, data_vio->decrement_completion.result); + if (completion->result != VDO_SUCCESS) { + handle_data_vio_error(completion); + return; + } + + completion->error_handler = handle_data_vio_error; + if (data_vio->hash_lock != NULL) + set_data_vio_hash_zone_callback(data_vio, vdo_continue_hash_lock); + else + completion->callback = complete_data_vio; + + data_vio->last_async_operation = VIO_ASYNC_OP_PUT_MAPPED_BLOCK; + vdo_put_mapped_block(data_vio); +} + +static void decrement_reference_count(struct vdo_completion *completion) +{ + struct data_vio *data_vio = container_of(completion, struct data_vio, + decrement_completion); + + assert_data_vio_in_mapped_zone(data_vio); + + vdo_set_completion_callback(completion, update_block_map, + data_vio->logical.zone->thread_id); + completion->error_handler = update_block_map; + vdo_modify_reference_count(completion, &data_vio->decrement_updater); +} + +static void increment_reference_count(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_new_mapped_zone(data_vio); + + if (data_vio->downgrade_allocation_lock) { + /* + * Now that the data has been written, it's safe to deduplicate against the + * block. Downgrade the allocation lock to a read lock so it can be used later by + * the hash lock. This is done here since it needs to happen sometime before we + * return to the hash zone, and we are currently on the correct thread. For + * compressed blocks, the downgrade will have already been done. + */ + vdo_downgrade_pbn_write_lock(data_vio->allocation.lock, false); + } + + set_data_vio_logical_callback(data_vio, update_block_map); + completion->error_handler = update_block_map; + vdo_modify_reference_count(completion, &data_vio->increment_updater); +} + +/** journal_remapping() - Add a recovery journal entry for a data remapping. */ +static void journal_remapping(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_journal_zone(data_vio); + + data_vio->decrement_updater.operation = VDO_JOURNAL_DATA_REMAPPING; + data_vio->decrement_updater.zpbn = data_vio->mapped; + if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) { + data_vio->first_reference_operation_complete = true; + if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) + set_data_vio_logical_callback(data_vio, update_block_map); + } else { + set_data_vio_new_mapped_zone_callback(data_vio, + increment_reference_count); + } + + if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) { + data_vio->first_reference_operation_complete = true; + } else { + vdo_set_completion_callback(&data_vio->decrement_completion, + decrement_reference_count, + data_vio->mapped.zone->thread_id); + } + + data_vio->last_async_operation = VIO_ASYNC_OP_JOURNAL_REMAPPING; + vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio); +} + +/** + * read_old_block_mapping() - Get the previous PBN/LBN mapping of an in-progress write. + * + * Gets the previous PBN mapped to this LBN from the block map, so as to make an appropriate + * journal entry referencing the removal of this LBN->PBN mapping. + */ +static void read_old_block_mapping(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_logical_zone(data_vio); + + data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE; + set_data_vio_journal_callback(data_vio, journal_remapping); + vdo_get_mapped_block(data_vio); +} + +void update_metadata_for_data_vio_write(struct data_vio *data_vio, struct pbn_lock *lock) +{ + data_vio->increment_updater = (struct reference_updater) { + .operation = VDO_JOURNAL_DATA_REMAPPING, + .increment = true, + .zpbn = data_vio->new_mapped, + .lock = lock, + }; + + launch_data_vio_logical_callback(data_vio, read_old_block_mapping); +} + +/** + * pack_compressed_data() - Attempt to pack the compressed data_vio into a block. + * + * This is the callback registered in launch_compress_data_vio(). + */ +static void pack_compressed_data(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_packer_zone(data_vio); + + if (!vdo_get_compressing(vdo_from_data_vio(data_vio)) || + get_data_vio_compression_status(data_vio).may_not_compress) { + write_data_vio(data_vio); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_PACKING; + vdo_attempt_packing(data_vio); +} + +/** + * compress_data_vio() - Do the actual work of compressing the data on a CPU queue. + * + * This callback is registered in launch_compress_data_vio(). + */ +static void compress_data_vio(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + int size; + + assert_data_vio_on_cpu_thread(data_vio); + + /* + * By putting the compressed data at the start of the compressed block data field, we won't + * need to copy it if this data_vio becomes a compressed write agent. + */ + size = LZ4_compress_default(data_vio->vio.data, + data_vio->compression.block->data, VDO_BLOCK_SIZE, + VDO_MAX_COMPRESSED_FRAGMENT_SIZE, + (char *) vdo_get_work_queue_private_data()); + if ((size > 0) && (size < VDO_COMPRESSED_BLOCK_DATA_SIZE)) { + data_vio->compression.size = size; + launch_data_vio_packer_callback(data_vio, pack_compressed_data); + return; + } + + write_data_vio(data_vio); +} + +/** + * launch_compress_data_vio() - Continue a write by attempting to compress the data. + * + * This is a re-entry point to vio_write used by hash locks. + */ +void launch_compress_data_vio(struct data_vio *data_vio) +{ + ASSERT_LOG_ONLY(!data_vio->is_duplicate, "compressing a non-duplicate block"); + ASSERT_LOG_ONLY(data_vio->hash_lock != NULL, + "data_vio to compress has a hash_lock"); + ASSERT_LOG_ONLY(data_vio_has_allocation(data_vio), + "data_vio to compress has an allocation"); + + /* + * There are 4 reasons why a data_vio which has reached this point will not be eligible for + * compression: + * + * 1) Since data_vios can block indefinitely in the packer, it would be bad to do so if the + * write request also requests FUA. + * + * 2) A data_vio should not be compressed when compression is disabled for the vdo. + * + * 3) A data_vio could be doing a partial write on behalf of a larger discard which has not + * yet been acknowledged and hence blocking in the packer would be bad. + * + * 4) Some other data_vio may be waiting on this data_vio in which case blocking in the + * packer would also be bad. + */ + if (data_vio->fua || + !vdo_get_compressing(vdo_from_data_vio(data_vio)) || + ((data_vio->user_bio != NULL) && (bio_op(data_vio->user_bio) == REQ_OP_DISCARD)) || + (advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_COMPRESSING)) { + write_data_vio(data_vio); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_COMPRESS_DATA_VIO; + launch_data_vio_cpu_callback(data_vio, compress_data_vio, + CPU_Q_COMPRESS_BLOCK_PRIORITY); +} + +/** + * hash_data_vio() - Hash the data in a data_vio and set the hash zone (which also flags the record + * name as set). + + * This callback is registered in prepare_for_dedupe(). + */ +static void hash_data_vio(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_on_cpu_thread(data_vio); + ASSERT_LOG_ONLY(!data_vio->is_zero, "zero blocks should not be hashed"); + + murmurhash3_128(data_vio->vio.data, VDO_BLOCK_SIZE, 0x62ea60be, + &data_vio->record_name); + + data_vio->hash_zone = vdo_select_hash_zone(vdo_from_data_vio(data_vio)->hash_zones, + &data_vio->record_name); + data_vio->last_async_operation = VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK; + launch_data_vio_hash_zone_callback(data_vio, vdo_acquire_hash_lock); +} + +/** prepare_for_dedupe() - Prepare for the dedupe path after attempting to get an allocation. */ +static void prepare_for_dedupe(struct data_vio *data_vio) +{ + /* We don't care what thread we are on. */ + ASSERT_LOG_ONLY(!data_vio->is_zero, "must not prepare to dedupe zero blocks"); + + /* + * Before we can dedupe, we need to know the record name, so the first + * step is to hash the block data. + */ + data_vio->last_async_operation = VIO_ASYNC_OP_HASH_DATA_VIO; + launch_data_vio_cpu_callback(data_vio, hash_data_vio, CPU_Q_HASH_BLOCK_PRIORITY); +} + +/** + * write_bio_finished() - This is the bio_end_io function registered in write_block() to be called + * when a data_vio's write to the underlying storage has completed. + */ +static void write_bio_finished(struct bio *bio) +{ + struct data_vio *data_vio = vio_as_data_vio((struct vio *) bio->bi_private); + + vdo_count_completed_bios(bio); + vdo_set_completion_result(&data_vio->vio.completion, + blk_status_to_errno(bio->bi_status)); + data_vio->downgrade_allocation_lock = true; + update_metadata_for_data_vio_write(data_vio, data_vio->allocation.lock); +} + +/** write_data_vio() - Write a data block to storage without compression. */ +void write_data_vio(struct data_vio *data_vio) +{ + struct data_vio_compression_status status, new_status; + int result; + + if (!data_vio_has_allocation(data_vio)) { + /* + * There was no space to write this block and we failed to deduplicate or compress + * it. + */ + continue_data_vio_with_error(data_vio, VDO_NO_SPACE); + return; + } + + new_status = (struct data_vio_compression_status) { + .stage = DATA_VIO_POST_PACKER, + .may_not_compress = true, + }; + + do { + status = get_data_vio_compression_status(data_vio); + } while ((status.stage != DATA_VIO_POST_PACKER) && + !set_data_vio_compression_status(data_vio, status, new_status)); + + /* Write the data from the data block buffer. */ + result = vio_reset_bio(&data_vio->vio, data_vio->vio.data, + write_bio_finished, REQ_OP_WRITE, + data_vio->allocation.pbn); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_WRITE_DATA_VIO; + submit_data_vio_io(data_vio); +} + +/** + * acknowledge_write_callback() - Acknowledge a write to the requestor. + * + * This callback is registered in allocate_block() and continue_write_with_block_map_slot(). + */ +static void acknowledge_write_callback(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct vdo *vdo = completion->vdo; + + ASSERT_LOG_ONLY((!vdo_uses_bio_ack_queue(vdo) || + (vdo_get_callback_thread_id() == vdo->thread_config.bio_ack_thread)), + "%s() called on bio ack queue", __func__); + ASSERT_LOG_ONLY(data_vio_has_flush_generation_lock(data_vio), + "write VIO to be acknowledged has a flush generation lock"); + acknowledge_data_vio(data_vio); + if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) { + /* This is a zero write or discard */ + update_metadata_for_data_vio_write(data_vio, NULL); + return; + } + + prepare_for_dedupe(data_vio); +} + +/** + * allocate_block() - Attempt to allocate a block in the current allocation zone. + * + * This callback is registered in continue_write_with_block_map_slot(). + */ +static void allocate_block(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + + if (!vdo_allocate_block_in_zone(data_vio)) + return; + + completion->error_handler = handle_data_vio_error; + WRITE_ONCE(data_vio->allocation_succeeded, true); + data_vio->new_mapped = (struct zoned_pbn) { + .zone = data_vio->allocation.zone, + .pbn = data_vio->allocation.pbn, + .state = VDO_MAPPING_STATE_UNCOMPRESSED, + }; + + if (data_vio->fua) { + prepare_for_dedupe(data_vio); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE; + launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback); +} + +/** + * handle_allocation_error() - Handle an error attempting to allocate a block. + * + * This error handler is registered in continue_write_with_block_map_slot(). + */ +static void handle_allocation_error(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + if (completion->result == VDO_NO_SPACE) { + /* We failed to get an allocation, but we can try to dedupe. */ + vdo_reset_completion(completion); + completion->error_handler = handle_data_vio_error; + prepare_for_dedupe(data_vio); + return; + } + + /* We got a "real" error, not just a failure to allocate, so fail the request. */ + handle_data_vio_error(completion); +} + +static int assert_is_trim(struct data_vio *data_vio) +{ + int result = ASSERT(data_vio->is_trim, + "data_vio with no block map page is a trim"); + + return ((result == VDO_SUCCESS) ? result : VDO_READ_ONLY); +} + +/** + * continue_data_vio_with_block_map_slot() - Read the data_vio's mapping from the block map. + * + * This callback is registered in launch_read_data_vio(). + */ +void continue_data_vio_with_block_map_slot(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_logical_zone(data_vio); + if (data_vio->read) { + set_data_vio_logical_callback(data_vio, read_block); + data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ; + vdo_get_mapped_block(data_vio); + return; + } + + vdo_acquire_flush_generation_lock(data_vio); + + if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) { + /* + * This is a trim for a block on a block map page which has not been allocated, so + * there's nothing more we need to do. + */ + completion->callback = complete_data_vio; + continue_data_vio_with_error(data_vio, assert_is_trim(data_vio)); + return; + } + + /* + * We need an allocation if this is neither a full-block trim nor a + * full-block zero write. + */ + if (!data_vio->is_zero && (!data_vio->is_trim || data_vio->is_partial)) { + data_vio_allocate_data_block(data_vio, VIO_WRITE_LOCK, allocate_block, + handle_allocation_error); + return; + } + + + /* + * We don't need to write any data, so skip allocation and just update the block map and + * reference counts (via the journal). + */ + data_vio->new_mapped.pbn = VDO_ZERO_BLOCK; + if (data_vio->is_zero) + data_vio->new_mapped.state = VDO_MAPPING_STATE_UNCOMPRESSED; + + if (data_vio->remaining_discard > VDO_BLOCK_SIZE) { + /* This is not the final block of a discard so we can't acknowledge it yet. */ + update_metadata_for_data_vio_write(data_vio, NULL); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE; + launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback); +} diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h new file mode 100644 index 000000000000..aa415b8c7d91 --- /dev/null +++ b/drivers/md/dm-vdo/data-vio.h @@ -0,0 +1,669 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef DATA_VIO_H +#define DATA_VIO_H + +#include +#include +#include + +#include "permassert.h" +#include "uds.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "dedupe.h" +#include "encodings.h" +#include "logical-zone.h" +#include "physical-zone.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +/* Codes for describing the last asynchronous operation performed on a vio. */ +enum async_operation_number { + MIN_VIO_ASYNC_OPERATION_NUMBER, + VIO_ASYNC_OP_LAUNCH = MIN_VIO_ASYNC_OPERATION_NUMBER, + VIO_ASYNC_OP_ACKNOWLEDGE_WRITE, + VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK, + VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK, + VIO_ASYNC_OP_LOCK_DUPLICATE_PBN, + VIO_ASYNC_OP_CHECK_FOR_DUPLICATION, + VIO_ASYNC_OP_CLEANUP, + VIO_ASYNC_OP_COMPRESS_DATA_VIO, + VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT, + VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ, + VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE, + VIO_ASYNC_OP_HASH_DATA_VIO, + VIO_ASYNC_OP_JOURNAL_REMAPPING, + VIO_ASYNC_OP_ATTEMPT_PACKING, + VIO_ASYNC_OP_PUT_MAPPED_BLOCK, + VIO_ASYNC_OP_READ_DATA_VIO, + VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX, + VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS, + VIO_ASYNC_OP_VERIFY_DUPLICATION, + VIO_ASYNC_OP_WRITE_DATA_VIO, + MAX_VIO_ASYNC_OPERATION_NUMBER, +} __packed; + +struct lbn_lock { + logical_block_number_t lbn; + bool locked; + struct wait_queue waiters; + struct logical_zone *zone; +}; + +/* A position in the arboreal block map at a specific level. */ +struct block_map_tree_slot { + page_number_t page_index; + struct block_map_slot block_map_slot; +}; + +/* Fields for using the arboreal block map. */ +struct tree_lock { + /* The current height at which this data_vio is operating */ + height_t height; + /* The block map tree for this LBN */ + root_count_t root_index; + /* Whether we hold a page lock */ + bool locked; + /* The key for the lock map */ + u64 key; + /* The queue of waiters for the page this vio is allocating or loading */ + struct wait_queue waiters; + /* The block map tree slots for this LBN */ + struct block_map_tree_slot tree_slots[VDO_BLOCK_MAP_TREE_HEIGHT + 1]; +}; + +struct zoned_pbn { + physical_block_number_t pbn; + enum block_mapping_state state; + struct physical_zone *zone; +}; + +/* + * Where a data_vio is on the compression path; advance_compression_stage() depends on the order of + * this enum. + */ +enum data_vio_compression_stage { + /* A data_vio which has not yet entered the compression path */ + DATA_VIO_PRE_COMPRESSOR, + /* A data_vio which is in the compressor */ + DATA_VIO_COMPRESSING, + /* A data_vio which is blocked in the packer */ + DATA_VIO_PACKING, + /* A data_vio which is no longer on the compression path (and never will be) */ + DATA_VIO_POST_PACKER, +}; + +struct data_vio_compression_status { + enum data_vio_compression_stage stage; + bool may_not_compress; +}; + +struct compression_state { + /* + * The current compression status of this data_vio. This field contains a value which + * consists of a data_vio_compression_stage and a flag indicating whether a request has + * been made to cancel (or prevent) compression for this data_vio. + * + * This field should be accessed through the get_data_vio_compression_status() and + * set_data_vio_compression_status() methods. It should not be accessed directly. + */ + atomic_t status; + + /* The compressed size of this block */ + u16 size; + + /* The packer input or output bin slot which holds the enclosing data_vio */ + slot_number_t slot; + + /* The packer bin to which the enclosing data_vio has been assigned */ + struct packer_bin *bin; + + /* A link in the chain of data_vios which have been packed together */ + struct data_vio *next_in_batch; + + /* A vio which is blocked in the packer while holding a lock this vio needs. */ + struct data_vio *lock_holder; + + /* + * The compressed block used to hold the compressed form of this block and that of any + * other blocks for which this data_vio is the compressed write agent. + */ + struct compressed_block *block; +}; + +/* Fields supporting allocation of data blocks. */ +struct allocation { + /* The physical zone in which to allocate a physical block */ + struct physical_zone *zone; + + /* The block allocated to this vio */ + physical_block_number_t pbn; + + /* + * If non-NULL, the pooled PBN lock held on the allocated block. Must be a write lock until + * the block has been written, after which it will become a read lock. + */ + struct pbn_lock *lock; + + /* The type of write lock to obtain on the allocated block */ + enum pbn_lock_type write_lock_type; + + /* The zone which was the start of the current allocation cycle */ + zone_count_t first_allocation_zone; + + /* Whether this vio should wait for a clean slab */ + bool wait_for_clean_slab; +}; + +struct reference_updater { + enum journal_operation operation; + bool increment; + struct zoned_pbn zpbn; + struct pbn_lock *lock; + struct waiter waiter; +}; + +/* A vio for processing user data requests. */ +struct data_vio { + /* The wait_queue entry structure */ + struct waiter waiter; + + /* The logical block of this request */ + struct lbn_lock logical; + + /* The state for traversing the block map tree */ + struct tree_lock tree_lock; + + /* The current partition address of this block */ + struct zoned_pbn mapped; + + /* The hash of this vio (if not zero) */ + struct uds_record_name record_name; + + /* Used for logging and debugging */ + enum async_operation_number last_async_operation; + + /* The operations to record in the recovery and slab journals */ + struct reference_updater increment_updater; + struct reference_updater decrement_updater; + + u16 read : 1; + u16 write : 1; + u16 fua : 1; + u16 is_zero : 1; + u16 is_trim : 1; + u16 is_partial : 1; + u16 is_duplicate : 1; + u16 first_reference_operation_complete : 1; + u16 downgrade_allocation_lock : 1; + + struct allocation allocation; + + /* + * Whether this vio has received an allocation. This field is examined from threads not in + * the allocation zone. + */ + bool allocation_succeeded; + + /* The new partition address of this block after the vio write completes */ + struct zoned_pbn new_mapped; + + /* The hash zone responsible for the name (NULL if is_zero_block) */ + struct hash_zone *hash_zone; + + /* The lock this vio holds or shares with other vios with the same data */ + struct hash_lock *hash_lock; + + /* All data_vios sharing a hash lock are kept in a list linking these list entries */ + struct list_head hash_lock_entry; + + /* The block number in the partition of the UDS deduplication advice */ + struct zoned_pbn duplicate; + + /* + * The sequence number of the recovery journal block containing the increment entry for + * this vio. + */ + sequence_number_t recovery_sequence_number; + + /* The point in the recovery journal where this write last made an entry */ + struct journal_point recovery_journal_point; + + /* The list of vios in user initiated write requests */ + struct list_head write_entry; + + /* The generation number of the VDO that this vio belongs to */ + sequence_number_t flush_generation; + + /* The completion to use for fetching block map pages for this vio */ + struct vdo_page_completion page_completion; + + /* The user bio that initiated this VIO */ + struct bio *user_bio; + + /* partial block support */ + block_size_t offset; + + /* + * The number of bytes to be discarded. For discards, this field will always be positive, + * whereas for non-discards it will always be 0. Hence it can be used to determine whether + * a data_vio is processing a discard, even after the user_bio has been acknowledged. + */ + u32 remaining_discard; + + struct dedupe_context *dedupe_context; + + /* Fields beyond this point will not be reset when a pooled data_vio is reused. */ + + struct vio vio; + + /* The completion for making reference count decrements */ + struct vdo_completion decrement_completion; + + /* All of the fields necessary for the compression path */ + struct compression_state compression; + + /* A block used as output during compression or uncompression */ + char *scratch_block; + + struct list_head pool_entry; +}; + +static inline struct data_vio *vio_as_data_vio(struct vio *vio) +{ + ASSERT_LOG_ONLY((vio->type == VIO_TYPE_DATA), "vio is a data_vio"); + return container_of(vio, struct data_vio, vio); +} + +static inline struct data_vio *as_data_vio(struct vdo_completion *completion) +{ + return vio_as_data_vio(as_vio(completion)); +} + +static inline struct data_vio *waiter_as_data_vio(struct waiter *waiter) +{ + if (waiter == NULL) + return NULL; + + return container_of(waiter, struct data_vio, waiter); +} + +static inline struct data_vio *data_vio_from_reference_updater(struct reference_updater *updater) +{ + if (updater->increment) + return container_of(updater, struct data_vio, increment_updater); + + return container_of(updater, struct data_vio, decrement_updater); +} + +static inline bool data_vio_has_flush_generation_lock(struct data_vio *data_vio) +{ + return !list_empty(&data_vio->write_entry); +} + +static inline struct vdo *vdo_from_data_vio(struct data_vio *data_vio) +{ + return data_vio->vio.completion.vdo; +} + +static inline bool data_vio_has_allocation(struct data_vio *data_vio) +{ + return (data_vio->allocation.pbn != VDO_ZERO_BLOCK); +} + +struct data_vio_compression_status __must_check +advance_data_vio_compression_stage(struct data_vio *data_vio); +struct data_vio_compression_status __must_check +get_data_vio_compression_status(struct data_vio *data_vio); +bool cancel_data_vio_compression(struct data_vio *data_vio); + +struct data_vio_pool; + +int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, + data_vio_count_t discard_limit, struct data_vio_pool **pool_ptr); +void free_data_vio_pool(struct data_vio_pool *pool); +void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio); +void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion); +void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion); + +void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios); +data_vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool); +int __must_check set_data_vio_pool_discard_limit(struct data_vio_pool *pool, + data_vio_count_t limit); +data_vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool); + +void complete_data_vio(struct vdo_completion *completion); +void handle_data_vio_error(struct vdo_completion *completion); + +static inline void continue_data_vio(struct data_vio *data_vio) +{ + vdo_launch_completion(&data_vio->vio.completion); +} + +/** + * continue_data_vio_with_error() - Set an error code and then continue processing a data_vio. + * + * This will not mask older errors. This function can be called with a success code, but it is more + * efficient to call continue_data_vio() if the caller knows the result was a success. + */ +static inline void continue_data_vio_with_error(struct data_vio *data_vio, int result) +{ + vdo_continue_completion(&data_vio->vio.completion, result); +} + +const char * __must_check get_data_vio_operation_name(struct data_vio *data_vio); + +static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->hash_zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + /* + * It's odd to use the LBN, but converting the record name to hex is a bit clunky for an + * inline, and the LBN better than nothing as an identifier. + */ + ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for logical block %llu on thread %u, should be on hash zone thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, expected); +} + +static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->hash_zone->thread_id); +} + +/** + * launch_data_vio_hash_zone_callback() - Set a callback as a hash zone operation and invoke it + * immediately. + */ +static inline void launch_data_vio_hash_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_hash_zone_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->logical.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for logical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, expected); +} + +static inline void set_data_vio_logical_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->logical.zone->thread_id); +} + +/** + * launch_data_vio_logical_callback() - Set a callback as a logical block operation and invoke it + * immediately. + */ +static inline void launch_data_vio_logical_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_logical_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->allocation.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((expected == thread_id), + "struct data_vio for allocated physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->allocation.pbn, thread_id, + expected); +} + +static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->allocation.zone->thread_id); +} + +/** + * launch_data_vio_allocated_zone_callback() - Set a callback as a physical block operation in a + * data_vio's allocated zone and queue the data_vio and + * invoke it immediately. + */ +static inline void launch_data_vio_allocated_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_allocated_zone_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->duplicate.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for duplicate physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->duplicate.pbn, thread_id, + expected); +} + +static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->duplicate.zone->thread_id); +} + +/** + * launch_data_vio_duplicate_zone_callback() - Set a callback as a physical block operation in a + * data_vio's duplicate zone and queue the data_vio and + * invoke it immediately. + */ +static inline void launch_data_vio_duplicate_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_duplicate_zone_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->mapped.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for mapped physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->mapped.pbn, thread_id, expected); +} + +static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->mapped.zone->thread_id); +} + +static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->new_mapped.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for new_mapped physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->new_mapped.pbn, thread_id, + expected); +} + +static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->new_mapped.zone->thread_id); +} + +static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio) +{ + thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((journal_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on journal thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + journal_thread); +} + +static inline void set_data_vio_journal_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; + + vdo_set_completion_callback(&data_vio->vio.completion, callback, journal_thread); +} + +/** + * launch_data_vio_journal_callback() - Set a callback as a journal operation and invoke it + * immediately. + */ +static inline void launch_data_vio_journal_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_journal_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio) +{ + thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((packer_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on packer thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + packer_thread); +} + +static inline void set_data_vio_packer_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; + + vdo_set_completion_callback(&data_vio->vio.completion, callback, packer_thread); +} + +/** + * launch_data_vio_packer_callback() - Set a callback as a packer operation and invoke it + * immediately. + */ +static inline void launch_data_vio_packer_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_packer_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio) +{ + thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((cpu_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on cpu thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + cpu_thread); +} + +static inline void set_data_vio_cpu_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; + + vdo_set_completion_callback(&data_vio->vio.completion, callback, cpu_thread); +} + +/** + * launch_data_vio_cpu_callback() - Set a callback to run on the CPU queues and invoke it + * immediately. + */ +static inline void launch_data_vio_cpu_callback(struct data_vio *data_vio, + vdo_action_fn callback, + enum vdo_completion_priority priority) +{ + set_data_vio_cpu_callback(data_vio, callback); + vdo_launch_completion_with_priority(&data_vio->vio.completion, priority); +} + +static inline void set_data_vio_bio_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + get_vio_bio_zone_thread_id(&data_vio->vio)); +} + +/** + * launch_data_vio_bio_zone_callback() - Set a callback as a bio zone operation and invoke it + * immediately. + */ +static inline void launch_data_vio_bio_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_bio_zone_callback(data_vio, callback); + vdo_launch_completion_with_priority(&data_vio->vio.completion, + BIO_Q_DATA_PRIORITY); +} + +/** + * launch_data_vio_on_bio_ack_queue() - If the vdo uses a bio_ack queue, set a callback to run on + * it and invoke it immediately, otherwise, just run the + * callback on the current thread. + */ +static inline void launch_data_vio_on_bio_ack_queue(struct data_vio *data_vio, + vdo_action_fn callback) +{ + struct vdo_completion *completion = &data_vio->vio.completion; + struct vdo *vdo = completion->vdo; + + if (!vdo_uses_bio_ack_queue(vdo)) { + callback(completion); + return; + } + + vdo_set_completion_callback(completion, callback, + vdo->thread_config.bio_ack_thread); + vdo_launch_completion_with_priority(completion, BIO_ACK_Q_ACK_PRIORITY); +} + +void data_vio_allocate_data_block(struct data_vio *data_vio, + enum pbn_lock_type write_lock_type, + vdo_action_fn callback, vdo_action_fn error_handler); + +void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset); + +int __must_check uncompress_data_vio(struct data_vio *data_vio, + enum block_mapping_state mapping_state, + char *buffer); + +void update_metadata_for_data_vio_write(struct data_vio *data_vio, + struct pbn_lock *lock); +void write_data_vio(struct data_vio *data_vio); +void launch_compress_data_vio(struct data_vio *data_vio); +void continue_data_vio_with_block_map_slot(struct vdo_completion *completion); + +#endif /* DATA_VIO_H */ From 062401f0de3540322db559946249f639c6a211ec Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:00:21 -0500 Subject: [PATCH 0568/1038] dm vdo: add flush support This patch adds support for handling incoming flush and/or FUA bios. Each such bio is assigned to a struct vdo_flush. These are allocated as needed, but there is always one kept in reserve in case allocations fail. In the event of an allocation failure, bios may need to wait for an outstanding flush to complete. The logical address space is partitioned into logical zones, each handled by its own thread. Each zone keeps a list of all data_vios handling write requests for logical addresses in that zone. When a flush bio is processed, each logical zone is informed of the flush. When all of the writes which are in progress at the time of the notification have completed in all zones, the flush bio is then allowed to complete. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/flush.c | 558 +++++++++++++++++++++++++++++++ drivers/md/dm-vdo/flush.h | 44 +++ drivers/md/dm-vdo/logical-zone.c | 375 +++++++++++++++++++++ drivers/md/dm-vdo/logical-zone.h | 89 +++++ 4 files changed, 1066 insertions(+) create mode 100644 drivers/md/dm-vdo/flush.c create mode 100644 drivers/md/dm-vdo/flush.h create mode 100644 drivers/md/dm-vdo/logical-zone.c create mode 100644 drivers/md/dm-vdo/logical-zone.h diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c new file mode 100644 index 000000000000..a99607e23fb0 --- /dev/null +++ b/drivers/md/dm-vdo/flush.c @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "flush.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "admin-state.h" +#include "completion.h" +#include "io-submitter.h" +#include "logical-zone.h" +#include "slab-depot.h" +#include "types.h" +#include "vdo.h" + +struct flusher { + struct vdo_completion completion; + /** The vdo to which this flusher belongs */ + struct vdo *vdo; + /** The administrative state of the flusher */ + struct admin_state state; + /** The current flush generation of the vdo */ + sequence_number_t flush_generation; + /** The first unacknowledged flush generation */ + sequence_number_t first_unacknowledged_generation; + /** The queue of flush requests waiting to notify other threads */ + struct wait_queue notifiers; + /** The queue of flush requests waiting for VIOs to complete */ + struct wait_queue pending_flushes; + /** The flush generation for which notifications are being sent */ + sequence_number_t notify_generation; + /** The logical zone to notify next */ + struct logical_zone *logical_zone_to_notify; + /** The ID of the thread on which flush requests should be made */ + thread_id_t thread_id; + /** The pool of flush requests */ + mempool_t *flush_pool; + /** Bios waiting for a flush request to become available */ + struct bio_list waiting_flush_bios; + /** The lock to protect the previous fields */ + spinlock_t lock; + /** The rotor for selecting the bio queue for submitting flush bios */ + zone_count_t bio_queue_rotor; + /** The number of flushes submitted to the current bio queue */ + int flush_count; +}; + +/** + * assert_on_flusher_thread() - Check that we are on the flusher thread. + * @flusher: The flusher. + * @caller: The function which is asserting. + */ +static inline void assert_on_flusher_thread(struct flusher *flusher, const char *caller) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == flusher->thread_id), + "%s() called from flusher thread", caller); +} + +/** + * as_flusher() - Convert a generic vdo_completion to a flusher. + * @completion: The completion to convert. + * + * Return: The completion as a flusher. + */ +static struct flusher *as_flusher(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_FLUSH_NOTIFICATION_COMPLETION); + return container_of(completion, struct flusher, completion); +} + +/** + * completion_as_vdo_flush() - Convert a generic vdo_completion to a vdo_flush. + * @completion: The completion to convert. + * + * Return: The completion as a vdo_flush. + */ +static inline struct vdo_flush *completion_as_vdo_flush(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_FLUSH_COMPLETION); + return container_of(completion, struct vdo_flush, completion); +} + +/** + * waiter_as_flush() - Convert a vdo_flush's generic wait queue entry back to the vdo_flush. + * @waiter: The wait queue entry to convert. + * + * Return: The wait queue entry as a vdo_flush. + */ +static struct vdo_flush *waiter_as_flush(struct waiter *waiter) +{ + return container_of(waiter, struct vdo_flush, waiter); +} + +static void *allocate_flush(gfp_t gfp_mask, void *pool_data) +{ + struct vdo_flush *flush; + + if ((gfp_mask & GFP_NOWAIT) == GFP_NOWAIT) { + flush = uds_allocate_memory_nowait(sizeof(struct vdo_flush), __func__); + } else { + int result = uds_allocate(1, struct vdo_flush, __func__, &flush); + + if (result != VDO_SUCCESS) + uds_log_error_strerror(result, "failed to allocate spare flush"); + } + + if (flush != NULL) { + struct flusher *flusher = pool_data; + + vdo_initialize_completion(&flush->completion, flusher->vdo, + VDO_FLUSH_COMPLETION); + } + + return flush; +} + +static void free_flush(void *element, void *pool_data __always_unused) +{ + uds_free(element); +} + +/** + * vdo_make_flusher() - Make a flusher for a vdo. + * @vdo: The vdo which owns the flusher. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make_flusher(struct vdo *vdo) +{ + int result = uds_allocate(1, struct flusher, __func__, &vdo->flusher); + + if (result != VDO_SUCCESS) + return result; + + vdo->flusher->vdo = vdo; + vdo->flusher->thread_id = vdo->thread_config.packer_thread; + vdo_set_admin_state_code(&vdo->flusher->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + vdo_initialize_completion(&vdo->flusher->completion, vdo, + VDO_FLUSH_NOTIFICATION_COMPLETION); + + spin_lock_init(&vdo->flusher->lock); + bio_list_init(&vdo->flusher->waiting_flush_bios); + vdo->flusher->flush_pool = mempool_create(1, allocate_flush, free_flush, + vdo->flusher); + return ((vdo->flusher->flush_pool == NULL) ? -ENOMEM : VDO_SUCCESS); +} + +/** + * vdo_free_flusher() - Free a flusher. + * @flusher: The flusher to free. + */ +void vdo_free_flusher(struct flusher *flusher) +{ + if (flusher == NULL) + return; + + if (flusher->flush_pool != NULL) + mempool_destroy(uds_forget(flusher->flush_pool)); + uds_free(flusher); +} + +/** + * vdo_get_flusher_thread_id() - Get the ID of the thread on which flusher functions should be + * called. + * @flusher: The flusher to query. + * + * Return: The ID of the thread which handles the flusher. + */ +thread_id_t vdo_get_flusher_thread_id(struct flusher *flusher) +{ + return flusher->thread_id; +} + +static void notify_flush(struct flusher *flusher); +static void vdo_complete_flush(struct vdo_flush *flush); + +/** + * finish_notification() - Finish the notification process. + * @completion: The flusher completion. + * + * Finishes the notification process by checking if any flushes have completed and then starting + * the notification of the next flush request if one came in while the current notification was in + * progress. This callback is registered in flush_packer_callback(). + */ +static void finish_notification(struct vdo_completion *completion) +{ + struct flusher *flusher = as_flusher(completion); + + assert_on_flusher_thread(flusher, __func__); + + vdo_enqueue_waiter(&flusher->pending_flushes, + vdo_dequeue_next_waiter(&flusher->notifiers)); + vdo_complete_flushes(flusher); + if (vdo_has_waiters(&flusher->notifiers)) + notify_flush(flusher); +} + +/** + * flush_packer_callback() - Flush the packer. + * @completion: The flusher completion. + * + * Flushes the packer now that all of the logical and physical zones have been notified of the new + * flush request. This callback is registered in increment_generation(). + */ +static void flush_packer_callback(struct vdo_completion *completion) +{ + struct flusher *flusher = as_flusher(completion); + + vdo_increment_packer_flush_generation(flusher->vdo->packer); + vdo_launch_completion_callback(completion, finish_notification, + flusher->thread_id); +} + +/** + * increment_generation() - Increment the flush generation in a logical zone. + * @completion: The flusher as a completion. + * + * If there are more logical zones, go on to the next one, otherwise, prepare the physical zones. + * This callback is registered both in notify_flush() and in itself. + */ +static void increment_generation(struct vdo_completion *completion) +{ + struct flusher *flusher = as_flusher(completion); + struct logical_zone *zone = flusher->logical_zone_to_notify; + + vdo_increment_logical_zone_flush_generation(zone, flusher->notify_generation); + if (zone->next == NULL) { + vdo_launch_completion_callback(completion, flush_packer_callback, + flusher->thread_id); + return; + } + + flusher->logical_zone_to_notify = zone->next; + vdo_launch_completion_callback(completion, increment_generation, + flusher->logical_zone_to_notify->thread_id); +} + +/** + * notify_flush() - Launch a flush notification. + * @flusher: The flusher doing the notification. + */ +static void notify_flush(struct flusher *flusher) +{ + struct vdo_flush *flush = waiter_as_flush(vdo_get_first_waiter(&flusher->notifiers)); + + flusher->notify_generation = flush->flush_generation; + flusher->logical_zone_to_notify = &flusher->vdo->logical_zones->zones[0]; + flusher->completion.requeue = true; + vdo_launch_completion_callback(&flusher->completion, increment_generation, + flusher->logical_zone_to_notify->thread_id); +} + +/** + * flush_vdo() - Start processing a flush request. + * @completion: A flush request (as a vdo_completion) + * + * This callback is registered in launch_flush(). + */ +static void flush_vdo(struct vdo_completion *completion) +{ + struct vdo_flush *flush = completion_as_vdo_flush(completion); + struct flusher *flusher = completion->vdo->flusher; + bool may_notify; + int result; + + assert_on_flusher_thread(flusher, __func__); + result = ASSERT(vdo_is_state_normal(&flusher->state), + "flusher is in normal operation"); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(flusher->vdo, result); + vdo_complete_flush(flush); + return; + } + + flush->flush_generation = flusher->flush_generation++; + may_notify = !vdo_has_waiters(&flusher->notifiers); + vdo_enqueue_waiter(&flusher->notifiers, &flush->waiter); + if (may_notify) + notify_flush(flusher); +} + +/** + * check_for_drain_complete() - Check whether the flusher has drained. + * @flusher: The flusher. + */ +static void check_for_drain_complete(struct flusher *flusher) +{ + bool drained; + + if (!vdo_is_state_draining(&flusher->state) || vdo_has_waiters(&flusher->pending_flushes)) + return; + + spin_lock(&flusher->lock); + drained = bio_list_empty(&flusher->waiting_flush_bios); + spin_unlock(&flusher->lock); + + if (drained) + vdo_finish_draining(&flusher->state); +} + +/** + * vdo_complete_flushes() - Attempt to complete any flushes which might have finished. + * @flusher: The flusher. + */ +void vdo_complete_flushes(struct flusher *flusher) +{ + sequence_number_t oldest_active_generation = U64_MAX; + struct logical_zone *zone; + + assert_on_flusher_thread(flusher, __func__); + + for (zone = &flusher->vdo->logical_zones->zones[0]; zone != NULL; zone = zone->next) + oldest_active_generation = + min(oldest_active_generation, + READ_ONCE(zone->oldest_active_generation)); + + while (vdo_has_waiters(&flusher->pending_flushes)) { + struct vdo_flush *flush = + waiter_as_flush(vdo_get_first_waiter(&flusher->pending_flushes)); + + if (flush->flush_generation >= oldest_active_generation) + return; + + ASSERT_LOG_ONLY((flush->flush_generation == + flusher->first_unacknowledged_generation), + "acknowledged next expected flush, %llu, was: %llu", + (unsigned long long) flusher->first_unacknowledged_generation, + (unsigned long long) flush->flush_generation); + vdo_dequeue_next_waiter(&flusher->pending_flushes); + vdo_complete_flush(flush); + flusher->first_unacknowledged_generation++; + } + + check_for_drain_complete(flusher); +} + +/** + * vdo_dump_flusher() - Dump the flusher, in a thread-unsafe fashion. + * @flusher: The flusher. + */ +void vdo_dump_flusher(const struct flusher *flusher) +{ + uds_log_info("struct flusher"); + uds_log_info(" flush_generation=%llu first_unacknowledged_generation=%llu", + (unsigned long long) flusher->flush_generation, + (unsigned long long) flusher->first_unacknowledged_generation); + uds_log_info(" notifiers queue is %s; pending_flushes queue is %s", + (vdo_has_waiters(&flusher->notifiers) ? "not empty" : "empty"), + (vdo_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty")); +} + +/** + * initialize_flush() - Initialize a vdo_flush structure. + * @flush: The flush to initialize. + * @vdo: The vdo being flushed. + * + * Initializes a vdo_flush structure, transferring all the bios in the flusher's waiting_flush_bios + * list to it. The caller MUST already hold the lock. + */ +static void initialize_flush(struct vdo_flush *flush, struct vdo *vdo) +{ + bio_list_init(&flush->bios); + bio_list_merge(&flush->bios, &vdo->flusher->waiting_flush_bios); + bio_list_init(&vdo->flusher->waiting_flush_bios); +} + +static void launch_flush(struct vdo_flush *flush) +{ + struct vdo_completion *completion = &flush->completion; + + vdo_prepare_completion(completion, flush_vdo, flush_vdo, + completion->vdo->thread_config.packer_thread, NULL); + vdo_enqueue_completion(completion, VDO_DEFAULT_Q_FLUSH_PRIORITY); +} + +/** + * vdo_launch_flush() - Function called to start processing a flush request. + * @vdo: The vdo. + * @bio: The bio containing an empty flush request. + * + * This is called when we receive an empty flush bio from the block layer, and before acknowledging + * a non-empty bio with the FUA flag set. + */ +void vdo_launch_flush(struct vdo *vdo, struct bio *bio) +{ + /* + * Try to allocate a vdo_flush to represent the flush request. If the allocation fails, + * we'll deal with it later. + */ + struct vdo_flush *flush = mempool_alloc(vdo->flusher->flush_pool, GFP_NOWAIT); + struct flusher *flusher = vdo->flusher; + const struct admin_state_code *code = vdo_get_admin_state_code(&flusher->state); + + ASSERT_LOG_ONLY(!code->quiescent, "Flushing not allowed in state %s", + code->name); + + spin_lock(&flusher->lock); + + /* We have a new bio to start. Add it to the list. */ + bio_list_add(&flusher->waiting_flush_bios, bio); + + if (flush == NULL) { + spin_unlock(&flusher->lock); + return; + } + + /* We have flushes to start. Capture them in the vdo_flush structure. */ + initialize_flush(flush, vdo); + spin_unlock(&flusher->lock); + + /* Finish launching the flushes. */ + launch_flush(flush); +} + +/** + * release_flush() - Release a vdo_flush structure that has completed its work. + * @flush: The completed flush structure to re-use or free. + * + * If there are any pending flush requests whose vdo_flush allocation failed, they will be launched + * by immediately re-using the released vdo_flush. If there is no spare vdo_flush, the released + * structure will become the spare. Otherwise, the vdo_flush will be freed. + */ +static void release_flush(struct vdo_flush *flush) +{ + bool relaunch_flush; + struct flusher *flusher = flush->completion.vdo->flusher; + + spin_lock(&flusher->lock); + if (bio_list_empty(&flusher->waiting_flush_bios)) { + relaunch_flush = false; + } else { + /* We have flushes to start. Capture them in a flush request. */ + initialize_flush(flush, flusher->vdo); + relaunch_flush = true; + } + spin_unlock(&flusher->lock); + + if (relaunch_flush) { + /* Finish launching the flushes. */ + launch_flush(flush); + return; + } + + mempool_free(flush, flusher->flush_pool); +} + +/** + * vdo_complete_flush_callback() - Function called to complete and free a flush request, registered + * in vdo_complete_flush(). + * @completion: The flush request. + */ +static void vdo_complete_flush_callback(struct vdo_completion *completion) +{ + struct vdo_flush *flush = completion_as_vdo_flush(completion); + struct vdo *vdo = completion->vdo; + struct bio *bio; + + while ((bio = bio_list_pop(&flush->bios)) != NULL) { + /* + * We're not acknowledging this bio now, but we'll never touch it again, so this is + * the last chance to account for it. + */ + vdo_count_bios(&vdo->stats.bios_acknowledged, bio); + + /* Update the device, and send it on down... */ + bio_set_dev(bio, vdo_get_backing_device(vdo)); + atomic64_inc(&vdo->stats.flush_out); + submit_bio_noacct(bio); + } + + + /* + * Release the flush structure, freeing it, re-using it as the spare, or using it to launch + * any flushes that had to wait when allocations failed. + */ + release_flush(flush); +} + +/** + * select_bio_queue() - Select the bio queue on which to finish a flush request. + * @flusher: The flusher finishing the request. + */ +static thread_id_t select_bio_queue(struct flusher *flusher) +{ + struct vdo *vdo = flusher->vdo; + zone_count_t bio_threads = flusher->vdo->thread_config.bio_thread_count; + int interval; + + if (bio_threads == 1) + return vdo->thread_config.bio_threads[0]; + + interval = vdo->device_config->thread_counts.bio_rotation_interval; + if (flusher->flush_count == interval) { + flusher->flush_count = 1; + flusher->bio_queue_rotor = ((flusher->bio_queue_rotor + 1) % bio_threads); + } else { + flusher->flush_count++; + } + + return vdo->thread_config.bio_threads[flusher->bio_queue_rotor]; +} + +/** + * vdo_complete_flush() - Complete and free a vdo flush request. + * @flush: The flush request. + */ +static void vdo_complete_flush(struct vdo_flush *flush) +{ + struct vdo_completion *completion = &flush->completion; + + vdo_prepare_completion(completion, vdo_complete_flush_callback, + vdo_complete_flush_callback, + select_bio_queue(completion->vdo->flusher), NULL); + vdo_enqueue_completion(completion, BIO_Q_FLUSH_PRIORITY); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct flusher, state)); +} + +/** + * vdo_drain_flusher() - Drain the flusher. + * @flusher: The flusher to drain. + * @completion: The completion to finish when the flusher has drained. + * + * Drains the flusher by preventing any more VIOs from entering the flusher and then flushing. The + * flusher will be left in the suspended state. + */ +void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion) +{ + assert_on_flusher_thread(flusher, __func__); + vdo_start_draining(&flusher->state, VDO_ADMIN_STATE_SUSPENDING, completion, + initiate_drain); +} + +/** + * vdo_resume_flusher() - Resume a flusher which has been suspended. + * @flusher: The flusher to resume. + * @parent: The completion to finish when the flusher has resumed. + */ +void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent) +{ + assert_on_flusher_thread(flusher, __func__); + vdo_continue_completion(parent, vdo_resume_if_quiescent(&flusher->state)); +} diff --git a/drivers/md/dm-vdo/flush.h b/drivers/md/dm-vdo/flush.h new file mode 100644 index 000000000000..4d40908462bb --- /dev/null +++ b/drivers/md/dm-vdo/flush.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_FLUSH_H +#define VDO_FLUSH_H + +#include "funnel-workqueue.h" +#include "types.h" +#include "vio.h" +#include "wait-queue.h" + +/* A marker for tracking which journal entries are affected by a flush request. */ +struct vdo_flush { + /* The completion for enqueueing this flush request. */ + struct vdo_completion completion; + /* The flush bios covered by this request */ + struct bio_list bios; + /* The wait queue entry for this flush */ + struct waiter waiter; + /* Which flush this struct represents */ + sequence_number_t flush_generation; +}; + +struct flusher; + +int __must_check vdo_make_flusher(struct vdo *vdo); + +void vdo_free_flusher(struct flusher *flusher); + +thread_id_t __must_check vdo_get_flusher_thread_id(struct flusher *flusher); + +void vdo_complete_flushes(struct flusher *flusher); + +void vdo_dump_flusher(const struct flusher *flusher); + +void vdo_launch_flush(struct vdo *vdo, struct bio *bio); + +void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion); + +void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent); + +#endif /* VDO_FLUSH_H */ diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c new file mode 100644 index 000000000000..7140e202798f --- /dev/null +++ b/drivers/md/dm-vdo/logical-zone.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "logical-zone.h" + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "flush.h" +#include "int-map.h" +#include "physical-zone.h" +#include "vdo.h" + +enum { + ALLOCATIONS_PER_ZONE = 128, +}; + +/** + * as_logical_zone() - Convert a generic vdo_completion to a logical_zone. + * @completion: The completion to convert. + * + * Return: The completion as a logical_zone. + */ +static struct logical_zone *as_logical_zone(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_GENERATION_FLUSHED_COMPLETION); + return container_of(completion, struct logical_zone, completion); +} + +/* get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */ +static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number) +{ + struct logical_zones *zones = context; + + return zones->zones[zone_number].thread_id; +} + +/** + * initialize_zone() - Initialize a logical zone. + * @zones: The logical_zones to which this zone belongs. + * @zone_number: The logical_zone's index. + */ +static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number) +{ + int result; + struct vdo *vdo = zones->vdo; + struct logical_zone *zone = &zones->zones[zone_number]; + zone_count_t allocation_zone_number; + + result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->lbn_operations); + if (result != VDO_SUCCESS) + return result; + + if (zone_number < vdo->thread_config.logical_zone_count - 1) + zone->next = &zones->zones[zone_number + 1]; + + vdo_initialize_completion(&zone->completion, vdo, + VDO_GENERATION_FLUSHED_COMPLETION); + zone->zones = zones; + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.logical_threads[zone_number]; + zone->block_map_zone = &vdo->block_map->zones[zone_number]; + INIT_LIST_HEAD(&zone->write_vios); + vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + + allocation_zone_number = zone->thread_id % vdo->thread_config.physical_zone_count; + zone->allocation_zone = &vdo->physical_zones->zones[allocation_zone_number]; + + return vdo_make_default_thread(vdo, zone->thread_id); +} + +/** + * vdo_make_logical_zones() - Create a set of logical zones. + * @vdo: The vdo to which the zones will belong. + * @zones_ptr: A pointer to hold the new zones. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr) +{ + struct logical_zones *zones; + int result; + zone_count_t zone; + zone_count_t zone_count = vdo->thread_config.logical_zone_count; + + if (zone_count == 0) + return VDO_SUCCESS; + + result = uds_allocate_extended(struct logical_zones, zone_count, + struct logical_zone, __func__, &zones); + if (result != VDO_SUCCESS) + return result; + + zones->vdo = vdo; + zones->zone_count = zone_count; + for (zone = 0; zone < zone_count; zone++) { + result = initialize_zone(zones, zone); + if (result != VDO_SUCCESS) { + vdo_free_logical_zones(zones); + return result; + } + } + + result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone, + vdo->thread_config.admin_thread, zones, NULL, + vdo, &zones->manager); + if (result != VDO_SUCCESS) { + vdo_free_logical_zones(zones); + return result; + } + + *zones_ptr = zones; + return VDO_SUCCESS; +} + +/** + * vdo_free_logical_zones() - Free a set of logical zones. + * @zones: The set of zones to free. + */ +void vdo_free_logical_zones(struct logical_zones *zones) +{ + zone_count_t index; + + if (zones == NULL) + return; + + uds_free(uds_forget(zones->manager)); + + for (index = 0; index < zones->zone_count; index++) + vdo_free_int_map(uds_forget(zones->zones[index].lbn_operations)); + + uds_free(zones); +} + +static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), + "%s() called on correct thread", what); +} + +/** + * check_for_drain_complete() - Check whether this zone has drained. + * @zone: The zone to check. + */ +static void check_for_drain_complete(struct logical_zone *zone) +{ + if (!vdo_is_state_draining(&zone->state) || zone->notifying || + !list_empty(&zone->write_vios)) + return; + + vdo_finish_draining(&zone->state); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct logical_zone, state)); +} + +/** + * drain_logical_zone() - Drain a logical zone. + * + * Implements vdo_zone_action_fn. + */ +static void drain_logical_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct logical_zones *zones = context; + + vdo_start_draining(&zones->zones[zone_number].state, + vdo_get_current_manager_operation(zones->manager), parent, + initiate_drain); +} + +void vdo_drain_logical_zones(struct logical_zones *zones, + const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + vdo_schedule_operation(zones->manager, operation, NULL, drain_logical_zone, NULL, + parent); +} + +/** + * resume_logical_zone() - Resume a logical zone. + * + * Implements vdo_zone_action_fn. + */ +static void resume_logical_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct logical_zone *zone = &(((struct logical_zones *) context)->zones[zone_number]); + + vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); +} + +/** + * vdo_resume_logical_zones() - Resume a set of logical zones. + * @zones: The logical zones to resume. + * @parent: The object to notify when the zones have resumed. + */ +void vdo_resume_logical_zones(struct logical_zones *zones, struct vdo_completion *parent) +{ + vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, NULL, + resume_logical_zone, NULL, parent); +} + +/** + * update_oldest_active_generation() - Update the oldest active generation. + * @zone: The zone. + * + * Return: true if the oldest active generation has changed. + */ +static bool update_oldest_active_generation(struct logical_zone *zone) +{ + struct data_vio *data_vio = + list_first_entry_or_null(&zone->write_vios, struct data_vio, + write_entry); + sequence_number_t oldest = + (data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation; + + if (oldest == zone->oldest_active_generation) + return false; + + WRITE_ONCE(zone->oldest_active_generation, oldest); + return true; +} + +/** + * vdo_increment_logical_zone_flush_generation() - Increment the flush generation in a logical + * zone. + * @zone: The logical zone. + * @expected_generation: The expected value of the flush generation before the increment. + */ +void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, + sequence_number_t expected_generation) +{ + assert_on_zone_thread(zone, __func__); + ASSERT_LOG_ONLY((zone->flush_generation == expected_generation), + "logical zone %u flush generation %llu should be %llu before increment", + zone->zone_number, (unsigned long long) zone->flush_generation, + (unsigned long long) expected_generation); + + zone->flush_generation++; + zone->ios_in_flush_generation = 0; + update_oldest_active_generation(zone); +} + +/** + * vdo_acquire_flush_generation_lock() - Acquire the shared lock on a flush generation by a write + * data_vio. + * @data_vio: The data_vio. + */ +void vdo_acquire_flush_generation_lock(struct data_vio *data_vio) +{ + struct logical_zone *zone = data_vio->logical.zone; + + assert_on_zone_thread(zone, __func__); + ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal"); + + data_vio->flush_generation = zone->flush_generation; + list_add_tail(&data_vio->write_entry, &zone->write_vios); + zone->ios_in_flush_generation++; +} + +static void attempt_generation_complete_notification(struct vdo_completion *completion); + +/** + * notify_flusher() - Notify the flush that at least one generation no longer has active VIOs. + * @completion: The zone completion. + * + * This callback is registered in attempt_generation_complete_notification(). + */ +static void notify_flusher(struct vdo_completion *completion) +{ + struct logical_zone *zone = as_logical_zone(completion); + + vdo_complete_flushes(zone->zones->vdo->flusher); + vdo_launch_completion_callback(completion, + attempt_generation_complete_notification, + zone->thread_id); +} + +/** + * void attempt_generation_complete_notification() - Notify the flusher if some generation no + * longer has active VIOs. + * @completion: The zone completion. + */ +static void attempt_generation_complete_notification(struct vdo_completion *completion) +{ + struct logical_zone *zone = as_logical_zone(completion); + + assert_on_zone_thread(zone, __func__); + if (zone->oldest_active_generation <= zone->notification_generation) { + zone->notifying = false; + check_for_drain_complete(zone); + return; + } + + zone->notifying = true; + zone->notification_generation = zone->oldest_active_generation; + vdo_launch_completion_callback(&zone->completion, notify_flusher, + vdo_get_flusher_thread_id(zone->zones->vdo->flusher)); +} + +/** + * vdo_release_flush_generation_lock() - Release the shared lock on a flush generation held by a + * write data_vio. + * @data_vio: The data_vio whose lock is to be released. + * + * If there are pending flushes, and this data_vio completes the oldest generation active in this + * zone, an attempt will be made to finish any flushes which may now be complete. + */ +void vdo_release_flush_generation_lock(struct data_vio *data_vio) +{ + struct logical_zone *zone = data_vio->logical.zone; + + assert_on_zone_thread(zone, __func__); + + if (!data_vio_has_flush_generation_lock(data_vio)) + return; + + list_del_init(&data_vio->write_entry); + ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation), + "data_vio releasing lock on generation %llu is not older than oldest active generation %llu", + (unsigned long long) data_vio->flush_generation, + (unsigned long long) zone->oldest_active_generation); + + if (!update_oldest_active_generation(zone) || zone->notifying) + return; + + attempt_generation_complete_notification(&zone->completion); +} + +struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone) +{ + if (zone->allocation_count == ALLOCATIONS_PER_ZONE) { + zone->allocation_count = 0; + zone->allocation_zone = zone->allocation_zone->next; + } + + zone->allocation_count++; + return zone->allocation_zone; +} + +/** + * vdo_dump_logical_zone() - Dump information about a logical zone to the log for debugging. + * @zone: The zone to dump + * + * Context: the information is dumped in a thread-unsafe fashion. + * + */ +void vdo_dump_logical_zone(const struct logical_zone *zone) +{ + uds_log_info("logical_zone %u", zone->zone_number); + uds_log_info(" flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu", + (unsigned long long) READ_ONCE(zone->flush_generation), + (unsigned long long) READ_ONCE(zone->oldest_active_generation), + (unsigned long long) READ_ONCE(zone->notification_generation), + uds_bool_to_string(READ_ONCE(zone->notifying)), + (unsigned long long) READ_ONCE(zone->ios_in_flush_generation)); +} diff --git a/drivers/md/dm-vdo/logical-zone.h b/drivers/md/dm-vdo/logical-zone.h new file mode 100644 index 000000000000..1b666c84a193 --- /dev/null +++ b/drivers/md/dm-vdo/logical-zone.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_LOGICAL_ZONE_H +#define VDO_LOGICAL_ZONE_H + +#include + +#include "admin-state.h" +#include "int-map.h" +#include "types.h" + +struct physical_zone; + +struct logical_zone { + /* The completion for flush notifications */ + struct vdo_completion completion; + /* The owner of this zone */ + struct logical_zones *zones; + /* Which logical zone this is */ + zone_count_t zone_number; + /* The thread id for this zone */ + thread_id_t thread_id; + /* In progress operations keyed by LBN */ + struct int_map *lbn_operations; + /* The logical to physical map */ + struct block_map_zone *block_map_zone; + /* The current flush generation */ + sequence_number_t flush_generation; + /* + * The oldest active generation in this zone. This is mutated only on the logical zone + * thread but is queried from the flusher thread. + */ + sequence_number_t oldest_active_generation; + /* The number of IOs in the current flush generation */ + block_count_t ios_in_flush_generation; + /* The youngest generation of the current notification */ + sequence_number_t notification_generation; + /* Whether a notification is in progress */ + bool notifying; + /* The queue of active data write VIOs */ + struct list_head write_vios; + /* The administrative state of the zone */ + struct admin_state state; + /* The physical zone from which to allocate */ + struct physical_zone *allocation_zone; + /* The number of allocations done from the current allocation_zone */ + block_count_t allocation_count; + /* The next zone */ + struct logical_zone *next; +}; + +struct logical_zones { + /* The vdo whose zones these are */ + struct vdo *vdo; + /* The manager for administrative actions */ + struct action_manager *manager; + /* The number of zones */ + zone_count_t zone_count; + /* The logical zones themselves */ + struct logical_zone zones[]; +}; + +int __must_check vdo_make_logical_zones(struct vdo *vdo, + struct logical_zones **zones_ptr); + +void vdo_free_logical_zones(struct logical_zones *zones); + +void vdo_drain_logical_zones(struct logical_zones *zones, + const struct admin_state_code *operation, + struct vdo_completion *completion); + +void vdo_resume_logical_zones(struct logical_zones *zones, + struct vdo_completion *parent); + +void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, + sequence_number_t expected_generation); + +void vdo_acquire_flush_generation_lock(struct data_vio *data_vio); + +void vdo_release_flush_generation_lock(struct data_vio *data_vio); + +struct physical_zone * __must_check vdo_get_next_allocation_zone(struct logical_zone *zone); + +void vdo_dump_logical_zone(const struct logical_zone *zone); + +#endif /* VDO_LOGICAL_ZONE_H */ From c65bfacedc3e9d394c06dacd7b9c70e6bdf41702 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:00:53 -0500 Subject: [PATCH 0569/1038] dm vdo: add the vdo io_submitter The io_submitter handles bio submission from vdo data store to the storage below. It will merge bios when possible. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/io-submitter.c | 480 +++++++++++++++++++++++++++++++ drivers/md/dm-vdo/io-submitter.h | 47 +++ 2 files changed, 527 insertions(+) create mode 100644 drivers/md/dm-vdo/io-submitter.c create mode 100644 drivers/md/dm-vdo/io-submitter.h diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c new file mode 100644 index 000000000000..ea27043f06e1 --- /dev/null +++ b/drivers/md/dm-vdo/io-submitter.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "io-submitter.h" + +#include +#include +#include + +#include "memory-alloc.h" +#include "permassert.h" + +#include "data-vio.h" +#include "logger.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" + +/* + * Submission of bio operations to the underlying storage device will go through a separate work + * queue thread (or more than one) to prevent blocking in other threads if the storage device has a + * full queue. The plug structure allows that thread to do better batching of requests to make the + * I/O more efficient. + * + * When multiple worker threads are used, a thread is chosen for a I/O operation submission based + * on the PBN, so a given PBN will consistently wind up on the same thread. Flush operations are + * assigned round-robin. + * + * The map (protected by the mutex) collects pending I/O operations so that the worker thread can + * reorder them to try to encourage I/O request merging in the request queue underneath. + */ +struct bio_queue_data { + struct vdo_work_queue *queue; + struct blk_plug plug; + struct int_map *map; + struct mutex lock; + unsigned int queue_number; +}; + +struct io_submitter { + unsigned int num_bio_queues_used; + unsigned int bio_queue_rotation_interval; + struct bio_queue_data bio_queue_data[]; +}; + +static void start_bio_queue(void *ptr) +{ + struct bio_queue_data *bio_queue_data = ptr; + + blk_start_plug(&bio_queue_data->plug); +} + +static void finish_bio_queue(void *ptr) +{ + struct bio_queue_data *bio_queue_data = ptr; + + blk_finish_plug(&bio_queue_data->plug); +} + +static const struct vdo_work_queue_type bio_queue_type = { + .start = start_bio_queue, + .finish = finish_bio_queue, + .max_priority = BIO_Q_MAX_PRIORITY, + .default_priority = BIO_Q_DATA_PRIORITY, +}; + +/** + * count_all_bios() - Determine which bio counter to use. + * @vio: The vio associated with the bio. + * @bio: The bio to count. + */ +static void count_all_bios(struct vio *vio, struct bio *bio) +{ + struct atomic_statistics *stats = &vio->completion.vdo->stats; + + if (is_data_vio(vio)) { + vdo_count_bios(&stats->bios_out, bio); + return; + } + + vdo_count_bios(&stats->bios_meta, bio); + if (vio->type == VIO_TYPE_RECOVERY_JOURNAL) + vdo_count_bios(&stats->bios_journal, bio); + else if (vio->type == VIO_TYPE_BLOCK_MAP) + vdo_count_bios(&stats->bios_page_cache, bio); +} + +/** + * assert_in_bio_zone() - Assert that a vio is in the correct bio zone and not in interrupt + * context. + * @vio: The vio to check. + */ +static void assert_in_bio_zone(struct vio *vio) +{ + ASSERT_LOG_ONLY(!in_interrupt(), "not in interrupt context"); + assert_vio_in_bio_zone(vio); +} + +/** + * send_bio_to_device() - Update stats and tracing info, then submit the supplied bio to the OS for + * processing. + * @vio: The vio associated with the bio. + * @bio: The bio to submit to the OS. + */ +static void send_bio_to_device(struct vio *vio, struct bio *bio) +{ + struct vdo *vdo = vio->completion.vdo; + + assert_in_bio_zone(vio); + atomic64_inc(&vdo->stats.bios_submitted); + count_all_bios(vio, bio); + bio_set_dev(bio, vdo_get_backing_device(vdo)); + submit_bio_noacct(bio); +} + +static sector_t get_bio_sector(struct bio *bio) +{ + return bio->bi_iter.bi_sector; +} + +/** + * process_vio_io() - Submits a vio's bio to the underlying block device. May block if the device + * is busy. This callback should be used by vios which did not attempt to merge. + */ +void process_vio_io(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + + send_bio_to_device(vio, vio->bio); +} + +/** + * get_bio_list() - Extract the list of bios to submit from a vio. + * @vio: The vio submitting I/O. + * + * The list will always contain at least one entry (the bio for the vio on which it is called), but + * other bios may have been merged with it as well. + * + * Return: bio The head of the bio list to submit. + */ +static struct bio *get_bio_list(struct vio *vio) +{ + struct bio *bio; + struct io_submitter *submitter = vio->completion.vdo->io_submitter; + struct bio_queue_data *bio_queue_data = &(submitter->bio_queue_data[vio->bio_zone]); + + assert_in_bio_zone(vio); + + mutex_lock(&bio_queue_data->lock); + vdo_int_map_remove(bio_queue_data->map, get_bio_sector(vio->bios_merged.head)); + vdo_int_map_remove(bio_queue_data->map, get_bio_sector(vio->bios_merged.tail)); + bio = vio->bios_merged.head; + bio_list_init(&vio->bios_merged); + mutex_unlock(&bio_queue_data->lock); + + return bio; +} + +/** + * process_data_vio_io() - Submit a data_vio's bio to the storage below along with any bios that + * have been merged with it. + * + * Context: This call may block and so should only be called from a bio thread. + */ +static void process_data_vio_io(struct vdo_completion *completion) +{ + struct bio *bio, *next; + struct vio *vio = as_vio(completion); + + assert_in_bio_zone(vio); + for (bio = get_bio_list(vio); bio != NULL; bio = next) { + next = bio->bi_next; + bio->bi_next = NULL; + send_bio_to_device((struct vio *) bio->bi_private, bio); + } +} + +/** + * get_mergeable_locked() - Attempt to find an already queued bio that the current bio can be + * merged with. + * @map: The bio map to use for merging. + * @vio: The vio we want to merge. + * @back_merge: Set to true for a back merge, false for a front merge. + * + * There are two types of merging possible, forward and backward, which are distinguished by a flag + * that uses kernel elevator terminology. + * + * Return: the vio to merge to, NULL if no merging is possible. + */ +static struct vio *get_mergeable_locked(struct int_map *map, struct vio *vio, + bool back_merge) +{ + struct bio *bio = vio->bio; + sector_t merge_sector = get_bio_sector(bio); + struct vio *vio_merge; + + if (back_merge) + merge_sector -= VDO_SECTORS_PER_BLOCK; + else + merge_sector += VDO_SECTORS_PER_BLOCK; + + vio_merge = vdo_int_map_get(map, merge_sector); + + if (vio_merge == NULL) + return NULL; + + if (vio->completion.priority != vio_merge->completion.priority) + return NULL; + + if (bio_data_dir(bio) != bio_data_dir(vio_merge->bio)) + return NULL; + + if (bio_list_empty(&vio_merge->bios_merged)) + return NULL; + + if (back_merge) { + return (get_bio_sector(vio_merge->bios_merged.tail) == merge_sector ? + vio_merge : NULL); + } + + return (get_bio_sector(vio_merge->bios_merged.head) == merge_sector ? + vio_merge : NULL); +} + +static int map_merged_vio(struct int_map *bio_map, struct vio *vio) +{ + int result; + + result = vdo_int_map_put(bio_map, get_bio_sector(vio->bios_merged.head), vio, + true, NULL); + if (result != VDO_SUCCESS) + return result; + + return vdo_int_map_put(bio_map, get_bio_sector(vio->bios_merged.tail), vio, true, + NULL); +} + +static int merge_to_prev_tail(struct int_map *bio_map, struct vio *vio, + struct vio *prev_vio) +{ + vdo_int_map_remove(bio_map, get_bio_sector(prev_vio->bios_merged.tail)); + bio_list_merge(&prev_vio->bios_merged, &vio->bios_merged); + return map_merged_vio(bio_map, prev_vio); +} + +static int merge_to_next_head(struct int_map *bio_map, struct vio *vio, + struct vio *next_vio) +{ + /* + * Handle "next merge" and "gap fill" cases the same way so as to reorder bios in a way + * that's compatible with using funnel queues in work queues. This avoids removing an + * existing completion. + */ + vdo_int_map_remove(bio_map, get_bio_sector(next_vio->bios_merged.head)); + bio_list_merge_head(&next_vio->bios_merged, &vio->bios_merged); + return map_merged_vio(bio_map, next_vio); +} + +/** + * try_bio_map_merge() - Attempt to merge a vio's bio with other pending I/Os. + * @vio: The vio to merge. + * + * Currently this is only used for data_vios, but is broken out for future use with metadata vios. + * + * Return: whether or not the vio was merged. + */ +static bool try_bio_map_merge(struct vio *vio) +{ + int result; + bool merged = true; + struct bio *bio = vio->bio; + struct vio *prev_vio, *next_vio; + struct vdo *vdo = vio->completion.vdo; + struct bio_queue_data *bio_queue_data = + &vdo->io_submitter->bio_queue_data[vio->bio_zone]; + + bio->bi_next = NULL; + bio_list_init(&vio->bios_merged); + bio_list_add(&vio->bios_merged, bio); + + mutex_lock(&bio_queue_data->lock); + prev_vio = get_mergeable_locked(bio_queue_data->map, vio, true); + next_vio = get_mergeable_locked(bio_queue_data->map, vio, false); + if (prev_vio == next_vio) + next_vio = NULL; + + if ((prev_vio == NULL) && (next_vio == NULL)) { + /* no merge. just add to bio_queue */ + merged = false; + result = vdo_int_map_put(bio_queue_data->map, + get_bio_sector(bio), + vio, true, NULL); + } else if (next_vio == NULL) { + /* Only prev. merge to prev's tail */ + result = merge_to_prev_tail(bio_queue_data->map, vio, prev_vio); + } else { + /* Only next. merge to next's head */ + result = merge_to_next_head(bio_queue_data->map, vio, next_vio); + } + + mutex_unlock(&bio_queue_data->lock); + + /* We don't care about failure of int_map_put in this case. */ + ASSERT_LOG_ONLY(result == UDS_SUCCESS, "bio map insertion succeeds"); + return merged; +} + +/** + * submit_data_vio_io() - Submit I/O for a data_vio. + * @data_vio: the data_vio for which to issue I/O. + * + * If possible, this I/O will be merged other pending I/Os. Otherwise, the data_vio will be sent to + * the appropriate bio zone directly. + */ +void submit_data_vio_io(struct data_vio *data_vio) +{ + if (try_bio_map_merge(&data_vio->vio)) + return; + + launch_data_vio_bio_zone_callback(data_vio, process_data_vio_io); +} + +/** + * vdo_submit_metadata_io() - Submit I/O for a metadata vio. + * @vio: the vio for which to issue I/O + * @physical: the physical block number to read or write + * @callback: the bio endio function which will be called after the I/O completes + * @error_handler: the handler for submission or I/O errors (may be NULL) + * @operation: the type of I/O to perform + * @data: the buffer to read or write (may be NULL) + * + * The vio is enqueued on a vdo bio queue so that bio submission (which may block) does not block + * other vdo threads. + * + * That the error handler will run on the correct thread is only true so long as the thread calling + * this function, and the thread set in the endio callback are the same, as well as the fact that + * no error can occur on the bio queue. Currently this is true for all callers, but additional care + * will be needed if this ever changes. + */ +void vdo_submit_metadata_io(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + unsigned int operation, char *data) +{ + struct vdo_completion *completion = &vio->completion; + int result; + const struct admin_state_code *code = vdo_get_admin_state(completion->vdo); + + + ASSERT_LOG_ONLY(!code->quiescent, "I/O not allowed in state %s", code->name); + ASSERT_LOG_ONLY(vio->bio->bi_next == NULL, "metadata bio has no next bio"); + + vdo_reset_completion(completion); + completion->error_handler = error_handler; + result = vio_reset_bio(vio, data, callback, operation | REQ_META, physical); + if (result != VDO_SUCCESS) { + continue_vio(vio, result); + return; + } + + vdo_set_completion_callback(completion, process_vio_io, + get_vio_bio_zone_thread_id(vio)); + vdo_launch_completion_with_priority(completion, get_metadata_priority(vio)); +} + +/** + * vdo_make_io_submitter() - Create an io_submitter structure. + * @thread_count: Number of bio-submission threads to set up. + * @rotation_interval: Interval to use when rotating between bio-submission threads when enqueuing + * completions. + * @max_requests_active: Number of bios for merge tracking. + * @vdo: The vdo which will use this submitter. + * @io_submitter: pointer to the new data structure. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_interval, + unsigned int max_requests_active, struct vdo *vdo, + struct io_submitter **io_submitter_ptr) +{ + unsigned int i; + struct io_submitter *io_submitter; + int result; + + result = uds_allocate_extended(struct io_submitter, thread_count, + struct bio_queue_data, "bio submission data", + &io_submitter); + if (result != UDS_SUCCESS) + return result; + + io_submitter->bio_queue_rotation_interval = rotation_interval; + + /* Setup for each bio-submission work queue */ + for (i = 0; i < thread_count; i++) { + struct bio_queue_data *bio_queue_data = &io_submitter->bio_queue_data[i]; + + mutex_init(&bio_queue_data->lock); + /* + * One I/O operation per request, but both first & last sector numbers. + * + * If requests are assigned to threads round-robin, they should be distributed + * quite evenly. But if they're assigned based on PBN, things can sometimes be very + * uneven. So for now, we'll assume that all requests *may* wind up on one thread, + * and thus all in the same map. + */ + result = vdo_make_int_map(max_requests_active * 2, 0, + &bio_queue_data->map); + if (result != 0) { + /* + * Clean up the partially initialized bio-queue entirely and indicate that + * initialization failed. + */ + uds_log_error("bio map initialization failed %d", result); + vdo_cleanup_io_submitter(io_submitter); + vdo_free_io_submitter(io_submitter); + return result; + } + + bio_queue_data->queue_number = i; + result = vdo_make_thread(vdo, vdo->thread_config.bio_threads[i], + &bio_queue_type, 1, (void **) &bio_queue_data); + if (result != VDO_SUCCESS) { + /* + * Clean up the partially initialized bio-queue entirely and indicate that + * initialization failed. + */ + vdo_free_int_map(uds_forget(bio_queue_data->map)); + uds_log_error("bio queue initialization failed %d", result); + vdo_cleanup_io_submitter(io_submitter); + vdo_free_io_submitter(io_submitter); + return result; + } + + bio_queue_data->queue = vdo->threads[vdo->thread_config.bio_threads[i]].queue; + io_submitter->num_bio_queues_used++; + } + + *io_submitter_ptr = io_submitter; + + return VDO_SUCCESS; +} + +/** + * vdo_cleanup_io_submitter() - Tear down the io_submitter fields as needed for a physical layer. + * @io_submitter: The I/O submitter data to tear down (may be NULL). + */ +void vdo_cleanup_io_submitter(struct io_submitter *io_submitter) +{ + int i; + + if (io_submitter == NULL) + return; + + for (i = io_submitter->num_bio_queues_used - 1; i >= 0; i--) + vdo_finish_work_queue(io_submitter->bio_queue_data[i].queue); +} + +/** + * vdo_free_io_submitter() - Free the io_submitter fields and structure as needed. + * @io_submitter: The I/O submitter data to destroy. + * + * This must be called after vdo_cleanup_io_submitter(). It is used to release resources late in + * the shutdown process to avoid or reduce the chance of race conditions. + */ +void vdo_free_io_submitter(struct io_submitter *io_submitter) +{ + int i; + + if (io_submitter == NULL) + return; + + for (i = io_submitter->num_bio_queues_used - 1; i >= 0; i--) { + io_submitter->num_bio_queues_used--; + /* vdo_destroy() will free the work queue, so just give up our reference to it. */ + uds_forget(io_submitter->bio_queue_data[i].queue); + vdo_free_int_map(uds_forget(io_submitter->bio_queue_data[i].map)); + } + uds_free(io_submitter); +} diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h new file mode 100644 index 000000000000..96befb3739e9 --- /dev/null +++ b/drivers/md/dm-vdo/io-submitter.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_IO_SUBMITTER_H +#define VDO_IO_SUBMITTER_H + +#include + +#include "types.h" + +struct io_submitter; + +int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_interval, + unsigned int max_requests_active, struct vdo *vdo, + struct io_submitter **io_submitter); + +void vdo_cleanup_io_submitter(struct io_submitter *io_submitter); + +void vdo_free_io_submitter(struct io_submitter *io_submitter); + +void process_vio_io(struct vdo_completion *completion); + +void submit_data_vio_io(struct data_vio *data_vio); + +void vdo_submit_metadata_io(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + unsigned int operation, char *data); + +static inline void submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + unsigned int operation) +{ + vdo_submit_metadata_io(vio, physical, callback, error_handler, + operation, vio->data); +} + +static inline void submit_flush_vio(struct vio *vio, bio_end_io_t callback, + vdo_action_fn error_handler) +{ + /* FIXME: Can we just use REQ_OP_FLUSH? */ + vdo_submit_metadata_io(vio, 0, callback, error_handler, + REQ_OP_WRITE | REQ_PREFLUSH, NULL); +} + +#endif /* VDO_IO_SUBMITTER_H */ From cfaf07fae7a262865a67b6010eef3bc0a1ae50b2 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:11:44 -0500 Subject: [PATCH 0570/1038] dm vdo: add hash locks and hash zones In order to deduplicate concurrent writes of the same data (to different locations), data_vios which are writing the same data are grouped together in a "hash lock," named for and keyed by the hash of the data being written. Each hash lock is assigned to a hash zone based on a portion of its hash. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dedupe.c | 2443 ++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/dedupe.h | 94 ++ 2 files changed, 2537 insertions(+) create mode 100644 drivers/md/dm-vdo/dedupe.c create mode 100644 drivers/md/dm-vdo/dedupe.h diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c new file mode 100644 index 000000000000..f40cc9643b1c --- /dev/null +++ b/drivers/md/dm-vdo/dedupe.c @@ -0,0 +1,2443 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/** + * DOC: + * + * Hash Locks: + * + * A hash_lock controls and coordinates writing, index access, and dedupe among groups of data_vios + * concurrently writing identical blocks, allowing them to deduplicate not only against advice but + * also against each other. This saves on index queries and allows those data_vios to concurrently + * deduplicate against a single block instead of being serialized through a PBN read lock. Only one + * index query is needed for each hash_lock, instead of one for every data_vio. + * + * A hash_lock acts like a state machine perhaps more than as a lock. Other than the starting and + * ending states INITIALIZING and BYPASSING, every state represents and is held for the duration of + * an asynchronous operation. All state transitions are performed on the thread of the hash_zone + * containing the lock. An asynchronous operation is almost always performed upon entering a state, + * and the callback from that operation triggers exiting the state and entering a new state. + * + * In all states except DEDUPING, there is a single data_vio, called the lock agent, performing the + * asynchronous operations on behalf of the lock. The agent will change during the lifetime of the + * lock if the lock is shared by more than one data_vio. data_vios waiting to deduplicate are kept + * on a wait queue. Viewed a different way, the agent holds the lock exclusively until the lock + * enters the DEDUPING state, at which point it becomes a shared lock that all the waiters (and any + * new data_vios that arrive) use to share a PBN lock. In state DEDUPING, there is no agent. When + * the last data_vio in the lock calls back in DEDUPING, it becomes the agent and the lock becomes + * exclusive again. New data_vios that arrive in the lock will also go on the wait queue. + * + * The existence of lock waiters is a key factor controlling which state the lock transitions to + * next. When the lock is new or has waiters, it will always try to reach DEDUPING, and when it + * doesn't, it will try to clean up and exit. + * + * Deduping requires holding a PBN lock on a block that is known to contain data identical to the + * data_vios in the lock, so the lock will send the agent to the duplicate zone to acquire the PBN + * lock (LOCKING), to the kernel I/O threads to read and verify the data (VERIFYING), or to write a + * new copy of the data to a full data block or a slot in a compressed block (WRITING). + * + * Cleaning up consists of updating the index when the data location is different from the initial + * index query (UPDATING, triggered by stale advice, compression, and rollover), releasing the PBN + * lock on the duplicate block (UNLOCKING), and if the agent is the last data_vio referencing the + * lock, releasing the hash_lock itself back to the hash zone (BYPASSING). + * + * The shortest sequence of states is for non-concurrent writes of new data: + * INITIALIZING -> QUERYING -> WRITING -> BYPASSING + * This sequence is short because no PBN read lock or index update is needed. + * + * Non-concurrent, finding valid advice looks like this (endpoints elided): + * -> QUERYING -> LOCKING -> VERIFYING -> DEDUPING -> UNLOCKING -> + * Or with stale advice (endpoints elided): + * -> QUERYING -> LOCKING -> VERIFYING -> UNLOCKING -> WRITING -> UPDATING -> + * + * When there are not enough available reference count increments available on a PBN for a data_vio + * to deduplicate, a new lock is forked and the excess waiters roll over to the new lock (which + * goes directly to WRITING). The new lock takes the place of the old lock in the lock map so new + * data_vios will be directed to it. The two locks will proceed independently, but only the new + * lock will have the right to update the index (unless it also forks). + * + * Since rollover happens in a lock instance, once a valid data location has been selected, it will + * not change. QUERYING and WRITING are only performed once per lock lifetime. All other + * non-endpoint states can be re-entered. + * + * The function names in this module follow a convention referencing the states and transitions in + * the state machine. For example, for the LOCKING state, there are start_locking() and + * finish_locking() functions. start_locking() is invoked by the finish function of the state (or + * states) that transition to LOCKING. It performs the actual lock state change and must be invoked + * on the hash zone thread. finish_locking() is called by (or continued via callback from) the + * code actually obtaining the lock. It does any bookkeeping or decision-making required and + * invokes the appropriate start function of the state being transitioned to after LOCKING. + * + * ---------------------------------------------------------------------- + * + * Index Queries: + * + * A query to the UDS index is handled asynchronously by the index's threads. When the query is + * complete, a callback supplied with the query will be called from one of the those threads. Under + * heavy system load, the index may be slower to respond then is desirable for reasonable I/O + * throughput. Since deduplication of writes is not necessary for correct operation of a VDO + * device, it is acceptable to timeout out slow index queries and proceed to fulfill a write + * request without deduplicating. However, because the uds_request struct itself is supplied by the + * caller, we can not simply reuse a uds_request object which we have chosen to timeout. Hence, + * each hash_zone maintains a pool of dedupe_contexts which each contain a uds_request along with a + * reference to the data_vio on behalf of which they are performing a query. + * + * When a hash_lock needs to query the index, it attempts to acquire an unused dedupe_context from + * its hash_zone's pool. If one is available, that context is prepared, associated with the + * hash_lock's agent, added to the list of pending contexts, and then sent to the index. The + * context's state will be transitioned from DEDUPE_CONTEXT_IDLE to DEDUPE_CONTEXT_PENDING. If all + * goes well, the dedupe callback will be called by the index which will change the context's state + * to DEDUPE_CONTEXT_COMPLETE, and the associated data_vio will be enqueued to run back in the hash + * zone where the query results will be processed and the context will be put back in the idle + * state and returned to the hash_zone's available list. + * + * The first time an index query is launched from a given hash_zone, a timer is started. When the + * timer fires, the hash_zone's completion is enqueued to run in the hash_zone where the zone's + * pending list will be searched for any contexts in the pending state which have been running for + * too long. Those contexts are transitioned to the DEDUPE_CONTEXT_TIMED_OUT state and moved to the + * zone's timed_out list where they won't be examined again if there is a subsequent time out). The + * data_vios associated with timed out contexts are sent to continue processing their write + * operation without deduplicating. The timer is also restarted. + * + * When the dedupe callback is run for a context which is in the timed out state, that context is + * moved to the DEDUPE_CONTEXT_TIMED_OUT_COMPLETE state. No other action need be taken as the + * associated data_vios have already been dispatched. + * + * If a hash_lock needs a dedupe context, and the available list is empty, the timed_out list will + * be searched for any contexts which are timed out and complete. One of these will be used + * immediately, and the rest will be returned to the available list and marked idle. + */ + +#include "dedupe.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" +#include "uds.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "io-submitter.h" +#include "packer.h" +#include "physical-zone.h" +#include "pointer-map.h" +#include "slab-depot.h" +#include "statistics.h" +#include "types.h" +#include "vdo.h" +#include "wait-queue.h" + +enum hash_lock_state { + /* State for locks that are not in use or are being initialized. */ + VDO_HASH_LOCK_INITIALIZING, + + /* This is the sequence of states typically used on the non-dedupe path. */ + VDO_HASH_LOCK_QUERYING, + VDO_HASH_LOCK_WRITING, + VDO_HASH_LOCK_UPDATING, + + /* The remaining states are typically used on the dedupe path in this order. */ + VDO_HASH_LOCK_LOCKING, + VDO_HASH_LOCK_VERIFYING, + VDO_HASH_LOCK_DEDUPING, + VDO_HASH_LOCK_UNLOCKING, + + /* + * Terminal state for locks returning to the pool. Must be last both because it's the final + * state, and also because it's used to count the states. + */ + VDO_HASH_LOCK_BYPASSING, +}; + +static const char * const LOCK_STATE_NAMES[] = { + [VDO_HASH_LOCK_BYPASSING] = "BYPASSING", + [VDO_HASH_LOCK_DEDUPING] = "DEDUPING", + [VDO_HASH_LOCK_INITIALIZING] = "INITIALIZING", + [VDO_HASH_LOCK_LOCKING] = "LOCKING", + [VDO_HASH_LOCK_QUERYING] = "QUERYING", + [VDO_HASH_LOCK_UNLOCKING] = "UNLOCKING", + [VDO_HASH_LOCK_UPDATING] = "UPDATING", + [VDO_HASH_LOCK_VERIFYING] = "VERIFYING", + [VDO_HASH_LOCK_WRITING] = "WRITING", +}; + +struct hash_lock { + /* The block hash covered by this lock */ + struct uds_record_name hash; + + /* When the lock is unused, this list entry allows the lock to be pooled */ + struct list_head pool_node; + + /* + * A list containing the data VIOs sharing this lock, all having the same record name and + * data block contents, linked by their hash_lock_node fields. + */ + struct list_head duplicate_ring; + + /* The number of data_vios sharing this lock instance */ + data_vio_count_t reference_count; + + /* The maximum value of reference_count in the lifetime of this lock */ + data_vio_count_t max_references; + + /* The current state of this lock */ + enum hash_lock_state state; + + /* True if the UDS index should be updated with new advice */ + bool update_advice; + + /* True if the advice has been verified to be a true duplicate */ + bool verified; + + /* True if the lock has already accounted for an initial verification */ + bool verify_counted; + + /* True if this lock is registered in the lock map (cleared on rollover) */ + bool registered; + + /* + * If verified is false, this is the location of a possible duplicate. If verified is true, + * it is the verified location of a true duplicate. + */ + struct zoned_pbn duplicate; + + /* The PBN lock on the block containing the duplicate data */ + struct pbn_lock *duplicate_lock; + + /* The data_vio designated to act on behalf of the lock */ + struct data_vio *agent; + + /* + * Other data_vios with data identical to the agent who are currently waiting for the agent + * to get the information they all need to deduplicate--either against each other, or + * against an existing duplicate on disk. + */ + struct wait_queue waiters; +}; + +enum { + LOCK_POOL_CAPACITY = MAXIMUM_VDO_USER_VIOS, +}; + +struct hash_zones { + struct action_manager *manager; + struct kobject dedupe_directory; + struct uds_parameters parameters; + struct uds_index_session *index_session; + struct ratelimit_state ratelimiter; + atomic64_t timeouts; + atomic64_t dedupe_context_busy; + + /* This spinlock protects the state fields and the starting of dedupe requests. */ + spinlock_t lock; + + /* The fields in the next block are all protected by the lock */ + struct vdo_completion completion; + enum index_state index_state; + enum index_state index_target; + struct admin_state state; + bool changing; + bool create_flag; + bool dedupe_flag; + bool error_flag; + u64 reported_timeouts; + + /* The number of zones */ + zone_count_t zone_count; + /* The hash zones themselves */ + struct hash_zone zones[]; +}; + +static inline struct hash_zone *as_hash_zone(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_HASH_ZONE_COMPLETION); + return container_of(completion, struct hash_zone, completion); +} + +static inline struct hash_zones *as_hash_zones(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_HASH_ZONES_COMPLETION); + return container_of(completion, struct hash_zones, completion); +} + +static inline void assert_in_hash_zone(struct hash_zone *zone, const char *name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), + "%s called on hash zone thread", name); +} + +/** + * return_hash_lock_to_pool() - (Re)initialize a hash lock and return it to its pool. + * @zone: The zone from which the lock was borrowed. + * @lock: The lock that is no longer in use. + */ +static void return_hash_lock_to_pool(struct hash_zone *zone, struct hash_lock *lock) +{ + memset(lock, 0, sizeof(*lock)); + INIT_LIST_HEAD(&lock->pool_node); + INIT_LIST_HEAD(&lock->duplicate_ring); + vdo_initialize_wait_queue(&lock->waiters); + list_add_tail(&lock->pool_node, &zone->lock_pool); +} + +/** + * vdo_get_duplicate_lock() - Get the PBN lock on the duplicate data location for a data_vio from + * the hash_lock the data_vio holds (if there is one). + * @data_vio: The data_vio to query. + * + * Return: The PBN lock on the data_vio's duplicate location. + */ +struct pbn_lock *vdo_get_duplicate_lock(struct data_vio *data_vio) +{ + if (data_vio->hash_lock == NULL) + return NULL; + + return data_vio->hash_lock->duplicate_lock; +} + +/** + * get_hash_lock_state_name() - Get the string representation of a hash lock state. + * @state: The hash lock state. + * + * Return: The short string representing the state + */ +static const char *get_hash_lock_state_name(enum hash_lock_state state) +{ + /* Catch if a state has been added without updating the name array. */ + BUILD_BUG_ON((VDO_HASH_LOCK_BYPASSING + 1) != ARRAY_SIZE(LOCK_STATE_NAMES)); + return (state < ARRAY_SIZE(LOCK_STATE_NAMES)) ? LOCK_STATE_NAMES[state] : "INVALID"; +} + +/** + * assert_hash_lock_agent() - Assert that a data_vio is the agent of its hash lock, and that this + * is being called in the hash zone. + * @data_vio: The data_vio expected to be the lock agent. + * @where: A string describing the function making the assertion. + */ +static void assert_hash_lock_agent(struct data_vio *data_vio, const char *where) +{ + /* Not safe to access the agent field except from the hash zone. */ + assert_data_vio_in_hash_zone(data_vio); + ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent, + "%s must be for the hash lock agent", where); +} + +/** + * set_duplicate_lock() - Set the duplicate lock held by a hash lock. May only be called in the + * physical zone of the PBN lock. + * @hash_lock: The hash lock to update. + * @pbn_lock: The PBN read lock to use as the duplicate lock. + */ +static void set_duplicate_lock(struct hash_lock *hash_lock, struct pbn_lock *pbn_lock) +{ + ASSERT_LOG_ONLY((hash_lock->duplicate_lock == NULL), + "hash lock must not already hold a duplicate lock"); + + pbn_lock->holder_count += 1; + hash_lock->duplicate_lock = pbn_lock; +} + +/** + * dequeue_lock_waiter() - Remove the first data_vio from the lock's wait queue and return it. + * @lock: The lock containing the wait queue. + * + * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. + */ +static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock) +{ + return waiter_as_data_vio(vdo_dequeue_next_waiter(&lock->waiters)); +} + +/** + * set_hash_lock() - Set, change, or clear the hash lock a data_vio is using. + * @data_vio: The data_vio to update. + * @new_lock: The hash lock the data_vio is joining. + * + * Updates the hash lock (or locks) to reflect the change in membership. + */ +static void set_hash_lock(struct data_vio *data_vio, struct hash_lock *new_lock) +{ + struct hash_lock *old_lock = data_vio->hash_lock; + + if (old_lock != NULL) { + ASSERT_LOG_ONLY(data_vio->hash_zone != NULL, + "must have a hash zone when holding a hash lock"); + ASSERT_LOG_ONLY(!list_empty(&data_vio->hash_lock_entry), + "must be on a hash lock ring when holding a hash lock"); + ASSERT_LOG_ONLY(old_lock->reference_count > 0, + "hash lock reference must be counted"); + + if ((old_lock->state != VDO_HASH_LOCK_BYPASSING) && + (old_lock->state != VDO_HASH_LOCK_UNLOCKING)) { + /* + * If the reference count goes to zero in a non-terminal state, we're most + * likely leaking this lock. + */ + ASSERT_LOG_ONLY(old_lock->reference_count > 1, + "hash locks should only become unreferenced in a terminal state, not state %s", + get_hash_lock_state_name(old_lock->state)); + } + + list_del_init(&data_vio->hash_lock_entry); + old_lock->reference_count -= 1; + + data_vio->hash_lock = NULL; + } + + if (new_lock != NULL) { + /* + * Keep all data_vios sharing the lock on a ring since they can complete in any + * order and we'll always need a pointer to one to compare data. + */ + list_move_tail(&data_vio->hash_lock_entry, &new_lock->duplicate_ring); + new_lock->reference_count += 1; + if (new_lock->max_references < new_lock->reference_count) + new_lock->max_references = new_lock->reference_count; + + data_vio->hash_lock = new_lock; + } +} + +/* There are loops in the state diagram, so some forward decl's are needed. */ +static void start_deduping(struct hash_lock *lock, struct data_vio *agent, + bool agent_is_done); +static void start_locking(struct hash_lock *lock, struct data_vio *agent); +static void start_writing(struct hash_lock *lock, struct data_vio *agent); +static void unlock_duplicate_pbn(struct vdo_completion *completion); +static void transfer_allocation_lock(struct data_vio *data_vio); + +/** + * exit_hash_lock() - Bottleneck for data_vios that have written or deduplicated and that are no + * longer needed to be an agent for the hash lock. + * @data_vio: The data_vio to complete and send to be cleaned up. + */ +static void exit_hash_lock(struct data_vio *data_vio) +{ + /* Release the hash lock now, saving a thread transition in cleanup. */ + vdo_release_hash_lock(data_vio); + + /* Complete the data_vio and start the clean-up path to release any locks it still holds. */ + data_vio->vio.completion.callback = complete_data_vio; + + continue_data_vio(data_vio); +} + +/** + * set_duplicate_location() - Set the location of the duplicate block for data_vio, updating the + * is_duplicate and duplicate fields from a zoned_pbn. + * @data_vio: The data_vio to modify. + * @source: The location of the duplicate. + */ +static void set_duplicate_location(struct data_vio *data_vio, + const struct zoned_pbn source) +{ + data_vio->is_duplicate = (source.pbn != VDO_ZERO_BLOCK); + data_vio->duplicate = source; +} + +/** + * retire_lock_agent() - Retire the active lock agent, replacing it with the first lock waiter, and + * make the retired agent exit the hash lock. + * @lock: The hash lock to update. + * + * Return: The new lock agent (which will be NULL if there was no waiter) + */ +static struct data_vio *retire_lock_agent(struct hash_lock *lock) +{ + struct data_vio *old_agent = lock->agent; + struct data_vio *new_agent = dequeue_lock_waiter(lock); + + lock->agent = new_agent; + exit_hash_lock(old_agent); + if (new_agent != NULL) + set_duplicate_location(new_agent, lock->duplicate); + return new_agent; +} + +/** + * wait_on_hash_lock() - Add a data_vio to the lock's queue of waiters. + * @lock: The hash lock on which to wait. + * @data_vio: The data_vio to add to the queue. + */ +static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio) +{ + vdo_enqueue_waiter(&lock->waiters, &data_vio->waiter); + + /* + * Make sure the agent doesn't block indefinitely in the packer since it now has at least + * one other data_vio waiting on it. + */ + if ((lock->state != VDO_HASH_LOCK_WRITING) || !cancel_data_vio_compression(lock->agent)) + return; + + /* + * Even though we're waiting, we also have to send ourselves as a one-way message to the + * packer to ensure the agent continues executing. This is safe because + * cancel_vio_compression() guarantees the agent won't continue executing until this + * message arrives in the packer, and because the wait queue link isn't used for sending + * the message. + */ + data_vio->compression.lock_holder = lock->agent; + launch_data_vio_packer_callback(data_vio, vdo_remove_lock_holder_from_packer); +} + +/** + * abort_waiter() - waiter_callback_fn function that shunts waiters to write their blocks without + * optimization. + * @waiter: The data_vio's waiter link. + * @context: Not used. + */ +static void abort_waiter(struct waiter *waiter, void *context __always_unused) +{ + write_data_vio(waiter_as_data_vio(waiter)); +} + +/** + * start_bypassing() - Stop using the hash lock. + * @lock: The hash lock. + * @agent: The data_vio acting as the agent for the lock. + * + * Stops using the hash lock. This is the final transition for hash locks which did not get an + * error. + */ +static void start_bypassing(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_BYPASSING; + exit_hash_lock(agent); +} + +void vdo_clean_failed_hash_lock(struct data_vio *data_vio) +{ + struct hash_lock *lock = data_vio->hash_lock; + + if (lock->state == VDO_HASH_LOCK_BYPASSING) { + exit_hash_lock(data_vio); + return; + } + + if (lock->agent == NULL) { + lock->agent = data_vio; + } else if (data_vio != lock->agent) { + exit_hash_lock(data_vio); + return; + } + + lock->state = VDO_HASH_LOCK_BYPASSING; + + /* Ensure we don't attempt to update advice when cleaning up. */ + lock->update_advice = false; + + vdo_notify_all_waiters(&lock->waiters, abort_waiter, NULL); + + if (lock->duplicate_lock != NULL) { + /* The agent must reference the duplicate zone to launch it. */ + data_vio->duplicate = lock->duplicate; + launch_data_vio_duplicate_zone_callback(data_vio, unlock_duplicate_pbn); + return; + } + + lock->agent = NULL; + data_vio->is_duplicate = false; + exit_hash_lock(data_vio); +} + +/** + * finish_unlocking() - Handle the result of the agent for the lock releasing a read lock on + * duplicate candidate. + * @completion: The completion of the data_vio acting as the lock's agent. + * + * This continuation is registered in unlock_duplicate_pbn(). + */ +static void finish_unlocking(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must have released the duplicate lock for the hash lock"); + + if (!lock->verified) { + /* + * UNLOCKING -> WRITING transition: The lock we released was on an unverified + * block, so it must have been a lock on advice we were verifying, not on a + * location that was used for deduplication. Go write (or compress) the block to + * get a location to dedupe against. + */ + start_writing(lock, agent); + return; + } + + /* + * With the lock released, the verified duplicate block may already have changed and will + * need to be re-verified if a waiter arrived. + */ + lock->verified = false; + + if (vdo_has_waiters(&lock->waiters)) { + /* + * UNLOCKING -> LOCKING transition: A new data_vio entered the hash lock while the + * agent was releasing the PBN lock. The current agent exits and the waiter has to + * re-lock and re-verify the duplicate location. + * + * TODO: If we used the current agent to re-acquire the PBN lock we wouldn't need + * to re-verify. + */ + agent = retire_lock_agent(lock); + start_locking(lock, agent); + return; + } + + /* + * UNLOCKING -> BYPASSING transition: The agent is done with the lock and no other + * data_vios reference it, so remove it from the lock map and return it to the pool. + */ + start_bypassing(lock, agent); +} + +/** + * unlock_duplicate_pbn() - Release a read lock on the PBN of the block that may or may not have + * contained duplicate data. + * @completion: The completion of the data_vio acting as the lock's agent. + * + * This continuation is launched by start_unlocking(), and calls back to finish_unlocking() on the + * hash zone thread. + */ +static void unlock_duplicate_pbn(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_data_vio_in_duplicate_zone(agent); + ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, + "must have a duplicate lock to release"); + + vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, agent->duplicate.pbn, + uds_forget(lock->duplicate_lock)); + if (lock->state == VDO_HASH_LOCK_BYPASSING) { + complete_data_vio(completion); + return; + } + + launch_data_vio_hash_zone_callback(agent, finish_unlocking); +} + +/** + * start_unlocking() - Release a read lock on the PBN of the block that may or may not have + * contained duplicate data. + * @lock: The hash lock. + * @agent: The data_vio currently acting as the agent for the lock. + */ +static void start_unlocking(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_UNLOCKING; + launch_data_vio_duplicate_zone_callback(agent, unlock_duplicate_pbn); +} + +static void release_context(struct dedupe_context *context) +{ + struct hash_zone *zone = context->zone; + + WRITE_ONCE(zone->active, zone->active - 1); + list_move(&context->list_entry, &zone->available); +} + +static void process_update_result(struct data_vio *agent) +{ + struct dedupe_context *context = agent->dedupe_context; + + if ((context == NULL) || + !change_context_state(context, DEDUPE_CONTEXT_COMPLETE, DEDUPE_CONTEXT_IDLE)) + return; + + release_context(context); +} + +/** + * finish_updating() - Process the result of a UDS update performed by the agent for the lock. + * @completion: The completion of the data_vio that performed the update + * + * This continuation is registered in start_querying(). + */ +static void finish_updating(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + process_update_result(agent); + + /* + * UDS was updated successfully, so don't update again unless the duplicate location + * changes due to rollover. + */ + lock->update_advice = false; + + if (vdo_has_waiters(&lock->waiters)) { + /* + * UPDATING -> DEDUPING transition: A new data_vio arrived during the UDS update. + * Send it on the verified dedupe path. The agent is done with the lock, but the + * lock may still need to use it to clean up after rollover. + */ + start_deduping(lock, agent, true); + return; + } + + if (lock->duplicate_lock != NULL) { + /* + * UPDATING -> UNLOCKING transition: No one is waiting to dedupe, but we hold a + * duplicate PBN lock, so go release it. + */ + start_unlocking(lock, agent); + return; + } + + /* + * UPDATING -> BYPASSING transition: No one is waiting to dedupe and there's no lock to + * release. + */ + start_bypassing(lock, agent); +} + +static void query_index(struct data_vio *data_vio, enum uds_request_type operation); + +/** + * start_updating() - Continue deduplication with the last step, updating UDS with the location of + * the duplicate that should be returned as advice in the future. + * @lock: The hash lock. + * @agent: The data_vio currently acting as the agent for the lock. + */ +static void start_updating(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_UPDATING; + + ASSERT_LOG_ONLY(lock->verified, "new advice should have been verified"); + ASSERT_LOG_ONLY(lock->update_advice, "should only update advice if needed"); + + agent->last_async_operation = VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX; + set_data_vio_hash_zone_callback(agent, finish_updating); + query_index(agent, UDS_UPDATE); +} + +/** + * finish_deduping() - Handle a data_vio that has finished deduplicating against the block locked + * by the hash lock. + * @lock: The hash lock. + * @data_vio: The lock holder that has finished deduplicating. + * + * If there are other data_vios still sharing the lock, this will just release the data_vio's share + * of the lock and finish processing the data_vio. If this is the last data_vio holding the lock, + * this makes the data_vio the lock agent and uses it to advance the state of the lock so it can + * eventually be released. + */ +static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio) +{ + struct data_vio *agent = data_vio; + + ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING"); + ASSERT_LOG_ONLY(!vdo_has_waiters(&lock->waiters), + "shouldn't have any lock waiters in DEDUPING"); + + /* Just release the lock reference if other data_vios are still deduping. */ + if (lock->reference_count > 1) { + exit_hash_lock(data_vio); + return; + } + + /* The hash lock must have an agent for all other lock states. */ + lock->agent = agent; + if (lock->update_advice) { + /* + * DEDUPING -> UPDATING transition: The location of the duplicate block changed + * since the initial UDS query because of compression, rollover, or because the + * query agent didn't have an allocation. The UDS update was delayed in case there + * was another change in location, but with only this data_vio using the hash lock, + * it's time to update the advice. + */ + start_updating(lock, agent); + } else { + /* + * DEDUPING -> UNLOCKING transition: Release the PBN read lock on the duplicate + * location so the hash lock itself can be released (contingent on no new data_vios + * arriving in the lock before the agent returns). + */ + start_unlocking(lock, agent); + } +} + +/** + * acquire_lock() - Get the lock for a record name. + * @zone: The zone responsible for the hash. + * @hash: The hash to lock. + * @replace_lock: If non-NULL, the lock already registered for the hash which should be replaced by + * the new lock. + * @lock_ptr: A pointer to receive the hash lock. + * + * Gets the lock for the hash (record name) of the data in a data_vio, or if one does not exist (or + * if we are explicitly rolling over), initialize a new lock for the hash and register it in the + * zone. This must only be called in the correct thread for the zone. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check acquire_lock(struct hash_zone *zone, + const struct uds_record_name *hash, + struct hash_lock *replace_lock, + struct hash_lock **lock_ptr) +{ + struct hash_lock *lock, *new_lock; + int result; + + /* + * Borrow and prepare a lock from the pool so we don't have to do two pointer_map accesses + * in the common case of no lock contention. + */ + result = ASSERT(!list_empty(&zone->lock_pool), + "never need to wait for a free hash lock"); + if (result != VDO_SUCCESS) + return result; + + new_lock = list_entry(zone->lock_pool.prev, struct hash_lock, pool_node); + list_del_init(&new_lock->pool_node); + + /* + * Fill in the hash of the new lock so we can map it, since we have to use the hash as the + * map key. + */ + new_lock->hash = *hash; + + result = vdo_pointer_map_put(zone->hash_lock_map, &new_lock->hash, new_lock, + (replace_lock != NULL), (void **) &lock); + if (result != VDO_SUCCESS) { + return_hash_lock_to_pool(zone, uds_forget(new_lock)); + return result; + } + + if (replace_lock != NULL) { + /* On mismatch put the old lock back and return a severe error */ + ASSERT_LOG_ONLY(lock == replace_lock, + "old lock must have been in the lock map"); + /* TODO: Check earlier and bail out? */ + ASSERT_LOG_ONLY(replace_lock->registered, + "old lock must have been marked registered"); + replace_lock->registered = false; + } + + if (lock == replace_lock) { + lock = new_lock; + lock->registered = true; + } else { + /* There's already a lock for the hash, so we don't need the borrowed lock. */ + return_hash_lock_to_pool(zone, uds_forget(new_lock)); + } + + *lock_ptr = lock; + return VDO_SUCCESS; +} + +/** + * enter_forked_lock() - Bind the data_vio to a new hash lock. + * + * Implements waiter_callback_fn. Binds the data_vio that was waiting to a new hash lock and waits + * on that lock. + */ +static void enter_forked_lock(struct waiter *waiter, void *context) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct hash_lock *new_lock = context; + + set_hash_lock(data_vio, new_lock); + wait_on_hash_lock(new_lock, data_vio); +} + +/** + * fork_hash_lock() - Fork a hash lock because it has run out of increments on the duplicate PBN. + * @old_lock: The hash lock to fork. + * @new_agent: The data_vio that will be the agent for the new lock. + * + * Transfers the new agent and any lock waiters to a new hash lock instance which takes the place + * of the old lock in the lock map. The old lock remains active, but will not update advice. + */ +static void fork_hash_lock(struct hash_lock *old_lock, struct data_vio *new_agent) +{ + struct hash_lock *new_lock; + int result; + + result = acquire_lock(new_agent->hash_zone, &new_agent->record_name, old_lock, + &new_lock); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(new_agent, result); + return; + } + + /* + * Only one of the two locks should update UDS. The old lock is out of references, so it + * would be poor dedupe advice in the short term. + */ + old_lock->update_advice = false; + new_lock->update_advice = true; + + set_hash_lock(new_agent, new_lock); + new_lock->agent = new_agent; + + vdo_notify_all_waiters(&old_lock->waiters, enter_forked_lock, new_lock); + + new_agent->is_duplicate = false; + start_writing(new_lock, new_agent); +} + +/** + * launch_dedupe() - Reserve a reference count increment for a data_vio and launch it on the dedupe + * path. + * @lock: The hash lock. + * @data_vio: The data_vio to deduplicate using the hash lock. + * @has_claim: true if the data_vio already has claimed an increment from the duplicate lock. + * + * If no increments are available, this will roll over to a new hash lock and launch the data_vio + * as the writing agent for that lock. + */ +static void launch_dedupe(struct hash_lock *lock, struct data_vio *data_vio, + bool has_claim) +{ + if (!has_claim && !vdo_claim_pbn_lock_increment(lock->duplicate_lock)) { + /* Out of increments, so must roll over to a new lock. */ + fork_hash_lock(lock, data_vio); + return; + } + + /* Deduplicate against the lock's verified location. */ + set_duplicate_location(data_vio, lock->duplicate); + data_vio->new_mapped = data_vio->duplicate; + update_metadata_for_data_vio_write(data_vio, lock->duplicate_lock); +} + +/** + * start_deduping() - Enter the hash lock state where data_vios deduplicate in parallel against a + * true copy of their data on disk. + * @lock: The hash lock. + * @agent: The data_vio acting as the agent for the lock. + * @agent_is_done: true only if the agent has already written or deduplicated against its data. + * + * If the agent itself needs to deduplicate, an increment for it must already have been claimed + * from the duplicate lock, ensuring the hash lock will still have a data_vio holding it. + */ +static void start_deduping(struct hash_lock *lock, struct data_vio *agent, + bool agent_is_done) +{ + lock->state = VDO_HASH_LOCK_DEDUPING; + + /* + * We don't take the downgraded allocation lock from the agent unless we actually need to + * deduplicate against it. + */ + if (lock->duplicate_lock == NULL) { + ASSERT_LOG_ONLY(!vdo_is_state_compressed(agent->new_mapped.state), + "compression must have shared a lock"); + ASSERT_LOG_ONLY(agent_is_done, + "agent must have written the new duplicate"); + transfer_allocation_lock(agent); + } + + ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(lock->duplicate_lock), + "duplicate_lock must be a PBN read lock"); + + /* + * This state is not like any of the other states. There is no designated agent--the agent + * transitioning to this state and all the waiters will be launched to deduplicate in + * parallel. + */ + lock->agent = NULL; + + /* + * Launch the agent (if not already deduplicated) and as many lock waiters as we have + * available increments for on the dedupe path. If we run out of increments, rollover will + * be triggered and the remaining waiters will be transferred to the new lock. + */ + if (!agent_is_done) { + launch_dedupe(lock, agent, true); + agent = NULL; + } + while (vdo_has_waiters(&lock->waiters)) + launch_dedupe(lock, dequeue_lock_waiter(lock), false); + + if (agent_is_done) { + /* + * In the degenerate case where all the waiters rolled over to a new lock, this + * will continue to use the old agent to clean up this lock, and otherwise it just + * lets the agent exit the lock. + */ + finish_deduping(lock, agent); + } +} + +/** + * increment_stat() - Increment a statistic counter in a non-atomic yet thread-safe manner. + * @stat: The statistic field to increment. + */ +static void increment_stat(u64 *stat) +{ + /* + * Must only be mutated on the hash zone thread. Prevents any compiler shenanigans from + * affecting other threads reading stats. + */ + WRITE_ONCE(*stat, *stat + 1); +} + +/** + * finish_verifying() - Handle the result of the agent for the lock comparing its data to the + * duplicate candidate. + * @completion: The completion of the data_vio used to verify dedupe + * + * This continuation is registered in start_verifying(). + */ +static void finish_verifying(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + lock->verified = agent->is_duplicate; + + /* + * Only count the result of the initial verification of the advice as valid or stale, and + * not any re-verifications due to PBN lock releases. + */ + if (!lock->verify_counted) { + lock->verify_counted = true; + if (lock->verified) + increment_stat(&agent->hash_zone->statistics.dedupe_advice_valid); + else + increment_stat(&agent->hash_zone->statistics.dedupe_advice_stale); + } + + /* + * Even if the block is a verified duplicate, we can't start to deduplicate unless we can + * claim a reference count increment for the agent. + */ + if (lock->verified && !vdo_claim_pbn_lock_increment(lock->duplicate_lock)) { + agent->is_duplicate = false; + lock->verified = false; + } + + if (lock->verified) { + /* + * VERIFYING -> DEDUPING transition: The advice is for a true duplicate, so start + * deduplicating against it, if references are available. + */ + start_deduping(lock, agent, false); + } else { + /* + * VERIFYING -> UNLOCKING transition: Either the verify failed or we'd try to + * dedupe and roll over immediately, which would fail because it would leave the + * lock without an agent to release the PBN lock. In both cases, the data will have + * to be written or compressed, but first the advice PBN must be unlocked by the + * VERIFYING agent. + */ + lock->update_advice = true; + start_unlocking(lock, agent); + } +} + +static bool blocks_equal(char *block1, char *block2) +{ + int i; + + + for (i = 0; i < VDO_BLOCK_SIZE; i += sizeof(u64)) { + if (*((u64 *) &block1[i]) != *((u64 *) &block2[i])) + return false; + } + + return true; +} + +static void verify_callback(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + + agent->is_duplicate = blocks_equal(agent->vio.data, agent->scratch_block); + launch_data_vio_hash_zone_callback(agent, finish_verifying); +} + +static void uncompress_and_verify(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + int result; + + result = uncompress_data_vio(agent, agent->duplicate.state, + agent->scratch_block); + if (result == VDO_SUCCESS) { + verify_callback(completion); + return; + } + + agent->is_duplicate = false; + launch_data_vio_hash_zone_callback(agent, finish_verifying); +} + +static void verify_endio(struct bio *bio) +{ + struct data_vio *agent = vio_as_data_vio(bio->bi_private); + int result = blk_status_to_errno(bio->bi_status); + + vdo_count_completed_bios(bio); + if (result != VDO_SUCCESS) { + agent->is_duplicate = false; + launch_data_vio_hash_zone_callback(agent, finish_verifying); + return; + } + + if (vdo_is_state_compressed(agent->duplicate.state)) { + launch_data_vio_cpu_callback(agent, uncompress_and_verify, + CPU_Q_COMPRESS_BLOCK_PRIORITY); + return; + } + + launch_data_vio_cpu_callback(agent, verify_callback, + CPU_Q_COMPLETE_READ_PRIORITY); +} + +/** + * start_verifying() - Begin the data verification phase. + * @lock: The hash lock (must be LOCKING). + * @agent: The data_vio to use to read and compare candidate data. + * + * Continue the deduplication path for a hash lock by using the agent to read (and possibly + * decompress) the data at the candidate duplicate location, comparing it to the data in the agent + * to verify that the candidate is identical to all the data_vios sharing the hash. If so, it can + * be deduplicated against, otherwise a data_vio allocation will have to be written to and used for + * dedupe. + */ +static void start_verifying(struct hash_lock *lock, struct data_vio *agent) +{ + int result; + struct vio *vio = &agent->vio; + char *buffer = (vdo_is_state_compressed(agent->duplicate.state) ? + (char *) agent->compression.block : + agent->scratch_block); + + lock->state = VDO_HASH_LOCK_VERIFYING; + ASSERT_LOG_ONLY(!lock->verified, "hash lock only verifies advice once"); + + agent->last_async_operation = VIO_ASYNC_OP_VERIFY_DUPLICATION; + result = vio_reset_bio(vio, buffer, verify_endio, REQ_OP_READ, + agent->duplicate.pbn); + if (result != VDO_SUCCESS) { + set_data_vio_hash_zone_callback(agent, finish_verifying); + continue_data_vio_with_error(agent, result); + return; + } + + set_data_vio_bio_zone_callback(agent, process_vio_io); + vdo_launch_completion_with_priority(&vio->completion, BIO_Q_VERIFY_PRIORITY); +} + +/** + * finish_locking() - Handle the result of the agent for the lock attempting to obtain a PBN read + * lock on the candidate duplicate block. + * @completion: The completion of the data_vio that attempted to get the read lock. + * + * This continuation is registered in lock_duplicate_pbn(). + */ +static void finish_locking(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + if (!agent->is_duplicate) { + ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must not hold duplicate_lock if not flagged as a duplicate"); + /* + * LOCKING -> WRITING transition: The advice block is being modified or has no + * available references, so try to write or compress the data, remembering to + * update UDS later with the new advice. + */ + increment_stat(&agent->hash_zone->statistics.dedupe_advice_stale); + lock->update_advice = true; + start_writing(lock, agent); + return; + } + + ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, + "must hold duplicate_lock if flagged as a duplicate"); + + if (!lock->verified) { + /* + * LOCKING -> VERIFYING transition: Continue on the unverified dedupe path, reading + * the candidate duplicate and comparing it to the agent's data to decide whether + * it is a true duplicate or stale advice. + */ + start_verifying(lock, agent); + return; + } + + if (!vdo_claim_pbn_lock_increment(lock->duplicate_lock)) { + /* + * LOCKING -> UNLOCKING transition: The verified block was re-locked, but has no + * available increments left. Must first release the useless PBN read lock before + * rolling over to a new copy of the block. + */ + agent->is_duplicate = false; + lock->verified = false; + lock->update_advice = true; + start_unlocking(lock, agent); + return; + } + + /* + * LOCKING -> DEDUPING transition: Continue on the verified dedupe path, deduplicating + * against a location that was previously verified or written to. + */ + start_deduping(lock, agent, false); +} + +static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_lock *lock, + struct slab_depot *depot) +{ + /* Ensure that the newly-locked block is referenced. */ + struct vdo_slab *slab = vdo_get_slab(depot, agent->duplicate.pbn); + int result = vdo_acquire_provisional_reference(slab, agent->duplicate.pbn, lock); + + if (result == VDO_SUCCESS) + return true; + + uds_log_warning_strerror(result, + "Error acquiring provisional reference for dedupe candidate; aborting dedupe"); + agent->is_duplicate = false; + vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, + agent->duplicate.pbn, lock); + continue_data_vio_with_error(agent, result); + return false; +} + +/** + * lock_duplicate_pbn() - Acquire a read lock on the PBN of the block containing candidate + * duplicate data (compressed or uncompressed). + * @completion: The completion of the data_vio attempting to acquire the physical block lock on + * behalf of its hash lock. + * + * If the PBN is already locked for writing, the lock attempt is abandoned and is_duplicate will be + * cleared before calling back. this continuation is launched from start_locking(), and calls back + * to finish_locking() on the hash zone thread. + */ +static void lock_duplicate_pbn(struct vdo_completion *completion) +{ + unsigned int increment_limit; + struct pbn_lock *lock; + int result; + + struct data_vio *agent = as_data_vio(completion); + struct slab_depot *depot = vdo_from_data_vio(agent)->depot; + struct physical_zone *zone = agent->duplicate.zone; + + assert_data_vio_in_duplicate_zone(agent); + + set_data_vio_hash_zone_callback(agent, finish_locking); + + /* + * While in the zone that owns it, find out how many additional references can be made to + * the block if it turns out to truly be a duplicate. + */ + increment_limit = vdo_get_increment_limit(depot, agent->duplicate.pbn); + if (increment_limit == 0) { + /* + * We could deduplicate against it later if a reference happened to be released + * during verification, but it's probably better to bail out now. + */ + agent->is_duplicate = false; + continue_data_vio(agent); + return; + } + + result = vdo_attempt_physical_zone_pbn_lock(zone, agent->duplicate.pbn, + VIO_READ_LOCK, &lock); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(agent, result); + return; + } + + if (!vdo_is_pbn_read_lock(lock)) { + /* + * There are three cases of write locks: uncompressed data block writes, compressed + * (packed) block writes, and block map page writes. In all three cases, we give up + * on trying to verify the advice and don't bother to try deduplicate against the + * data in the write lock holder. + * + * 1) We don't ever want to try to deduplicate against a block map page. + * + * 2a) It's very unlikely we'd deduplicate against an entire packed block, both + * because of the chance of matching it, and because we don't record advice for it, + * but for the uncompressed representation of all the fragments it contains. The + * only way we'd be getting lock contention is if we've written the same + * representation coincidentally before, had it become unreferenced, and it just + * happened to be packed together from compressed writes when we go to verify the + * lucky advice. Giving up is a minuscule loss of potential dedupe. + * + * 2b) If the advice is for a slot of a compressed block, it's about to get + * smashed, and the write smashing it cannot contain our data--it would have to be + * writing on behalf of our hash lock, but that's impossible since we're the lock + * agent. + * + * 3a) If the lock is held by a data_vio with different data, the advice is already + * stale or is about to become stale. + * + * 3b) If the lock is held by a data_vio that matches us, we may as well either + * write it ourselves (or reference the copy we already wrote) instead of + * potentially having many duplicates wait for the lock holder to write, journal, + * hash, and finally arrive in the hash lock. We lose a chance to avoid a UDS + * update in the very rare case of advice for a free block that just happened to be + * allocated to a data_vio with the same hash. There's also a chance to save on a + * block write, at the cost of a block verify. Saving on a full block compare in + * all stale advice cases almost certainly outweighs saving a UDS update and + * trading a write for a read in a lucky case where advice would have been saved + * from becoming stale. + */ + agent->is_duplicate = false; + continue_data_vio(agent); + return; + } + + if (lock->holder_count == 0) { + if (!acquire_provisional_reference(agent, lock, depot)) + return; + + /* + * The increment limit we grabbed earlier is still valid. The lock now holds the + * rights to acquire all those references. Those rights will be claimed by hash + * locks sharing this read lock. + */ + lock->increment_limit = increment_limit; + } + + /* + * We've successfully acquired a read lock on behalf of the hash lock, so mark it as such. + */ + set_duplicate_lock(agent->hash_lock, lock); + + /* + * TODO: Optimization: We could directly launch the block verify, then switch to a hash + * thread. + */ + continue_data_vio(agent); +} + +/** + * start_locking() - Continue deduplication for a hash lock that has obtained valid advice of a + * potential duplicate through its agent. + * @lock: The hash lock (currently must be QUERYING). + * @agent: The data_vio bearing the dedupe advice. + */ +static void start_locking(struct hash_lock *lock, struct data_vio *agent) +{ + ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must not acquire a duplicate lock when already holding it"); + + lock->state = VDO_HASH_LOCK_LOCKING; + + /* + * TODO: Optimization: If we arrange to continue on the duplicate zone thread when + * accepting the advice, and don't explicitly change lock states (or use an agent-local + * state, or an atomic), we can avoid a thread transition here. + */ + agent->last_async_operation = VIO_ASYNC_OP_LOCK_DUPLICATE_PBN; + launch_data_vio_duplicate_zone_callback(agent, lock_duplicate_pbn); +} + +/** + * finish_writing() - Re-entry point for the lock agent after it has finished writing or + * compressing its copy of the data block. + * @lock: The hash lock, which must be in state WRITING. + * @agent: The data_vio that wrote its data for the lock. + * + * The agent will never need to dedupe against anything, so it's done with the lock, but the lock + * may not be finished with it, as a UDS update might still be needed. + * + * If there are other lock holders, the agent will hand the job to one of them and exit, leaving + * the lock to deduplicate against the just-written block. If there are no other lock holders, the + * agent either exits (and later tears down the hash lock), or it remains the agent and updates + * UDS. + */ +static void finish_writing(struct hash_lock *lock, struct data_vio *agent) +{ + /* + * Dedupe against the data block or compressed block slot the agent wrote. Since we know + * the write succeeded, there's no need to verify it. + */ + lock->duplicate = agent->new_mapped; + lock->verified = true; + + if (vdo_is_state_compressed(lock->duplicate.state) && + lock->registered) + /* + * Compression means the location we gave in the UDS query is not the location + * we're using to deduplicate. + */ + lock->update_advice = true; + + /* If there are any waiters, we need to start deduping them. */ + if (vdo_has_waiters(&lock->waiters)) { + /* + * WRITING -> DEDUPING transition: an asynchronously-written block failed to + * compress, so the PBN lock on the written copy was already transferred. The agent + * is done with the lock, but the lock may still need to use it to clean up after + * rollover. + */ + start_deduping(lock, agent, true); + return; + } + + /* + * There are no waiters and the agent has successfully written, so take a step towards + * being able to release the hash lock (or just release it). + */ + if (lock->update_advice) { + /* + * WRITING -> UPDATING transition: There's no waiter and a UDS update is needed, so + * retain the WRITING agent and use it to launch the update. The happens on + * compression, rollover, or the QUERYING agent not having an allocation. + */ + start_updating(lock, agent); + } else if (lock->duplicate_lock != NULL) { + /* + * WRITING -> UNLOCKING transition: There's no waiter and no update needed, but the + * compressed write gave us a shared duplicate lock that we must release. + */ + set_duplicate_location(agent, lock->duplicate); + start_unlocking(lock, agent); + } else { + /* + * WRITING -> BYPASSING transition: There's no waiter, no update needed, and no + * duplicate lock held, so both the agent and lock have no more work to do. The + * agent will release its allocation lock in cleanup. + */ + start_bypassing(lock, agent); + } +} + +/** + * select_writing_agent() - Search through the lock waiters for a data_vio that has an allocation. + * @lock: The hash lock to modify. + * + * If an allocation is found, swap agents, put the old agent at the head of the wait queue, then + * return the new agent. Otherwise, just return the current agent. + */ +static struct data_vio *select_writing_agent(struct hash_lock *lock) +{ + struct wait_queue temp_queue; + struct data_vio *data_vio; + + vdo_initialize_wait_queue(&temp_queue); + + /* + * Move waiters to the temp queue one-by-one until we find an allocation. Not ideal to + * search, but it only happens when nearly out of space. + */ + while (((data_vio = dequeue_lock_waiter(lock)) != NULL) && + !data_vio_has_allocation(data_vio)) { + /* Use the lower-level enqueue since we're just moving waiters around. */ + vdo_enqueue_waiter(&temp_queue, &data_vio->waiter); + } + + if (data_vio != NULL) { + /* + * Move the rest of the waiters over to the temp queue, preserving the order they + * arrived at the lock. + */ + vdo_transfer_all_waiters(&lock->waiters, &temp_queue); + + /* + * The current agent is being replaced and will have to wait to dedupe; make it the + * first waiter since it was the first to reach the lock. + */ + vdo_enqueue_waiter(&lock->waiters, &lock->agent->waiter); + lock->agent = data_vio; + } else { + /* No one has an allocation, so keep the current agent. */ + data_vio = lock->agent; + } + + /* Swap all the waiters back onto the lock's queue. */ + vdo_transfer_all_waiters(&temp_queue, &lock->waiters); + return data_vio; +} + +/** + * start_writing() - Begin the non-duplicate write path. + * @lock: The hash lock (currently must be QUERYING). + * @agent: The data_vio currently acting as the agent for the lock. + * + * Begins the non-duplicate write path for a hash lock that had no advice, selecting a data_vio + * with an allocation as a new agent, if necessary, then resuming the agent on the data_vio write + * path. + */ +static void start_writing(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_WRITING; + + /* + * The agent might not have received an allocation and so can't be used for writing, but + * it's entirely possible that one of the waiters did. + */ + if (!data_vio_has_allocation(agent)) { + agent = select_writing_agent(lock); + /* If none of the waiters had an allocation, the writes all have to fail. */ + if (!data_vio_has_allocation(agent)) { + /* + * TODO: Should we keep a variant of BYPASSING that causes new arrivals to + * fail immediately if they don't have an allocation? It might be possible + * that on some path there would be non-waiters still referencing the lock, + * so it would remain in the map as everything is currently spelled, even + * if the agent and all waiters release. + */ + continue_data_vio_with_error(agent, VDO_NO_SPACE); + return; + } + } + + /* + * If the agent compresses, it might wait indefinitely in the packer, which would be bad if + * there are any other data_vios waiting. + */ + if (vdo_has_waiters(&lock->waiters)) + cancel_data_vio_compression(agent); + + /* + * Send the agent to the compress/pack/write path in vioWrite. If it succeeds, it will + * return to the hash lock via vdo_continue_hash_lock() and call finish_writing(). + */ + launch_compress_data_vio(agent); +} + +/* + * Decode VDO duplicate advice from the old_metadata field of a UDS request. + * Returns true if valid advice was found and decoded + */ +static bool decode_uds_advice(struct dedupe_context *context) +{ + const struct uds_request *request = &context->request; + struct data_vio *data_vio = context->requestor; + size_t offset = 0; + const struct uds_record_data *encoding = &request->old_metadata; + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct zoned_pbn *advice = &data_vio->duplicate; + u8 version; + int result; + + if ((request->status != UDS_SUCCESS) || !request->found) + return false; + + version = encoding->data[offset++]; + if (version != UDS_ADVICE_VERSION) { + uds_log_error("invalid UDS advice version code %u", version); + return false; + } + + advice->state = encoding->data[offset++]; + advice->pbn = get_unaligned_le64(&encoding->data[offset]); + offset += sizeof(u64); + BUG_ON(offset != UDS_ADVICE_SIZE); + + /* Don't use advice that's clearly meaningless. */ + if ((advice->state == VDO_MAPPING_STATE_UNMAPPED) || (advice->pbn == VDO_ZERO_BLOCK)) { + uds_log_debug("Invalid advice from deduplication server: pbn %llu, state %u. Giving up on deduplication of logical block %llu", + (unsigned long long) advice->pbn, advice->state, + (unsigned long long) data_vio->logical.lbn); + atomic64_inc(&vdo->stats.invalid_advice_pbn_count); + return false; + } + + result = vdo_get_physical_zone(vdo, advice->pbn, &advice->zone); + if ((result != VDO_SUCCESS) || (advice->zone == NULL)) { + uds_log_debug("Invalid physical block number from deduplication server: %llu, giving up on deduplication of logical block %llu", + (unsigned long long) advice->pbn, + (unsigned long long) data_vio->logical.lbn); + atomic64_inc(&vdo->stats.invalid_advice_pbn_count); + return false; + } + + return true; +} + +static void process_query_result(struct data_vio *agent) +{ + struct dedupe_context *context = agent->dedupe_context; + + if (context == NULL) + return; + + if (change_context_state(context, DEDUPE_CONTEXT_COMPLETE, DEDUPE_CONTEXT_IDLE)) { + agent->is_duplicate = decode_uds_advice(context); + release_context(context); + } +} + +/** + * finish_querying() - Process the result of a UDS query performed by the agent for the lock. + * @completion: The completion of the data_vio that performed the query. + * + * This continuation is registered in start_querying(). + */ +static void finish_querying(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + process_query_result(agent); + + if (agent->is_duplicate) { + lock->duplicate = agent->duplicate; + /* + * QUERYING -> LOCKING transition: Valid advice was obtained from UDS. Use the + * QUERYING agent to start the hash lock on the unverified dedupe path, verifying + * that the advice can be used. + */ + start_locking(lock, agent); + } else { + /* + * The agent will be used as the duplicate if has an allocation; if it does, that + * location was posted to UDS, so no update will be needed. + */ + lock->update_advice = !data_vio_has_allocation(agent); + /* + * QUERYING -> WRITING transition: There was no advice or the advice wasn't valid, + * so try to write or compress the data. + */ + start_writing(lock, agent); + } +} + +/** + * start_querying() - Start deduplication for a hash lock. + * @lock: The initialized hash lock. + * @data_vio: The data_vio that has just obtained the new lock. + * + * Starts deduplication for a hash lock that has finished initializing by making the data_vio that + * requested it the agent, entering the QUERYING state, and using the agent to perform the UDS + * query on behalf of the lock. + */ +static void start_querying(struct hash_lock *lock, struct data_vio *data_vio) +{ + lock->agent = data_vio; + lock->state = VDO_HASH_LOCK_QUERYING; + data_vio->last_async_operation = VIO_ASYNC_OP_CHECK_FOR_DUPLICATION; + set_data_vio_hash_zone_callback(data_vio, finish_querying); + query_index(data_vio, + (data_vio_has_allocation(data_vio) ? UDS_POST : UDS_QUERY)); +} + +/** + * report_bogus_lock_state() - Complain that a data_vio has entered a hash_lock that is in an + * unimplemented or unusable state and continue the data_vio with an + * error. + * @lock: The hash lock. + * @data_vio: The data_vio attempting to enter the lock. + */ +static void report_bogus_lock_state(struct hash_lock *lock, struct data_vio *data_vio) +{ + ASSERT_LOG_ONLY(false, "hash lock must not be in unimplemented state %s", + get_hash_lock_state_name(lock->state)); + continue_data_vio_with_error(data_vio, VDO_LOCK_ERROR); +} + +/** + * vdo_continue_hash_lock() - Continue the processing state after writing, compressing, or + * deduplicating. + * @data_vio: The data_vio to continue processing in its hash lock. + * + * Asynchronously continue processing a data_vio in its hash lock after it has finished writing, + * compressing, or deduplicating, so it can share the result with any data_vios waiting in the hash + * lock, or update the UDS index, or simply release its share of the lock. + * + * Context: This must only be called in the correct thread for the hash zone. + */ +void vdo_continue_hash_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct hash_lock *lock = data_vio->hash_lock; + + switch (lock->state) { + case VDO_HASH_LOCK_WRITING: + ASSERT_LOG_ONLY(data_vio == lock->agent, + "only the lock agent may continue the lock"); + finish_writing(lock, data_vio); + break; + + case VDO_HASH_LOCK_DEDUPING: + finish_deduping(lock, data_vio); + break; + + case VDO_HASH_LOCK_BYPASSING: + /* This data_vio has finished the write path and the lock doesn't need it. */ + exit_hash_lock(data_vio); + break; + + case VDO_HASH_LOCK_INITIALIZING: + case VDO_HASH_LOCK_QUERYING: + case VDO_HASH_LOCK_UPDATING: + case VDO_HASH_LOCK_LOCKING: + case VDO_HASH_LOCK_VERIFYING: + case VDO_HASH_LOCK_UNLOCKING: + /* A lock in this state should never be re-entered. */ + report_bogus_lock_state(lock, data_vio); + break; + + default: + report_bogus_lock_state(lock, data_vio); + } +} + +/** + * is_hash_collision() - Check to see if a hash collision has occurred. + * @lock: The lock to check. + * @candidate: The data_vio seeking to share the lock. + * + * Check whether the data in data_vios sharing a lock is different than in a data_vio seeking to + * share the lock, which should only be possible in the extremely unlikely case of a hash + * collision. + * + * Return: true if the given data_vio must not share the lock because it doesn't have the same data + * as the lock holders. + */ +static bool is_hash_collision(struct hash_lock *lock, struct data_vio *candidate) +{ + struct data_vio *lock_holder; + struct hash_zone *zone; + bool collides; + + if (list_empty(&lock->duplicate_ring)) + return false; + + lock_holder = list_first_entry(&lock->duplicate_ring, struct data_vio, + hash_lock_entry); + zone = candidate->hash_zone; + collides = !blocks_equal(lock_holder->vio.data, candidate->vio.data); + if (collides) + increment_stat(&zone->statistics.concurrent_hash_collisions); + else + increment_stat(&zone->statistics.concurrent_data_matches); + + return collides; +} + +static inline int assert_hash_lock_preconditions(const struct data_vio *data_vio) +{ + int result; + + /* FIXME: BUG_ON() and/or enter read-only mode? */ + result = ASSERT(data_vio->hash_lock == NULL, + "must not already hold a hash lock"); + if (result != VDO_SUCCESS) + return result; + + result = ASSERT(list_empty(&data_vio->hash_lock_entry), + "must not already be a member of a hash lock ring"); + if (result != VDO_SUCCESS) + return result; + + return ASSERT(data_vio->recovery_sequence_number == 0, + "must not hold a recovery lock when getting a hash lock"); +} + +/** + * vdo_acquire_hash_lock() - Acquire or share a lock on a record name. + * @data_vio: The data_vio acquiring a lock on its record name. + * + * Acquire or share a lock on the hash (record name) of the data in a data_vio, updating the + * data_vio to reference the lock. This must only be called in the correct thread for the zone. In + * the unlikely case of a hash collision, this function will succeed, but the data_vio will not get + * a lock reference. + */ +void vdo_acquire_hash_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct hash_lock *lock; + int result; + + assert_data_vio_in_hash_zone(data_vio); + + result = assert_hash_lock_preconditions(data_vio); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + result = acquire_lock(data_vio->hash_zone, &data_vio->record_name, NULL, &lock); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + if (is_hash_collision(lock, data_vio)) { + /* + * Hash collisions are extremely unlikely, but the bogus dedupe would be a data + * corruption. Bypass optimization entirely. We can't compress a data_vio without + * a hash_lock as the compressed write depends on the hash_lock to manage the + * references for the compressed block. + */ + write_data_vio(data_vio); + return; + } + + set_hash_lock(data_vio, lock); + switch (lock->state) { + case VDO_HASH_LOCK_INITIALIZING: + start_querying(lock, data_vio); + return; + + case VDO_HASH_LOCK_QUERYING: + case VDO_HASH_LOCK_WRITING: + case VDO_HASH_LOCK_UPDATING: + case VDO_HASH_LOCK_LOCKING: + case VDO_HASH_LOCK_VERIFYING: + case VDO_HASH_LOCK_UNLOCKING: + /* The lock is busy, and can't be shared yet. */ + wait_on_hash_lock(lock, data_vio); + return; + + case VDO_HASH_LOCK_BYPASSING: + /* We can't use this lock, so bypass optimization entirely. */ + vdo_release_hash_lock(data_vio); + write_data_vio(data_vio); + return; + + case VDO_HASH_LOCK_DEDUPING: + launch_dedupe(lock, data_vio, false); + return; + + default: + /* A lock in this state should not be acquired by new VIOs. */ + report_bogus_lock_state(lock, data_vio); + } +} + +/** + * vdo_release_hash_lock() - Release a data_vio's share of a hash lock, if held, and null out the + * data_vio's reference to it. + * @data_vio: The data_vio releasing its hash lock. + * + * If the data_vio is the only one holding the lock, this also releases any resources or locks used + * by the hash lock (such as a PBN read lock on a block containing data with the same hash) and + * returns the lock to the hash zone's lock pool. + * + * Context: This must only be called in the correct thread for the hash zone. + */ +void vdo_release_hash_lock(struct data_vio *data_vio) +{ + struct hash_lock *lock = data_vio->hash_lock; + struct hash_zone *zone = data_vio->hash_zone; + + if (lock == NULL) + return; + + set_hash_lock(data_vio, NULL); + + if (lock->reference_count > 0) { + /* The lock is still in use by other data_vios. */ + return; + } + + if (lock->registered) { + struct hash_lock *removed; + + removed = vdo_pointer_map_remove(zone->hash_lock_map, &lock->hash); + ASSERT_LOG_ONLY(lock == removed, + "hash lock being released must have been mapped"); + } else { + ASSERT_LOG_ONLY(lock != vdo_pointer_map_get(zone->hash_lock_map, &lock->hash), + "unregistered hash lock must not be in the lock map"); + } + + ASSERT_LOG_ONLY(!vdo_has_waiters(&lock->waiters), + "hash lock returned to zone must have no waiters"); + ASSERT_LOG_ONLY((lock->duplicate_lock == NULL), + "hash lock returned to zone must not reference a PBN lock"); + ASSERT_LOG_ONLY((lock->state == VDO_HASH_LOCK_BYPASSING), + "returned hash lock must not be in use with state %s", + get_hash_lock_state_name(lock->state)); + ASSERT_LOG_ONLY(list_empty(&lock->pool_node), + "hash lock returned to zone must not be in a pool ring"); + ASSERT_LOG_ONLY(list_empty(&lock->duplicate_ring), + "hash lock returned to zone must not reference DataVIOs"); + + return_hash_lock_to_pool(zone, lock); +} + +/** + * transfer_allocation_lock() - Transfer a data_vio's downgraded allocation PBN lock to the + * data_vio's hash lock, converting it to a duplicate PBN lock. + * @data_vio: The data_vio holding the allocation lock to transfer. + */ +static void transfer_allocation_lock(struct data_vio *data_vio) +{ + struct allocation *allocation = &data_vio->allocation; + struct hash_lock *hash_lock = data_vio->hash_lock; + + ASSERT_LOG_ONLY(data_vio->new_mapped.pbn == allocation->pbn, + "transferred lock must be for the block written"); + + allocation->pbn = VDO_ZERO_BLOCK; + + ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(allocation->lock), + "must have downgraded the allocation lock before transfer"); + + hash_lock->duplicate = data_vio->new_mapped; + data_vio->duplicate = data_vio->new_mapped; + + /* + * Since the lock is being transferred, the holder count doesn't change (and isn't even + * safe to examine on this thread). + */ + hash_lock->duplicate_lock = uds_forget(allocation->lock); +} + +/** + * vdo_share_compressed_write_lock() - Make a data_vio's hash lock a shared holder of the PBN lock + * on the compressed block to which its data was just written. + * @data_vio: The data_vio which was just compressed. + * @pbn_lock: The PBN lock on the compressed block. + * + * If the lock is still a write lock (as it will be for the first share), it will be converted to a + * read lock. This also reserves a reference count increment for the data_vio. + */ +void vdo_share_compressed_write_lock(struct data_vio *data_vio, + struct pbn_lock *pbn_lock) +{ + bool claimed; + + ASSERT_LOG_ONLY(vdo_get_duplicate_lock(data_vio) == NULL, + "a duplicate PBN lock should not exist when writing"); + ASSERT_LOG_ONLY(vdo_is_state_compressed(data_vio->new_mapped.state), + "lock transfer must be for a compressed write"); + assert_data_vio_in_new_mapped_zone(data_vio); + + /* First sharer downgrades the lock. */ + if (!vdo_is_pbn_read_lock(pbn_lock)) + vdo_downgrade_pbn_write_lock(pbn_lock, true); + + /* + * Get a share of the PBN lock, ensuring it cannot be released until after this data_vio + * has had a chance to journal a reference. + */ + data_vio->duplicate = data_vio->new_mapped; + data_vio->hash_lock->duplicate = data_vio->new_mapped; + set_duplicate_lock(data_vio->hash_lock, pbn_lock); + + /* + * Claim a reference for this data_vio. Necessary since another hash_lock might start + * deduplicating against it before our incRef. + */ + claimed = vdo_claim_pbn_lock_increment(pbn_lock); + ASSERT_LOG_ONLY(claimed, "impossible to fail to claim an initial increment"); +} + +/** compare_keys() - Implements pointer_key_comparator_fn. */ +static bool compare_keys(const void *this_key, const void *that_key) +{ + /* Null keys are not supported. */ + return (memcmp(this_key, that_key, sizeof(struct uds_record_name)) == 0); +} + +/** hash_key() - Implements pointer_key_comparator_fn. */ +static u32 hash_key(const void *key) +{ + const struct uds_record_name *name = key; + + /* Use a fragment of the record name as a hash code. */ + return get_unaligned_le32(&name->name[4]); +} + +static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zones, + zone_count_t zone_number) +{ + int result; + data_vio_count_t i; + struct hash_zone *zone = &zones->zones[zone_number]; + + result = vdo_make_pointer_map(VDO_LOCK_MAP_CAPACITY, 0, compare_keys, + hash_key, &zone->hash_lock_map); + if (result != VDO_SUCCESS) + return result; + + vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.hash_zone_threads[zone_number]; + vdo_initialize_completion(&zone->completion, vdo, VDO_HASH_ZONE_COMPLETION); + vdo_set_completion_callback(&zone->completion, timeout_index_operations_callback, + zone->thread_id); + INIT_LIST_HEAD(&zone->lock_pool); + result = uds_allocate(LOCK_POOL_CAPACITY, struct hash_lock, "hash_lock array", + &zone->lock_array); + if (result != VDO_SUCCESS) + return result; + + for (i = 0; i < LOCK_POOL_CAPACITY; i++) + return_hash_lock_to_pool(zone, &zone->lock_array[i]); + + INIT_LIST_HEAD(&zone->available); + INIT_LIST_HEAD(&zone->pending); + result = uds_make_funnel_queue(&zone->timed_out_complete); + if (result != VDO_SUCCESS) + return result; + + timer_setup(&zone->timer, timeout_index_operations, 0); + + for (i = 0; i < MAXIMUM_VDO_USER_VIOS; i++) { + struct dedupe_context *context = &zone->contexts[i]; + + context->zone = zone; + context->request.callback = finish_index_operation; + context->request.session = zones->index_session; + list_add(&context->list_entry, &zone->available); + } + + return vdo_make_default_thread(vdo, zone->thread_id); +} + +/** get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */ +static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number) +{ + struct hash_zones *zones = context; + + return zones->zones[zone_number].thread_id; +} + +/** + * vdo_make_hash_zones() - Create the hash zones. + * + * @vdo: The vdo to which the zone will belong. + * @zones_ptr: A pointer to hold the zones. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr) +{ + int result; + struct hash_zones *zones; + zone_count_t z; + zone_count_t zone_count = vdo->thread_config.hash_zone_count; + + if (zone_count == 0) + return VDO_SUCCESS; + + result = uds_allocate_extended(struct hash_zones, zone_count, struct hash_zone, + __func__, &zones); + if (result != VDO_SUCCESS) + return result; + + result = initialize_index(vdo, zones); + if (result != VDO_SUCCESS) { + uds_free(zones); + return result; + } + + vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NEW); + + zones->zone_count = zone_count; + for (z = 0; z < zone_count; z++) { + result = initialize_zone(vdo, zones, z); + if (result != VDO_SUCCESS) { + vdo_free_hash_zones(zones); + return result; + } + } + + result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone, + vdo->thread_config.admin_thread, zones, NULL, + vdo, &zones->manager); + if (result != VDO_SUCCESS) { + vdo_free_hash_zones(zones); + return result; + } + + *zones_ptr = zones; + return VDO_SUCCESS; +} + +void vdo_finish_dedupe_index(struct hash_zones *zones) +{ + if (zones == NULL) + return; + + uds_destroy_index_session(uds_forget(zones->index_session)); +} + +/** + * vdo_free_hash_zones() - Free the hash zones. + * @zones: The zone to free. + */ +void vdo_free_hash_zones(struct hash_zones *zones) +{ + zone_count_t i; + + if (zones == NULL) + return; + + uds_free(uds_forget(zones->manager)); + + for (i = 0; i < zones->zone_count; i++) { + struct hash_zone *zone = &zones->zones[i]; + + uds_free_funnel_queue(uds_forget(zone->timed_out_complete)); + vdo_free_pointer_map(uds_forget(zone->hash_lock_map)); + uds_free(uds_forget(zone->lock_array)); + } + + if (zones->index_session != NULL) + vdo_finish_dedupe_index(zones); + + ratelimit_state_exit(&zones->ratelimiter); + if (vdo_get_admin_state_code(&zones->state) == VDO_ADMIN_STATE_NEW) + uds_free(zones); + else + kobject_put(&zones->dedupe_directory); +} + +static void initiate_suspend_index(struct admin_state *state) +{ + struct hash_zones *zones = container_of(state, struct hash_zones, state); + enum index_state index_state; + + spin_lock(&zones->lock); + index_state = zones->index_state; + spin_unlock(&zones->lock); + + if (index_state != IS_CLOSED) { + bool save = vdo_is_state_saving(&zones->state); + int result; + + result = uds_suspend_index_session(zones->index_session, save); + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "Error suspending dedupe index"); + } + + vdo_finish_draining(state); +} + +/** + * suspend_index() - Suspend the UDS index prior to draining hash zones. + * + * Implements vdo_action_preamble_fn + */ +static void suspend_index(void *context, struct vdo_completion *completion) +{ + struct hash_zones *zones = context; + + vdo_start_draining(&zones->state, + vdo_get_current_manager_operation(zones->manager), completion, + initiate_suspend_index); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct hash_zone, state)); +} + +/** + * drain_hash_zone() - Drain a hash zone. + * + * Implements vdo_zone_action_fn. + */ +static void drain_hash_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct hash_zones *zones = context; + + vdo_start_draining(&zones->zones[zone_number].state, + vdo_get_current_manager_operation(zones->manager), parent, + initiate_drain); +} + +/** vdo_drain_hash_zones() - Drain all hash zones. */ +void vdo_drain_hash_zones(struct hash_zones *zones, struct vdo_completion *parent) +{ + vdo_schedule_operation(zones->manager, parent->vdo->suspend_type, suspend_index, + drain_hash_zone, NULL, parent); +} + +static void launch_dedupe_state_change(struct hash_zones *zones) +{ + /* ASSERTION: We enter with the lock held. */ + if (zones->changing || !vdo_is_state_normal(&zones->state)) + /* Either a change is already in progress, or changes are not allowed. */ + return; + + if (zones->create_flag || (zones->index_state != zones->index_target)) { + zones->changing = true; + vdo_launch_completion(&zones->completion); + return; + } + + /* ASSERTION: We exit with the lock held. */ +} + +/** + * resume_index() - Resume the UDS index prior to resuming hash zones. + * + * Implements vdo_action_preamble_fn + */ +static void resume_index(void *context, struct vdo_completion *parent) +{ + struct hash_zones *zones = context; + struct device_config *config = parent->vdo->device_config; + int result; + + zones->parameters.bdev = config->owned_device->bdev; + result = uds_resume_index_session(zones->index_session, zones->parameters.bdev); + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "Error resuming dedupe index"); + + spin_lock(&zones->lock); + vdo_resume_if_quiescent(&zones->state); + + if (config->deduplication) { + zones->index_target = IS_OPENED; + WRITE_ONCE(zones->dedupe_flag, true); + } else { + zones->index_target = IS_CLOSED; + } + + launch_dedupe_state_change(zones); + spin_unlock(&zones->lock); + + vdo_finish_completion(parent); +} + +/** + * resume_hash_zone() - Resume a hash zone. + * + * Implements vdo_zone_action_fn. + */ +static void resume_hash_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct hash_zone *zone = &(((struct hash_zones *) context)->zones[zone_number]); + + vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); +} + +/** + * vdo_resume_hash_zones() - Resume a set of hash zones. + * @zones: The hash zones to resume. + * @parent: The object to notify when the zones have resumed. + */ +void vdo_resume_hash_zones(struct hash_zones *zones, struct vdo_completion *parent) +{ + if (vdo_is_read_only(parent->vdo)) { + vdo_launch_completion(parent); + return; + } + + vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, resume_index, + resume_hash_zone, NULL, parent); +} + +/** + * get_hash_zone_statistics() - Add the statistics for this hash zone to the tally for all zones. + * @zone: The hash zone to query. + * @tally: The tally + */ +static void get_hash_zone_statistics(const struct hash_zone *zone, + struct hash_lock_statistics *tally) +{ + const struct hash_lock_statistics *stats = &zone->statistics; + + tally->dedupe_advice_valid += READ_ONCE(stats->dedupe_advice_valid); + tally->dedupe_advice_stale += READ_ONCE(stats->dedupe_advice_stale); + tally->concurrent_data_matches += READ_ONCE(stats->concurrent_data_matches); + tally->concurrent_hash_collisions += READ_ONCE(stats->concurrent_hash_collisions); + tally->curr_dedupe_queries += READ_ONCE(zone->active); +} + +static void get_index_statistics(struct hash_zones *zones, + struct index_statistics *stats) +{ + enum index_state state; + struct uds_index_stats index_stats; + int result; + + spin_lock(&zones->lock); + state = zones->index_state; + spin_unlock(&zones->lock); + + if (state != IS_OPENED) + return; + + result = uds_get_index_session_stats(zones->index_session, &index_stats); + if (result != UDS_SUCCESS) { + uds_log_error_strerror(result, "Error reading index stats"); + return; + } + + stats->entries_indexed = index_stats.entries_indexed; + stats->posts_found = index_stats.posts_found; + stats->posts_not_found = index_stats.posts_not_found; + stats->queries_found = index_stats.queries_found; + stats->queries_not_found = index_stats.queries_not_found; + stats->updates_found = index_stats.updates_found; + stats->updates_not_found = index_stats.updates_not_found; + stats->entries_discarded = index_stats.entries_discarded; +} + +/** + * vdo_get_dedupe_statistics() - Tally the statistics from all the hash zones and the UDS index. + * @hash_zones: The hash zones to query + * + * Return: The sum of the hash lock statistics from all hash zones plus the statistics from the UDS + * index + */ +void vdo_get_dedupe_statistics(struct hash_zones *zones, struct vdo_statistics *stats) + +{ + zone_count_t zone; + + for (zone = 0; zone < zones->zone_count; zone++) + get_hash_zone_statistics(&zones->zones[zone], &stats->hash_lock); + + get_index_statistics(zones, &stats->index); + + /* + * zones->timeouts gives the number of timeouts, and dedupe_context_busy gives the number + * of queries not made because of earlier timeouts. + */ + stats->dedupe_advice_timeouts = + (atomic64_read(&zones->timeouts) + atomic64_read(&zones->dedupe_context_busy)); +} + +/** + * vdo_select_hash_zone() - Select the hash zone responsible for locking a given record name. + * @zones: The hash_zones from which to select. + * @name: The record name. + * + * Return: The hash zone responsible for the record name. + */ +struct hash_zone *vdo_select_hash_zone(struct hash_zones *zones, + const struct uds_record_name *name) +{ + /* + * Use a fragment of the record name as a hash code. Eight bits of hash should suffice + * since the number of hash zones is small. + * TODO: Verify that the first byte is independent enough. + */ + u32 hash = name->name[0]; + + /* + * Scale the 8-bit hash fragment to a zone index by treating it as a binary fraction and + * multiplying that by the zone count. If the hash is uniformly distributed over [0 .. + * 2^8-1], then (hash * count / 2^8) should be uniformly distributed over [0 .. count-1]. + * The multiply and shift is much faster than a divide (modulus) on X86 CPUs. + */ + hash = (hash * zones->zone_count) >> 8; + return &zones->zones[hash]; +} + +/** + * dump_hash_lock() - Dump a compact description of hash_lock to the log if the lock is not on the + * free list. + * @lock: The hash lock to dump. + */ +static void dump_hash_lock(const struct hash_lock *lock) +{ + const char *state; + + if (!list_empty(&lock->pool_node)) { + /* This lock is on the free list. */ + return; + } + + /* + * Necessarily cryptic since we can log a lot of these. First three chars of state is + * unambiguous. 'U' indicates a lock not registered in the map. + */ + state = get_hash_lock_state_name(lock->state); + uds_log_info(" hl %px: %3.3s %c%llu/%u rc=%u wc=%zu agt=%px", + lock, state, (lock->registered ? 'D' : 'U'), + (unsigned long long) lock->duplicate.pbn, + lock->duplicate.state, lock->reference_count, + vdo_count_waiters(&lock->waiters), lock->agent); +} + +static const char *index_state_to_string(struct hash_zones *zones, + enum index_state state) +{ + if (!vdo_is_state_normal(&zones->state)) + return SUSPENDED; + + switch (state) { + case IS_CLOSED: + return zones->error_flag ? ERROR : CLOSED; + case IS_CHANGING: + return zones->index_target == IS_OPENED ? OPENING : CLOSING; + case IS_OPENED: + return READ_ONCE(zones->dedupe_flag) ? ONLINE : OFFLINE; + default: + return UNKNOWN; + } +} + +/** + * vdo_dump_hash_zone() - Dump information about a hash zone to the log for debugging. + * @zone: The zone to dump. + */ +static void dump_hash_zone(const struct hash_zone *zone) +{ + data_vio_count_t i; + + if (zone->hash_lock_map == NULL) { + uds_log_info("struct hash_zone %u: NULL map", zone->zone_number); + return; + } + + uds_log_info("struct hash_zone %u: mapSize=%zu", + zone->zone_number, vdo_pointer_map_size(zone->hash_lock_map)); + for (i = 0; i < LOCK_POOL_CAPACITY; i++) + dump_hash_lock(&zone->lock_array[i]); +} + +/** + * vdo_dump_hash_zones() - Dump information about the hash zones to the log for debugging. + * @zones: The zones to dump. + */ +void vdo_dump_hash_zones(struct hash_zones *zones) +{ + const char *state, *target; + zone_count_t zone; + + spin_lock(&zones->lock); + state = index_state_to_string(zones, zones->index_state); + target = (zones->changing ? index_state_to_string(zones, zones->index_target) : NULL); + spin_unlock(&zones->lock); + + uds_log_info("UDS index: state: %s", state); + if (target != NULL) + uds_log_info("UDS index: changing to state: %s", target); + + for (zone = 0; zone < zones->zone_count; zone++) + dump_hash_zone(&zones->zones[zone]); +} diff --git a/drivers/md/dm-vdo/dedupe.h b/drivers/md/dm-vdo/dedupe.h new file mode 100644 index 000000000000..f4b091efc27e --- /dev/null +++ b/drivers/md/dm-vdo/dedupe.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_DEDUPE_H +#define VDO_DEDUPE_H + +#include +#include + +#include "uds.h" + +#include "admin-state.h" +#include "constants.h" +#include "statistics.h" +#include "types.h" +#include "wait-queue.h" + +struct dedupe_context { + struct hash_zone *zone; + struct uds_request request; + struct list_head list_entry; + struct funnel_queue_entry queue_entry; + u64 submission_jiffies; + struct data_vio *requestor; + atomic_t state; +}; + +struct hash_lock; + +struct hash_zone { + /* Which hash zone this is */ + zone_count_t zone_number; + + /* The administrative state of the zone */ + struct admin_state state; + + /* The thread ID for this zone */ + thread_id_t thread_id; + + /* Mapping from record name fields to hash_locks */ + struct pointer_map *hash_lock_map; + + /* List containing all unused hash_locks */ + struct list_head lock_pool; + + /* + * Statistics shared by all hash locks in this zone. Only modified on the hash zone thread, + * but queried by other threads. + */ + struct hash_lock_statistics statistics; + + /* Array of all hash_locks */ + struct hash_lock *lock_array; + + /* These fields are used to manage the dedupe contexts */ + struct list_head available; + struct list_head pending; + struct funnel_queue *timed_out_complete; + struct timer_list timer; + struct vdo_completion completion; + unsigned int active; + atomic_t timer_state; + + /* The dedupe contexts for querying the index from this zone */ + struct dedupe_context contexts[MAXIMUM_VDO_USER_VIOS]; +}; + +struct hash_zones; + +struct pbn_lock * __must_check vdo_get_duplicate_lock(struct data_vio *data_vio); + +void vdo_acquire_hash_lock(struct vdo_completion *completion); +void vdo_continue_hash_lock(struct vdo_completion *completion); +void vdo_release_hash_lock(struct data_vio *data_vio); +void vdo_clean_failed_hash_lock(struct data_vio *data_vio); +void vdo_share_compressed_write_lock(struct data_vio *data_vio, + struct pbn_lock *pbn_lock); + +int __must_check vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr); + +void vdo_free_hash_zones(struct hash_zones *zones); + +void vdo_drain_hash_zones(struct hash_zones *zones, struct vdo_completion *parent); + +void vdo_get_dedupe_statistics(struct hash_zones *zones, struct vdo_statistics *stats); + +struct hash_zone * __must_check vdo_select_hash_zone(struct hash_zones *zones, + const struct uds_record_name *name); + +void vdo_dump_hash_zones(struct hash_zones *zones); + +#endif /* VDO_DEDUPE_H */ From b0530561331c12dd6abf480e09e3deff84899bd1 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:13:06 -0500 Subject: [PATCH 0571/1038] dm vdo: add use of deduplication index in hash zones Add the data and methods that manage queries to the deduplication index and the responses from the index. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dedupe.c | 625 +++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/dedupe.h | 26 ++ 2 files changed, 651 insertions(+) diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index f40cc9643b1c..4c4d697a4888 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -143,6 +143,48 @@ #include "vdo.h" #include "wait-queue.h" +struct uds_attribute { + struct attribute attr; + const char *(*show_string)(struct hash_zones *hash_zones); +}; + +enum timer_state { + DEDUPE_QUERY_TIMER_IDLE, + DEDUPE_QUERY_TIMER_RUNNING, + DEDUPE_QUERY_TIMER_FIRED, +}; + +enum dedupe_context_state { + DEDUPE_CONTEXT_IDLE, + DEDUPE_CONTEXT_PENDING, + DEDUPE_CONTEXT_TIMED_OUT, + DEDUPE_CONTEXT_COMPLETE, + DEDUPE_CONTEXT_TIMED_OUT_COMPLETE, +}; + +/* Possible index states: closed, opened, or transitioning between those two. */ +enum index_state { + IS_CLOSED, + IS_CHANGING, + IS_OPENED, +}; + +static const char *CLOSED = "closed"; +static const char *CLOSING = "closing"; +static const char *ERROR = "error"; +static const char *OFFLINE = "offline"; +static const char *ONLINE = "online"; +static const char *OPENING = "opening"; +static const char *SUSPENDED = "suspended"; +static const char *UNKNOWN = "unknown"; + +/* Version 2 uses the kernel space UDS index and is limited to 16 bytes */ +enum { + UDS_ADVICE_VERSION = 2, + /* version byte + state byte + 64-bit little-endian PBN */ + UDS_ADVICE_SIZE = 1 + 1 + sizeof(u64), +}; + enum hash_lock_state { /* State for locks that are not in use or are being initialized. */ VDO_HASH_LOCK_INITIALIZING, @@ -264,6 +306,13 @@ struct hash_zones { struct hash_zone zones[]; }; +/* These are in milliseconds. */ +unsigned int vdo_dedupe_index_timeout_interval = 5000; +unsigned int vdo_dedupe_index_min_timer_interval = 100; +/* Same two variables, in jiffies for easier consumption. */ +static u64 vdo_dedupe_index_timeout_jiffies; +static u64 vdo_dedupe_index_min_timer_jiffies; + static inline struct hash_zone *as_hash_zone(struct vdo_completion *completion) { vdo_assert_completion_type(completion, VDO_HASH_ZONE_COMPLETION); @@ -282,6 +331,16 @@ static inline void assert_in_hash_zone(struct hash_zone *zone, const char *name) "%s called on hash zone thread", name); } +static inline bool change_context_state(struct dedupe_context *context, int old, int new) +{ + return (atomic_cmpxchg(&context->state, old, new) == old); +} + +static inline bool change_timer_state(struct hash_zone *zone, int old, int new) +{ + return (atomic_cmpxchg(&zone->timer_state, old, new) == old); +} + /** * return_hash_lock_to_pool() - (Re)initialize a hash lock and return it to its pool. * @zone: The zone from which the lock was borrowed. @@ -1968,6 +2027,379 @@ static u32 hash_key(const void *key) return get_unaligned_le32(&name->name[4]); } +static void dedupe_kobj_release(struct kobject *directory) +{ + uds_free(container_of(directory, struct hash_zones, dedupe_directory)); +} + +static ssize_t dedupe_status_show(struct kobject *directory, struct attribute *attr, + char *buf) +{ + struct uds_attribute *ua = container_of(attr, struct uds_attribute, attr); + struct hash_zones *zones = container_of(directory, struct hash_zones, + dedupe_directory); + + if (ua->show_string != NULL) + return sprintf(buf, "%s\n", ua->show_string(zones)); + else + return -EINVAL; +} + +static ssize_t dedupe_status_store(struct kobject *kobj __always_unused, + struct attribute *attr __always_unused, + const char *buf __always_unused, + size_t length __always_unused) +{ + return -EINVAL; +} + +/*----------------------------------------------------------------------*/ + +static const struct sysfs_ops dedupe_sysfs_ops = { + .show = dedupe_status_show, + .store = dedupe_status_store, +}; + +static struct uds_attribute dedupe_status_attribute = { + .attr = {.name = "status", .mode = 0444, }, + .show_string = vdo_get_dedupe_index_state_name, +}; + +static struct attribute *dedupe_attrs[] = { + &dedupe_status_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dedupe); + +static const struct kobj_type dedupe_directory_type = { + .release = dedupe_kobj_release, + .sysfs_ops = &dedupe_sysfs_ops, + .default_groups = dedupe_groups, +}; + +static void start_uds_queue(void *ptr) +{ + /* + * Allow the UDS dedupe worker thread to do memory allocations. It will only do allocations + * during the UDS calls that open or close an index, but those allocations can safely sleep + * while reserving a large amount of memory. We could use an allocations_allowed boolean + * (like the base threads do), but it would be an unnecessary embellishment. + */ + struct vdo_thread *thread = vdo_get_work_queue_owner(vdo_get_current_work_queue()); + + uds_register_allocating_thread(&thread->allocating_thread, NULL); +} + +static void finish_uds_queue(void *ptr __always_unused) +{ + uds_unregister_allocating_thread(); +} + +static void close_index(struct hash_zones *zones) +{ + int result; + + /* + * Change the index state so that get_index_statistics() will not try to use the index + * session we are closing. + */ + zones->index_state = IS_CHANGING; + /* Close the index session, while not holding the lock. */ + spin_unlock(&zones->lock); + result = uds_close_index(zones->index_session); + + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "Error closing index"); + spin_lock(&zones->lock); + zones->index_state = IS_CLOSED; + zones->error_flag |= result != UDS_SUCCESS; + /* ASSERTION: We leave in IS_CLOSED state. */ +} + +static void open_index(struct hash_zones *zones) +{ + /* ASSERTION: We enter in IS_CLOSED state. */ + int result; + bool create_flag = zones->create_flag; + + zones->create_flag = false; + /* + * Change the index state so that the it will be reported to the outside world as + * "opening". + */ + zones->index_state = IS_CHANGING; + zones->error_flag = false; + + /* Open the index session, while not holding the lock */ + spin_unlock(&zones->lock); + result = uds_open_index(create_flag ? UDS_CREATE : UDS_LOAD, + &zones->parameters, zones->index_session); + if (result != UDS_SUCCESS) + uds_log_error_strerror(result, "Error opening index"); + + spin_lock(&zones->lock); + if (!create_flag) { + switch (result) { + case -ENOENT: + /* + * Either there is no index, or there is no way we can recover the index. + * We will be called again and try to create a new index. + */ + zones->index_state = IS_CLOSED; + zones->create_flag = true; + return; + default: + break; + } + } + if (result == UDS_SUCCESS) { + zones->index_state = IS_OPENED; + } else { + zones->index_state = IS_CLOSED; + zones->index_target = IS_CLOSED; + zones->error_flag = true; + spin_unlock(&zones->lock); + uds_log_info("Setting UDS index target state to error"); + spin_lock(&zones->lock); + } + /* + * ASSERTION: On success, we leave in IS_OPENED state. + * ASSERTION: On failure, we leave in IS_CLOSED state. + */ +} + +static void change_dedupe_state(struct vdo_completion *completion) +{ + struct hash_zones *zones = as_hash_zones(completion); + + spin_lock(&zones->lock); + + /* Loop until the index is in the target state and the create flag is clear. */ + while (vdo_is_state_normal(&zones->state) && + ((zones->index_state != zones->index_target) || zones->create_flag)) { + if (zones->index_state == IS_OPENED) + close_index(zones); + else + open_index(zones); + } + + zones->changing = false; + spin_unlock(&zones->lock); +} + +static void start_expiration_timer(struct dedupe_context *context) +{ + u64 start_time = context->submission_jiffies; + u64 end_time; + + if (!change_timer_state(context->zone, DEDUPE_QUERY_TIMER_IDLE, + DEDUPE_QUERY_TIMER_RUNNING)) + return; + + end_time = max(start_time + vdo_dedupe_index_timeout_jiffies, + jiffies + vdo_dedupe_index_min_timer_jiffies); + mod_timer(&context->zone->timer, end_time); +} + +/** + * report_dedupe_timeouts() - Record and eventually report that some dedupe requests reached their + * expiration time without getting answers, so we timed them out. + * @zones: the hash zones. + * @timeouts: the number of newly timed out requests. + */ +static void report_dedupe_timeouts(struct hash_zones *zones, unsigned int timeouts) +{ + atomic64_add(timeouts, &zones->timeouts); + spin_lock(&zones->lock); + if (__ratelimit(&zones->ratelimiter)) { + u64 unreported = atomic64_read(&zones->timeouts); + + unreported -= zones->reported_timeouts; + uds_log_debug("UDS index timeout on %llu requests", + (unsigned long long) unreported); + zones->reported_timeouts += unreported; + } + spin_unlock(&zones->lock); +} + +static int initialize_index(struct vdo *vdo, struct hash_zones *zones) +{ + int result; + off_t uds_offset; + struct volume_geometry geometry = vdo->geometry; + static const struct vdo_work_queue_type uds_queue_type = { + .start = start_uds_queue, + .finish = finish_uds_queue, + .max_priority = UDS_Q_MAX_PRIORITY, + .default_priority = UDS_Q_PRIORITY, + }; + + vdo_set_dedupe_index_timeout_interval(vdo_dedupe_index_timeout_interval); + vdo_set_dedupe_index_min_timer_interval(vdo_dedupe_index_min_timer_interval); + + /* + * Since we will save up the timeouts that would have been reported but were ratelimited, + * we don't need to report ratelimiting. + */ + ratelimit_default_init(&zones->ratelimiter); + ratelimit_set_flags(&zones->ratelimiter, RATELIMIT_MSG_ON_RELEASE); + uds_offset = ((vdo_get_index_region_start(geometry) - + geometry.bio_offset) * VDO_BLOCK_SIZE); + zones->parameters = (struct uds_parameters) { + .bdev = vdo->device_config->owned_device->bdev, + .offset = uds_offset, + .size = (vdo_get_index_region_size(geometry) * VDO_BLOCK_SIZE), + .memory_size = geometry.index_config.mem, + .sparse = geometry.index_config.sparse, + .nonce = (u64) geometry.nonce, + }; + + result = uds_create_index_session(&zones->index_session); + if (result != UDS_SUCCESS) + return result; + + result = vdo_make_thread(vdo, vdo->thread_config.dedupe_thread, &uds_queue_type, + 1, NULL); + if (result != VDO_SUCCESS) { + uds_destroy_index_session(uds_forget(zones->index_session)); + uds_log_error("UDS index queue initialization failed (%d)", result); + return result; + } + + vdo_initialize_completion(&zones->completion, vdo, VDO_HASH_ZONES_COMPLETION); + vdo_set_completion_callback(&zones->completion, change_dedupe_state, + vdo->thread_config.dedupe_thread); + kobject_init(&zones->dedupe_directory, &dedupe_directory_type); + return VDO_SUCCESS; +} + +/** + * finish_index_operation() - This is the UDS callback for index queries. + * @request: The uds request which has just completed. + */ +static void finish_index_operation(struct uds_request *request) +{ + struct dedupe_context *context = container_of(request, struct dedupe_context, + request); + + if (change_context_state(context, DEDUPE_CONTEXT_PENDING, + DEDUPE_CONTEXT_COMPLETE)) { + /* + * This query has not timed out, so send its data_vio back to its hash zone to + * process the results. + */ + continue_data_vio(context->requestor); + return; + } + + /* + * This query has timed out, so try to mark it complete and hence eligible for reuse. Its + * data_vio has already moved on. + */ + if (!change_context_state(context, DEDUPE_CONTEXT_TIMED_OUT, + DEDUPE_CONTEXT_TIMED_OUT_COMPLETE)) + ASSERT_LOG_ONLY(false, "uds request was timed out (state %d)", + atomic_read(&context->state)); + + uds_funnel_queue_put(context->zone->timed_out_complete, &context->queue_entry); +} + +/** + * check_for_drain_complete() - Check whether this zone has drained. + * @zone: The zone to check. + */ +static void check_for_drain_complete(struct hash_zone *zone) +{ + data_vio_count_t recycled = 0; + + if (!vdo_is_state_draining(&zone->state)) + return; + + if ((atomic_read(&zone->timer_state) == DEDUPE_QUERY_TIMER_IDLE) || + change_timer_state(zone, DEDUPE_QUERY_TIMER_RUNNING, + DEDUPE_QUERY_TIMER_IDLE)) { + del_timer_sync(&zone->timer); + } else { + /* + * There is an in flight time-out, which must get processed before we can continue. + */ + return; + } + + for (;;) { + struct dedupe_context *context; + struct funnel_queue_entry *entry; + + entry = uds_funnel_queue_poll(zone->timed_out_complete); + if (entry == NULL) + break; + + context = container_of(entry, struct dedupe_context, queue_entry); + atomic_set(&context->state, DEDUPE_CONTEXT_IDLE); + list_add(&context->list_entry, &zone->available); + recycled++; + } + + if (recycled > 0) + WRITE_ONCE(zone->active, zone->active - recycled); + ASSERT_LOG_ONLY(READ_ONCE(zone->active) == 0, "all contexts inactive"); + vdo_finish_draining(&zone->state); +} + +static void timeout_index_operations_callback(struct vdo_completion *completion) +{ + struct dedupe_context *context, *tmp; + struct hash_zone *zone = as_hash_zone(completion); + u64 timeout_jiffies = msecs_to_jiffies(vdo_dedupe_index_timeout_interval); + unsigned long cutoff = jiffies - timeout_jiffies; + unsigned int timed_out = 0; + + atomic_set(&zone->timer_state, DEDUPE_QUERY_TIMER_IDLE); + list_for_each_entry_safe(context, tmp, &zone->pending, list_entry) { + if (cutoff <= context->submission_jiffies) { + /* + * We have reached the oldest query which has not timed out yet, so restart + * the timer. + */ + start_expiration_timer(context); + break; + } + + if (!change_context_state(context, DEDUPE_CONTEXT_PENDING, + DEDUPE_CONTEXT_TIMED_OUT)) { + /* + * This context completed between the time the timeout fired, and now. We + * can treat it as a a successful query, its requestor is already enqueued + * to process it. + */ + continue; + } + + /* + * Remove this context from the pending list so we won't look at it again on a + * subsequent timeout. Once the index completes it, it will be reused. Meanwhile, + * send its requestor on its way. + */ + list_del_init(&context->list_entry); + continue_data_vio(context->requestor); + timed_out++; + } + + if (timed_out > 0) + report_dedupe_timeouts(completion->vdo->hash_zones, timed_out); + + check_for_drain_complete(zone); +} + +static void timeout_index_operations(struct timer_list *t) +{ + struct hash_zone *zone = from_timer(zone, t, timer); + + if (change_timer_state(zone, DEDUPE_QUERY_TIMER_RUNNING, + DEDUPE_QUERY_TIMER_FIRED)) + vdo_launch_completion(&zone->completion); +} + static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zones, zone_count_t zone_number) { @@ -2441,3 +2873,196 @@ void vdo_dump_hash_zones(struct hash_zones *zones) for (zone = 0; zone < zones->zone_count; zone++) dump_hash_zone(&zones->zones[zone]); } + +void vdo_set_dedupe_index_timeout_interval(unsigned int value) +{ + u64 alb_jiffies; + + /* Arbitrary maximum value is two minutes */ + if (value > 120000) + value = 120000; + /* Arbitrary minimum value is 2 jiffies */ + alb_jiffies = msecs_to_jiffies(value); + + if (alb_jiffies < 2) { + alb_jiffies = 2; + value = jiffies_to_msecs(alb_jiffies); + } + vdo_dedupe_index_timeout_interval = value; + vdo_dedupe_index_timeout_jiffies = alb_jiffies; +} + +void vdo_set_dedupe_index_min_timer_interval(unsigned int value) +{ + u64 min_jiffies; + + /* Arbitrary maximum value is one second */ + if (value > 1000) + value = 1000; + + /* Arbitrary minimum value is 2 jiffies */ + min_jiffies = msecs_to_jiffies(value); + + if (min_jiffies < 2) { + min_jiffies = 2; + value = jiffies_to_msecs(min_jiffies); + } + + vdo_dedupe_index_min_timer_interval = value; + vdo_dedupe_index_min_timer_jiffies = min_jiffies; +} + +/** + * acquire_context() - Acquire a dedupe context from a hash_zone if any are available. + * @zone: the hash zone + * + * Return: A dedupe_context or NULL if none are available + */ +static struct dedupe_context * __must_check acquire_context(struct hash_zone *zone) +{ + struct dedupe_context *context; + struct funnel_queue_entry *entry; + + assert_in_hash_zone(zone, __func__); + + if (!list_empty(&zone->available)) { + WRITE_ONCE(zone->active, zone->active + 1); + context = list_first_entry(&zone->available, struct dedupe_context, + list_entry); + list_del_init(&context->list_entry); + return context; + } + + entry = uds_funnel_queue_poll(zone->timed_out_complete); + return ((entry == NULL) ? + NULL : container_of(entry, struct dedupe_context, queue_entry)); +} + +static void prepare_uds_request(struct uds_request *request, struct data_vio *data_vio, + enum uds_request_type operation) +{ + request->record_name = data_vio->record_name; + request->type = operation; + if ((operation == UDS_POST) || (operation == UDS_UPDATE)) { + size_t offset = 0; + struct uds_record_data *encoding = &request->new_metadata; + + encoding->data[offset++] = UDS_ADVICE_VERSION; + encoding->data[offset++] = data_vio->new_mapped.state; + put_unaligned_le64(data_vio->new_mapped.pbn, &encoding->data[offset]); + offset += sizeof(u64); + BUG_ON(offset != UDS_ADVICE_SIZE); + } +} + +/* + * The index operation will inquire about data_vio.record_name, providing (if the operation is + * appropriate) advice from the data_vio's new_mapped fields. The advice found in the index (or + * NULL if none) will be returned via receive_data_vio_dedupe_advice(). dedupe_context.status is + * set to the return status code of any asynchronous index processing. + */ +static void query_index(struct data_vio *data_vio, enum uds_request_type operation) +{ + int result; + struct dedupe_context *context; + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct hash_zone *zone = data_vio->hash_zone; + + assert_data_vio_in_hash_zone(data_vio); + + if (!READ_ONCE(vdo->hash_zones->dedupe_flag)) { + continue_data_vio(data_vio); + return; + } + + context = acquire_context(zone); + if (context == NULL) { + atomic64_inc(&vdo->hash_zones->dedupe_context_busy); + continue_data_vio(data_vio); + return; + } + + data_vio->dedupe_context = context; + context->requestor = data_vio; + context->submission_jiffies = jiffies; + prepare_uds_request(&context->request, data_vio, operation); + atomic_set(&context->state, DEDUPE_CONTEXT_PENDING); + list_add_tail(&context->list_entry, &zone->pending); + start_expiration_timer(context); + result = uds_launch_request(&context->request); + if (result != UDS_SUCCESS) { + context->request.status = result; + finish_index_operation(&context->request); + } +} + +static void set_target_state(struct hash_zones *zones, enum index_state target, + bool change_dedupe, bool dedupe, bool set_create) +{ + const char *old_state, *new_state; + + spin_lock(&zones->lock); + old_state = index_state_to_string(zones, zones->index_target); + if (change_dedupe) + WRITE_ONCE(zones->dedupe_flag, dedupe); + + if (set_create) + zones->create_flag = true; + + zones->index_target = target; + launch_dedupe_state_change(zones); + new_state = index_state_to_string(zones, zones->index_target); + spin_unlock(&zones->lock); + + if (old_state != new_state) + uds_log_info("Setting UDS index target state to %s", new_state); +} + +const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones) +{ + const char *state; + + spin_lock(&zones->lock); + state = index_state_to_string(zones, zones->index_state); + spin_unlock(&zones->lock); + + return state; +} + +/* Handle a dmsetup message relevant to the index. */ +int vdo_message_dedupe_index(struct hash_zones *zones, const char *name) +{ + if (strcasecmp(name, "index-close") == 0) { + set_target_state(zones, IS_CLOSED, false, false, false); + return 0; + } else if (strcasecmp(name, "index-create") == 0) { + set_target_state(zones, IS_OPENED, false, false, true); + return 0; + } else if (strcasecmp(name, "index-disable") == 0) { + set_target_state(zones, IS_OPENED, true, false, false); + return 0; + } else if (strcasecmp(name, "index-enable") == 0) { + set_target_state(zones, IS_OPENED, true, true, false); + return 0; + } + + return -EINVAL; +} + +int vdo_add_dedupe_index_sysfs(struct hash_zones *zones) +{ + int result = kobject_add(&zones->dedupe_directory, + &zones->completion.vdo->vdo_directory, "dedupe"); + + if (result == 0) + vdo_set_admin_state_code(&zones->state, + VDO_ADMIN_STATE_NORMAL_OPERATION); + + return result; +} + +/* If create_flag, create a new index without first attempting to load an existing index. */ +void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag) +{ + set_target_state(zones, IS_OPENED, true, true, create_flag); +} diff --git a/drivers/md/dm-vdo/dedupe.h b/drivers/md/dm-vdo/dedupe.h index f4b091efc27e..90c5779bfe70 100644 --- a/drivers/md/dm-vdo/dedupe.h +++ b/drivers/md/dm-vdo/dedupe.h @@ -91,4 +91,30 @@ struct hash_zone * __must_check vdo_select_hash_zone(struct hash_zones *zones, void vdo_dump_hash_zones(struct hash_zones *zones); +const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones); + +u64 vdo_get_dedupe_index_timeout_count(struct hash_zones *zones); + +int vdo_message_dedupe_index(struct hash_zones *zones, const char *name); + +int vdo_add_dedupe_index_sysfs(struct hash_zones *zones); + +void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag); + +void vdo_resume_hash_zones(struct hash_zones *zones, struct vdo_completion *parent); + +void vdo_finish_dedupe_index(struct hash_zones *zones); + +/* Interval (in milliseconds) from submission until switching to fast path and skipping UDS. */ +extern unsigned int vdo_dedupe_index_timeout_interval; + +/* + * Minimum time interval (in milliseconds) between timer invocations to check for requests waiting + * for UDS that should now time out. + */ +extern unsigned int vdo_dedupe_index_min_timer_interval; + +void vdo_set_dedupe_index_timeout_interval(unsigned int value); +void vdo_set_dedupe_index_min_timer_interval(unsigned int value); + #endif /* VDO_DEDUPE_H */ From 58a55a59160cc2f59168431859b6ae154b154246 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:13:38 -0500 Subject: [PATCH 0572/1038] dm vdo: add the compressed block bin packer When blocks do not deduplicate, vdo will attempt to compress them. Up to 14 compressed blocks may be packed into a single data block (this limitation is imposed by the block map). The packer implements a simple best-fit packing algorithm and also manages the formatting and writing of compressed blocks when bins fill. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/packer.c | 784 +++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/packer.h | 122 ++++++ 2 files changed, 906 insertions(+) create mode 100644 drivers/md/dm-vdo/packer.c create mode 100644 drivers/md/dm-vdo/packer.h diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c new file mode 100644 index 000000000000..f09eb1ca4902 --- /dev/null +++ b/drivers/md/dm-vdo/packer.c @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "packer.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" + +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "encodings.h" +#include "io-submitter.h" +#include "physical-zone.h" +#include "status-codes.h" +#include "vdo.h" +#include "vio.h" + +static const struct version_number COMPRESSED_BLOCK_1_0 = { + .major_version = 1, + .minor_version = 0, +}; + +enum { + COMPRESSED_BLOCK_1_0_SIZE = 4 + 4 + (2 * VDO_MAX_COMPRESSION_SLOTS), +}; + +/** + * vdo_get_compressed_block_fragment() - Get a reference to a compressed fragment from a compressed + * block. + * @mapping_state [in] The mapping state for the look up. + * @compressed_block [in] The compressed block that was read from disk. + * @fragment_offset [out] The offset of the fragment within a compressed block. + * @fragment_size [out] The size of the fragment. + * + * Return: If a valid compressed fragment is found, VDO_SUCCESS; otherwise, VDO_INVALID_FRAGMENT if + * the fragment is invalid. + */ +int vdo_get_compressed_block_fragment(enum block_mapping_state mapping_state, + struct compressed_block *block, + u16 *fragment_offset, u16 *fragment_size) +{ + u16 compressed_size; + u16 offset = 0; + unsigned int i; + u8 slot; + struct version_number version; + + if (!vdo_is_state_compressed(mapping_state)) + return VDO_INVALID_FRAGMENT; + + version = vdo_unpack_version_number(block->header.version); + if (!vdo_are_same_version(version, COMPRESSED_BLOCK_1_0)) + return VDO_INVALID_FRAGMENT; + + slot = mapping_state - VDO_MAPPING_STATE_COMPRESSED_BASE; + if (slot >= VDO_MAX_COMPRESSION_SLOTS) + return VDO_INVALID_FRAGMENT; + + compressed_size = __le16_to_cpu(block->header.sizes[slot]); + for (i = 0; i < slot; i++) { + offset += __le16_to_cpu(block->header.sizes[i]); + if (offset >= VDO_COMPRESSED_BLOCK_DATA_SIZE) + return VDO_INVALID_FRAGMENT; + } + + if ((offset + compressed_size) > VDO_COMPRESSED_BLOCK_DATA_SIZE) + return VDO_INVALID_FRAGMENT; + + *fragment_offset = offset; + *fragment_size = compressed_size; + return VDO_SUCCESS; +} + +/** + * assert_on_packer_thread() - Check that we are on the packer thread. + * @packer: The packer. + * @caller: The function which is asserting. + */ +static inline void assert_on_packer_thread(struct packer *packer, const char *caller) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == packer->thread_id), + "%s() called from packer thread", caller); +} + +/** + * insert_in_sorted_list() - Insert a bin to the list. + * @packer: The packer. + * @bin: The bin to move to its sorted position. + * + * The list is in ascending order of free space. Since all bins are already in the list, this + * actually moves the bin to the correct position in the list. + */ +static void insert_in_sorted_list(struct packer *packer, struct packer_bin *bin) +{ + struct packer_bin *active_bin; + + list_for_each_entry(active_bin, &packer->bins, list) + if (active_bin->free_space > bin->free_space) { + list_move_tail(&bin->list, &active_bin->list); + return; + } + + list_move_tail(&bin->list, &packer->bins); +} + +/** + * make_bin() - Allocate a bin and put it into the packer's list. + * @packer: The packer. + */ +static int __must_check make_bin(struct packer *packer) +{ + struct packer_bin *bin; + int result; + + result = uds_allocate_extended(struct packer_bin, VDO_MAX_COMPRESSION_SLOTS, + struct vio *, __func__, &bin); + if (result != VDO_SUCCESS) + return result; + + bin->free_space = VDO_COMPRESSED_BLOCK_DATA_SIZE; + INIT_LIST_HEAD(&bin->list); + list_add_tail(&bin->list, &packer->bins); + return VDO_SUCCESS; +} + +/** + * vdo_make_packer() - Make a new block packer. + * + * @vdo: The vdo to which this packer belongs. + * @bin_count: The number of partial bins to keep in memory. + * @packer_ptr: A pointer to hold the new packer. + * + * Return: VDO_SUCCESS or an error + */ +int vdo_make_packer(struct vdo *vdo, block_count_t bin_count, struct packer **packer_ptr) +{ + struct packer *packer; + block_count_t i; + int result; + + result = uds_allocate(1, struct packer, __func__, &packer); + if (result != VDO_SUCCESS) + return result; + + packer->thread_id = vdo->thread_config.packer_thread; + packer->size = bin_count; + INIT_LIST_HEAD(&packer->bins); + vdo_set_admin_state_code(&packer->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + + for (i = 0; i < bin_count; i++) { + result = make_bin(packer); + if (result != VDO_SUCCESS) { + vdo_free_packer(packer); + return result; + } + } + + /* + * The canceled bin can hold up to half the number of user vios. Every canceled vio in the + * bin must have a canceler for which it is waiting, and any canceler will only have + * canceled one lock holder at a time. + */ + result = uds_allocate_extended(struct packer_bin, MAXIMUM_VDO_USER_VIOS / 2, + struct vio *, __func__, &packer->canceled_bin); + if (result != VDO_SUCCESS) { + vdo_free_packer(packer); + return result; + } + + result = vdo_make_default_thread(vdo, packer->thread_id); + if (result != VDO_SUCCESS) { + vdo_free_packer(packer); + return result; + } + + *packer_ptr = packer; + return VDO_SUCCESS; +} + +/** + * vdo_free_packer() - Free a block packer. + * @packer: The packer to free. + */ +void vdo_free_packer(struct packer *packer) +{ + struct packer_bin *bin, *tmp; + + if (packer == NULL) + return; + + list_for_each_entry_safe(bin, tmp, &packer->bins, list) { + list_del_init(&bin->list); + uds_free(bin); + } + + uds_free(uds_forget(packer->canceled_bin)); + uds_free(packer); +} + +/** + * get_packer_from_data_vio() - Get the packer from a data_vio. + * @data_vio: The data_vio. + * + * Return: The packer from the VDO to which the data_vio belongs. + */ +static inline struct packer *get_packer_from_data_vio(struct data_vio *data_vio) +{ + return vdo_from_data_vio(data_vio)->packer; +} + +/** + * vdo_get_packer_statistics() - Get the current statistics from the packer. + * @packer: The packer to query. + * + * Return: a copy of the current statistics for the packer. + */ +struct packer_statistics vdo_get_packer_statistics(const struct packer *packer) +{ + const struct packer_statistics *stats = &packer->statistics; + + return (struct packer_statistics) { + .compressed_fragments_written = READ_ONCE(stats->compressed_fragments_written), + .compressed_blocks_written = READ_ONCE(stats->compressed_blocks_written), + .compressed_fragments_in_packer = READ_ONCE(stats->compressed_fragments_in_packer), + }; +} + +/** + * abort_packing() - Abort packing a data_vio. + * @data_vio: The data_vio to abort. + */ +static void abort_packing(struct data_vio *data_vio) +{ + struct packer *packer = get_packer_from_data_vio(data_vio); + + WRITE_ONCE(packer->statistics.compressed_fragments_in_packer, + packer->statistics.compressed_fragments_in_packer - 1); + + write_data_vio(data_vio); +} + +/** + * release_compressed_write_waiter() - Update a data_vio for which a successful compressed write + * has completed and send it on its way. + + * @data_vio: The data_vio to release. + * @allocation: The allocation to which the compressed block was written. + */ +static void release_compressed_write_waiter(struct data_vio *data_vio, + struct allocation *allocation) +{ + data_vio->new_mapped = (struct zoned_pbn) { + .pbn = allocation->pbn, + .zone = allocation->zone, + .state = data_vio->compression.slot + VDO_MAPPING_STATE_COMPRESSED_BASE, + }; + + vdo_share_compressed_write_lock(data_vio, allocation->lock); + update_metadata_for_data_vio_write(data_vio, allocation->lock); +} + +/** + * finish_compressed_write() - Finish a compressed block write. + * @completion: The compressed write completion. + * + * This callback is registered in continue_after_allocation(). + */ +static void finish_compressed_write(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct data_vio *client, *next; + + assert_data_vio_in_allocated_zone(agent); + + /* + * Process all the non-agent waiters first to ensure that the pbn lock can not be released + * until all of them have had a chance to journal their increfs. + */ + for (client = agent->compression.next_in_batch; client != NULL; client = next) { + next = client->compression.next_in_batch; + release_compressed_write_waiter(client, &agent->allocation); + } + + completion->error_handler = handle_data_vio_error; + release_compressed_write_waiter(agent, &agent->allocation); +} + +static void handle_compressed_write_error(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct allocation *allocation = &agent->allocation; + struct data_vio *client, *next; + + if (vdo_requeue_completion_if_needed(completion, allocation->zone->thread_id)) + return; + + update_vio_error_stats(as_vio(completion), + "Completing compressed write vio for physical block %llu with error", + (unsigned long long) allocation->pbn); + + for (client = agent->compression.next_in_batch; client != NULL; client = next) { + next = client->compression.next_in_batch; + write_data_vio(client); + } + + /* Now that we've released the batch from the packer, forget the error and continue on. */ + vdo_reset_completion(completion); + completion->error_handler = handle_data_vio_error; + write_data_vio(agent); +} + +/** + * add_to_bin() - Put a data_vio in a specific packer_bin in which it will definitely fit. + * @bin: The bin in which to put the data_vio. + * @data_vio: The data_vio to add. + */ +static void add_to_bin(struct packer_bin *bin, struct data_vio *data_vio) +{ + data_vio->compression.bin = bin; + data_vio->compression.slot = bin->slots_used; + bin->incoming[bin->slots_used++] = data_vio; +} + +/** + * remove_from_bin() - Get the next data_vio whose compression has not been canceled from a bin. + * @packer: The packer. + * @bin: The bin from which to get a data_vio. + * + * Any canceled data_vios will be moved to the canceled bin. + * Return: An uncanceled data_vio from the bin or NULL if there are none. + */ +static struct data_vio *remove_from_bin(struct packer *packer, struct packer_bin *bin) +{ + while (bin->slots_used > 0) { + struct data_vio *data_vio = bin->incoming[--bin->slots_used]; + + if (!advance_data_vio_compression_stage(data_vio).may_not_compress) { + data_vio->compression.bin = NULL; + return data_vio; + } + + add_to_bin(packer->canceled_bin, data_vio); + } + + /* The bin is now empty. */ + bin->free_space = VDO_COMPRESSED_BLOCK_DATA_SIZE; + return NULL; +} + +/** + * initialize_compressed_block() - Initialize a compressed block. + * @block: The compressed block to initialize. + * @size: The size of the agent's fragment. + * + * This method initializes the compressed block in the compressed write agent. Because the + * compressor already put the agent's compressed fragment at the start of the compressed block's + * data field, it needn't be copied. So all we need do is initialize the header and set the size of + * the agent's fragment. + */ +static void initialize_compressed_block(struct compressed_block *block, u16 size) +{ + /* + * Make sure the block layout isn't accidentally changed by changing the length of the + * block header. + */ + BUILD_BUG_ON(sizeof(struct compressed_block_header) != COMPRESSED_BLOCK_1_0_SIZE); + + block->header.version = vdo_pack_version_number(COMPRESSED_BLOCK_1_0); + block->header.sizes[0] = __cpu_to_le16(size); +} + +/** + * pack_fragment() - Pack a data_vio's fragment into the compressed block in which it is already + * known to fit. + * @compression: The agent's compression_state to pack in to. + * @data_vio: The data_vio to pack. + * @offset: The offset into the compressed block at which to pack the fragment. + * @compressed_block: The compressed block which will be written out when batch is fully packed. + * + * Return: The new amount of space used. + */ +static block_size_t __must_check pack_fragment(struct compression_state *compression, + struct data_vio *data_vio, + block_size_t offset, slot_number_t slot, + struct compressed_block *block) +{ + struct compression_state *to_pack = &data_vio->compression; + char *fragment = to_pack->block->data; + + to_pack->next_in_batch = compression->next_in_batch; + compression->next_in_batch = data_vio; + to_pack->slot = slot; + block->header.sizes[slot] = __cpu_to_le16(to_pack->size); + memcpy(&block->data[offset], fragment, to_pack->size); + return (offset + to_pack->size); +} + +/** + * compressed_write_end_io() - The bio_end_io for a compressed block write. + * @bio: The bio for the compressed write. + */ +static void compressed_write_end_io(struct bio *bio) +{ + struct data_vio *data_vio = vio_as_data_vio(bio->bi_private); + + vdo_count_completed_bios(bio); + set_data_vio_allocated_zone_callback(data_vio, finish_compressed_write); + continue_data_vio_with_error(data_vio, blk_status_to_errno(bio->bi_status)); +} + +/** + * write_bin() - Write out a bin. + * @packer: The packer. + * @bin: The bin to write. + */ +static void write_bin(struct packer *packer, struct packer_bin *bin) +{ + int result; + block_size_t offset; + slot_number_t slot = 1; + struct compression_state *compression; + struct compressed_block *block; + struct data_vio *agent = remove_from_bin(packer, bin); + struct data_vio *client; + struct packer_statistics *stats; + + if (agent == NULL) + return; + + compression = &agent->compression; + compression->slot = 0; + block = compression->block; + initialize_compressed_block(block, compression->size); + offset = compression->size; + + while ((client = remove_from_bin(packer, bin)) != NULL) + offset = pack_fragment(compression, client, offset, slot++, block); + + /* + * If the batch contains only a single vio, then we save nothing by saving the compressed + * form. Continue processing the single vio in the batch. + */ + if (slot == 1) { + abort_packing(agent); + return; + } + + if (slot < VDO_MAX_COMPRESSION_SLOTS) { + /* Clear out the sizes of the unused slots */ + memset(&block->header.sizes[slot], 0, + (VDO_MAX_COMPRESSION_SLOTS - slot) * sizeof(__le16)); + } + + agent->vio.completion.error_handler = handle_compressed_write_error; + if (vdo_is_read_only(vdo_from_data_vio(agent))) { + continue_data_vio_with_error(agent, VDO_READ_ONLY); + return; + } + + result = vio_reset_bio(&agent->vio, (char *) block, compressed_write_end_io, + REQ_OP_WRITE, agent->allocation.pbn); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(agent, result); + return; + } + + /* + * Once the compressed write is submitted, the fragments are no longer in the packer, so + * update stats now. + */ + stats = &packer->statistics; + WRITE_ONCE(stats->compressed_fragments_in_packer, + (stats->compressed_fragments_in_packer - slot)); + WRITE_ONCE(stats->compressed_fragments_written, + (stats->compressed_fragments_written + slot)); + WRITE_ONCE(stats->compressed_blocks_written, + stats->compressed_blocks_written + 1); + + submit_data_vio_io(agent); +} + +/** + * add_data_vio_to_packer_bin() - Add a data_vio to a bin's incoming queue + * @packer: The packer. + * @bin: The bin to which to add the data_vio. + * @data_vio: The data_vio to add to the bin's queue. + * + * Adds a data_vio to a bin's incoming queue, handles logical space change, and calls physical + * space processor. + */ +static void add_data_vio_to_packer_bin(struct packer *packer, struct packer_bin *bin, + struct data_vio *data_vio) +{ + /* If the selected bin doesn't have room, start a new batch to make room. */ + if (bin->free_space < data_vio->compression.size) + write_bin(packer, bin); + + add_to_bin(bin, data_vio); + bin->free_space -= data_vio->compression.size; + + /* If we happen to exactly fill the bin, start a new batch. */ + if ((bin->slots_used == VDO_MAX_COMPRESSION_SLOTS) || + (bin->free_space == 0)) + write_bin(packer, bin); + + /* Now that we've finished changing the free space, restore the sort order. */ + insert_in_sorted_list(packer, bin); +} + +/** + * select_bin() - Select the bin that should be used to pack the compressed data in a data_vio with + * other data_vios. + * @packer: The packer. + * @data_vio: The data_vio. + */ +static struct packer_bin * __must_check select_bin(struct packer *packer, + struct data_vio *data_vio) +{ + /* + * First best fit: select the bin with the least free space that has enough room for the + * compressed data in the data_vio. + */ + struct packer_bin *bin, *fullest_bin; + + list_for_each_entry(bin, &packer->bins, list) { + if (bin->free_space >= data_vio->compression.size) + return bin; + } + + /* + * None of the bins have enough space for the data_vio. We're not allowed to create new + * bins, so we have to overflow one of the existing bins. It's pretty intuitive to select + * the fullest bin, since that "wastes" the least amount of free space in the compressed + * block. But if the space currently used in the fullest bin is smaller than the compressed + * size of the incoming block, it seems wrong to force that bin to write when giving up on + * compressing the incoming data_vio would likewise "waste" the least amount of free space. + */ + fullest_bin = list_first_entry(&packer->bins, struct packer_bin, list); + if (data_vio->compression.size >= + (VDO_COMPRESSED_BLOCK_DATA_SIZE - fullest_bin->free_space)) + return NULL; + + /* + * The fullest bin doesn't have room, but writing it out and starting a new batch with the + * incoming data_vio will increase the packer's free space. + */ + return fullest_bin; +} + +/** + * vdo_attempt_packing() - Attempt to rewrite the data in this data_vio as part of a compressed + * block. + * @data_vio: The data_vio to pack. + */ +void vdo_attempt_packing(struct data_vio *data_vio) +{ + int result; + struct packer_bin *bin; + struct data_vio_compression_status status = get_data_vio_compression_status(data_vio); + struct packer *packer = get_packer_from_data_vio(data_vio); + + assert_on_packer_thread(packer, __func__); + + result = ASSERT((status.stage == DATA_VIO_COMPRESSING), + "attempt to pack data_vio not ready for packing, stage: %u", + status.stage); + if (result != VDO_SUCCESS) + return; + + /* + * Increment whether or not this data_vio will be packed or not since abort_packing() + * always decrements the counter. + */ + WRITE_ONCE(packer->statistics.compressed_fragments_in_packer, + packer->statistics.compressed_fragments_in_packer + 1); + + /* + * If packing of this data_vio is disallowed for administrative reasons, give up before + * making any state changes. + */ + if (!vdo_is_state_normal(&packer->state) || + (data_vio->flush_generation < packer->flush_generation)) { + abort_packing(data_vio); + return; + } + + /* + * The check of may_vio_block_in_packer() here will set the data_vio's compression state to + * VIO_PACKING if the data_vio is allowed to be compressed (if it has already been + * canceled, we'll fall out here). Once the data_vio is in the VIO_PACKING state, it must + * be guaranteed to be put in a bin before any more requests can be processed by the packer + * thread. Otherwise, a canceling data_vio could attempt to remove the canceled data_vio + * from the packer and fail to rendezvous with it (VDO-2809). We must also make sure that + * we will actually bin the data_vio and not give up on it as being larger than the space + * used in the fullest bin. Hence we must call select_bin() before calling + * may_vio_block_in_packer() (VDO-2826). + */ + bin = select_bin(packer, data_vio); + if ((bin == NULL) || + (advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_PACKING)) { + abort_packing(data_vio); + return; + } + + add_data_vio_to_packer_bin(packer, bin, data_vio); +} + +/** + * check_for_drain_complete() - Check whether the packer has drained. + * @packer: The packer. + */ +static void check_for_drain_complete(struct packer *packer) +{ + if (vdo_is_state_draining(&packer->state) && (packer->canceled_bin->slots_used == 0)) + vdo_finish_draining(&packer->state); +} + +/** + * write_all_non_empty_bins() - Write out all non-empty bins on behalf of a flush or suspend. + * @packer: The packer being flushed. + */ +static void write_all_non_empty_bins(struct packer *packer) +{ + struct packer_bin *bin; + + list_for_each_entry(bin, &packer->bins, list) + write_bin(packer, bin); + /* + * We don't need to re-sort the bin here since this loop will make every bin have + * the same amount of free space, so every ordering is sorted. + */ + + check_for_drain_complete(packer); +} + +/** + * vdo_flush_packer() - Request that the packer flush asynchronously. + * @packer: The packer to flush. + * + * All bins with at least two compressed data blocks will be written out, and any solitary pending + * VIOs will be released from the packer. While flushing is in progress, any VIOs submitted to + * vdo_attempt_packing() will be continued immediately without attempting to pack them. + */ +void vdo_flush_packer(struct packer *packer) +{ + assert_on_packer_thread(packer, __func__); + if (vdo_is_state_normal(&packer->state)) + write_all_non_empty_bins(packer); +} + +/** + * vdo_remove_lock_holder_from_packer() - Remove a lock holder from the packer. + * @completion: The data_vio which needs a lock held by a data_vio in the packer. The data_vio's + * compression.lock_holder field will point to the data_vio to remove. + */ +void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct packer *packer = get_packer_from_data_vio(data_vio); + struct data_vio *lock_holder; + struct packer_bin *bin; + slot_number_t slot; + + assert_data_vio_in_packer_zone(data_vio); + + lock_holder = uds_forget(data_vio->compression.lock_holder); + bin = lock_holder->compression.bin; + ASSERT_LOG_ONLY((bin != NULL), "data_vio in packer has a bin"); + + slot = lock_holder->compression.slot; + bin->slots_used--; + if (slot < bin->slots_used) { + bin->incoming[slot] = bin->incoming[bin->slots_used]; + bin->incoming[slot]->compression.slot = slot; + } + + lock_holder->compression.bin = NULL; + lock_holder->compression.slot = 0; + + if (bin != packer->canceled_bin) { + bin->free_space += lock_holder->compression.size; + insert_in_sorted_list(packer, bin); + } + + abort_packing(lock_holder); + check_for_drain_complete(packer); +} + +/** + * vdo_increment_packer_flush_generation() - Increment the flush generation in the packer. + * @packer: The packer. + * + * This will also cause the packer to flush so that any VIOs from previous generations will exit + * the packer. + */ +void vdo_increment_packer_flush_generation(struct packer *packer) +{ + assert_on_packer_thread(packer, __func__); + packer->flush_generation++; + vdo_flush_packer(packer); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + struct packer *packer = container_of(state, struct packer, state); + + write_all_non_empty_bins(packer); +} + +/** + * vdo_drain_packer() - Drain the packer by preventing any more VIOs from entering the packer and + * then flushing. + * @packer: The packer to drain. + * @completion: The completion to finish when the packer has drained. + */ +void vdo_drain_packer(struct packer *packer, struct vdo_completion *completion) +{ + assert_on_packer_thread(packer, __func__); + vdo_start_draining(&packer->state, VDO_ADMIN_STATE_SUSPENDING, completion, + initiate_drain); +} + +/** + * vdo_resume_packer() - Resume a packer which has been suspended. + * @packer: The packer to resume. + * @parent: The completion to finish when the packer has resumed. + */ +void vdo_resume_packer(struct packer *packer, struct vdo_completion *parent) +{ + assert_on_packer_thread(packer, __func__); + vdo_continue_completion(parent, vdo_resume_if_quiescent(&packer->state)); +} + +static void dump_packer_bin(const struct packer_bin *bin, bool canceled) +{ + if (bin->slots_used == 0) + /* Don't dump empty bins. */ + return; + + uds_log_info(" %sBin slots_used=%u free_space=%zu", + (canceled ? "Canceled" : ""), bin->slots_used, bin->free_space); + + /* + * FIXME: dump vios in bin->incoming? The vios should have been dumped from the vio pool. + * Maybe just dump their addresses so it's clear they're here? + */ +} + +/** + * vdo_dump_packer() - Dump the packer. + * @packer: The packer. + * + * Context: dumps in a thread-unsafe fashion. + */ +void vdo_dump_packer(const struct packer *packer) +{ + struct packer_bin *bin; + + uds_log_info("packer"); + uds_log_info(" flushGeneration=%llu state %s packer_bin_count=%llu", + (unsigned long long) packer->flush_generation, + vdo_get_admin_state_code(&packer->state)->name, + (unsigned long long) packer->size); + + list_for_each_entry(bin, &packer->bins, list) + dump_packer_bin(bin, false); + + dump_packer_bin(packer->canceled_bin, true); +} diff --git a/drivers/md/dm-vdo/packer.h b/drivers/md/dm-vdo/packer.h new file mode 100644 index 000000000000..2dcc40bd4417 --- /dev/null +++ b/drivers/md/dm-vdo/packer.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_PACKER_H +#define VDO_PACKER_H + +#include + +#include "admin-state.h" +#include "constants.h" +#include "encodings.h" +#include "statistics.h" +#include "types.h" +#include "wait-queue.h" + +enum { + DEFAULT_PACKER_BINS = 16, +}; + +/* The header of a compressed block. */ +struct compressed_block_header { + /* Unsigned 32-bit major and minor versions, little-endian */ + struct packed_version_number version; + + /* List of unsigned 16-bit compressed block sizes, little-endian */ + __le16 sizes[VDO_MAX_COMPRESSION_SLOTS]; +} __packed; + +enum { + VDO_COMPRESSED_BLOCK_DATA_SIZE = VDO_BLOCK_SIZE - sizeof(struct compressed_block_header), + + /* + * A compressed block is only written if we can pack at least two fragments into it, so a + * fragment which fills the entire data portion of a compressed block is too big. + */ + VDO_MAX_COMPRESSED_FRAGMENT_SIZE = VDO_COMPRESSED_BLOCK_DATA_SIZE - 1, +}; + +/* * The compressed block overlay. */ +struct compressed_block { + struct compressed_block_header header; + char data[VDO_COMPRESSED_BLOCK_DATA_SIZE]; +} __packed; + +/* + * Each packer_bin holds an incomplete batch of data_vios that only partially fill a compressed + * block. The bins are kept in a ring sorted by the amount of unused space so the first bin with + * enough space to hold a newly-compressed data_vio can easily be found. When the bin fills up or + * is flushed, the first uncanceled data_vio in the bin is selected to be the agent for that bin. + * Upon entering the packer, each data_vio already has its compressed data in the first slot of the + * data_vio's compressed_block (overlaid on the data_vio's scratch_block). So the agent's fragment + * is already in place. The fragments for the other uncanceled data_vios in the bin are packed into + * the agent's compressed block. The agent then writes out the compressed block. If the write is + * successful, the agent shares its pbn lock which each of the other data_vios in its compressed + * block and sends each on its way. Finally the agent itself continues on the write path as before. + * + * There is one special bin which is used to hold data_vios which have been canceled and removed + * from their bin by the packer. These data_vios need to wait for the canceller to rendezvous with + * them (VDO-2809) and so they sit in this special bin. + */ +struct packer_bin { + /* List links for packer.packer_bins */ + struct list_head list; + /* The number of items in the bin */ + slot_number_t slots_used; + /* The number of compressed block bytes remaining in the current batch */ + size_t free_space; + /* The current partial batch of data_vios, waiting for more */ + struct data_vio *incoming[]; +}; + +struct packer { + /* The ID of the packer's callback thread */ + thread_id_t thread_id; + /* The number of bins */ + block_count_t size; + /* A list of all packer_bins, kept sorted by free_space */ + struct list_head bins; + /* + * A bin to hold data_vios which were canceled out of the packer and are waiting to + * rendezvous with the canceling data_vio. + */ + struct packer_bin *canceled_bin; + + /* The current flush generation */ + sequence_number_t flush_generation; + + /* The administrative state of the packer */ + struct admin_state state; + + /* Statistics are only updated on the packer thread, but are accessed from other threads */ + struct packer_statistics statistics; +}; + +int vdo_get_compressed_block_fragment(enum block_mapping_state mapping_state, + struct compressed_block *block, + u16 *fragment_offset, u16 *fragment_size); + +int __must_check vdo_make_packer(struct vdo *vdo, block_count_t bin_count, + struct packer **packer_ptr); + +void vdo_free_packer(struct packer *packer); + +struct packer_statistics __must_check vdo_get_packer_statistics(const struct packer *packer); + +void vdo_attempt_packing(struct data_vio *data_vio); + +void vdo_flush_packer(struct packer *packer); + +void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion); + +void vdo_increment_packer_flush_generation(struct packer *packer); + +void vdo_drain_packer(struct packer *packer, struct vdo_completion *completion); + +void vdo_resume_packer(struct packer *packer, struct vdo_completion *parent); + +void vdo_dump_packer(const struct packer *packer); + +#endif /* VDO_PACKER_H */ From 09eff388df02ac0fff7d726ca2136d623d1db907 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:56:30 -0500 Subject: [PATCH 0573/1038] dm vdo: add slab structure, slab journal and reference counters Most of the vdo volume belongs to the slab depot. The depot contains a collection of slabs. The slabs can be up to 32GB, and are divided into three sections. Most of a slab consists of a linear sequence of 4K blocks. These blocks are used either to store data, or to hold portions of the block map (see subsequent patches). In addition to the data blocks, each slab has a set of reference counters, using 1 byte for each data block. Finally each slab has a journal. Reference updates are written to the slab journal, which is written out one block at a time as each block fills. A copy of the reference counters is kept in memory, and are written out a block at a time, in oldest-dirtied-order whenever there is a need to reclaim slab journal space. The journal is used both to ensure that the main recovery journal (see subsequent patches) can regularly free up space, and also to amortize the cost of updating individual reference blocks. This patch adds the slab structure as well as the slab journal and reference counters. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/slab-depot.c | 2404 ++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/slab-depot.h | 252 ++++ 2 files changed, 2656 insertions(+) create mode 100644 drivers/md/dm-vdo/slab-depot.c create mode 100644 drivers/md/dm-vdo/slab-depot.h diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c new file mode 100644 index 000000000000..af3136f83e4f --- /dev/null +++ b/drivers/md/dm-vdo/slab-depot.c @@ -0,0 +1,2404 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "slab-depot.h" + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "encodings.h" +#include "io-submitter.h" +#include "physical-zone.h" +#include "priority-table.h" +#include "recovery-journal.h" +#include "repair.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +static const u64 BYTES_PER_WORD = sizeof(u64); +static const bool NORMAL_OPERATION = true; + +/** + * get_lock() - Get the lock object for a slab journal block by sequence number. + * @journal: vdo_slab journal to retrieve from. + * @sequence_number: Sequence number of the block. + * + * Return: The lock object for the given sequence number. + */ +static inline struct journal_lock * __must_check get_lock(struct slab_journal *journal, + sequence_number_t sequence_number) +{ + return &journal->locks[sequence_number % journal->size]; +} + +static bool is_slab_open(struct vdo_slab *slab) +{ + return (!vdo_is_state_quiescing(&slab->state) && + !vdo_is_state_quiescent(&slab->state)); +} + +/** + * must_make_entries_to_flush() - Check whether there are entry waiters which should delay a flush. + * @journal: The journal to check. + * + * Return: true if there are no entry waiters, or if the slab is unrecovered. + */ +static inline bool __must_check must_make_entries_to_flush(struct slab_journal *journal) +{ + return ((journal->slab->status != VDO_SLAB_REBUILDING) && + vdo_has_waiters(&journal->entry_waiters)); +} + +/** + * is_reaping() - Check whether a reap is currently in progress. + * @journal: The journal which may be reaping. + * + * Return: true if the journal is reaping. + */ +static inline bool __must_check is_reaping(struct slab_journal *journal) +{ + return (journal->head != journal->unreapable); +} + +/** + * initialize_tail_block() - Initialize tail block as a new block. + * @journal: The journal whose tail block is being initialized. + */ +static void initialize_tail_block(struct slab_journal *journal) +{ + struct slab_journal_block_header *header = &journal->tail_header; + + header->sequence_number = journal->tail; + header->entry_count = 0; + header->has_block_map_increments = false; +} + +/** + * initialize_journal_state() - Set all journal fields appropriately to start journaling. + * @journal: The journal to be reset, based on its tail sequence number. + */ +static void initialize_journal_state(struct slab_journal *journal) +{ + journal->unreapable = journal->head; + journal->reap_lock = get_lock(journal, journal->unreapable); + journal->next_commit = journal->tail; + journal->summarized = journal->last_summarized = journal->tail; + initialize_tail_block(journal); +} + +/** + * block_is_full() - Check whether a journal block is full. + * @journal: The slab journal for the block. + * + * Return: true if the tail block is full. + */ +static bool __must_check block_is_full(struct slab_journal *journal) +{ + journal_entry_count_t count = journal->tail_header.entry_count; + + return (journal->tail_header.has_block_map_increments ? + (journal->full_entries_per_block == count) : + (journal->entries_per_block == count)); +} + +static void add_entries(struct slab_journal *journal); +static void update_tail_block_location(struct slab_journal *journal); +static void release_journal_locks(struct waiter *waiter, void *context); + +/** + * is_slab_journal_blank() - Check whether a slab's journal is blank. + * + * A slab journal is blank if it has never had any entries recorded in it. + * + * Return: true if the slab's journal has never been modified. + */ +static bool is_slab_journal_blank(const struct vdo_slab *slab) +{ + return ((slab->journal.tail == 1) && + (slab->journal.tail_header.entry_count == 0)); +} + +/** + * mark_slab_journal_dirty() - Put a slab journal on the dirty ring of its allocator in the correct + * order. + * @journal: The journal to be marked dirty. + * @lock: The recovery journal lock held by the slab journal. + */ +static void mark_slab_journal_dirty(struct slab_journal *journal, sequence_number_t lock) +{ + struct slab_journal *dirty_journal; + struct list_head *dirty_list = &journal->slab->allocator->dirty_slab_journals; + + ASSERT_LOG_ONLY(journal->recovery_lock == 0, "slab journal was clean"); + + journal->recovery_lock = lock; + list_for_each_entry_reverse(dirty_journal, dirty_list, dirty_entry) { + if (dirty_journal->recovery_lock <= journal->recovery_lock) + break; + } + + list_move_tail(&journal->dirty_entry, dirty_journal->dirty_entry.next); +} + +static void mark_slab_journal_clean(struct slab_journal *journal) +{ + journal->recovery_lock = 0; + list_del_init(&journal->dirty_entry); +} + +static void check_if_slab_drained(struct vdo_slab *slab) +{ + bool read_only; + struct slab_journal *journal = &slab->journal; + const struct admin_state_code *code; + + if (!vdo_is_state_draining(&slab->state) || + must_make_entries_to_flush(journal) || + is_reaping(journal) || + journal->waiting_to_commit || + !list_empty(&journal->uncommitted_blocks) || + journal->updating_slab_summary || + (slab->active_count > 0)) + return; + + /* When not suspending or recovering, the slab must be clean. */ + code = vdo_get_admin_state_code(&slab->state); + read_only = vdo_is_read_only(slab->allocator->depot->vdo); + if (!read_only && + vdo_has_waiters(&slab->dirty_blocks) && + (code != VDO_ADMIN_STATE_SUSPENDING) && + (code != VDO_ADMIN_STATE_RECOVERING)) + return; + + vdo_finish_draining_with_result(&slab->state, + (read_only ? VDO_READ_ONLY : VDO_SUCCESS)); +} + +/** + * finish_reaping() - Actually advance the head of the journal now that any necessary flushes are + * complete. + * @journal: The journal to be reaped. + */ +static void finish_reaping(struct slab_journal *journal) +{ + journal->head = journal->unreapable; + add_entries(journal); + check_if_slab_drained(journal->slab); +} + +static void reap_slab_journal(struct slab_journal *journal); + +/** + * complete_reaping() - Finish reaping now that we have flushed the lower layer and then try + * reaping again in case we deferred reaping due to an outstanding vio. + * @completion: The flush vio. + */ +static void complete_reaping(struct vdo_completion *completion) +{ + struct slab_journal *journal = completion->parent; + + return_vio_to_pool(journal->slab->allocator->vio_pool, + vio_as_pooled_vio(as_vio(uds_forget(completion)))); + finish_reaping(journal); + reap_slab_journal(journal); +} + +/** + * handle_flush_error() - Handle an error flushing the lower layer. + * @completion: The flush vio. + */ +static void handle_flush_error(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + vdo_enter_read_only_mode(completion->vdo, completion->result); + complete_reaping(completion); +} + +static void flush_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, complete_reaping, + journal->slab->allocator->thread_id); +} + +/** + * flush_for_reaping() - A waiter callback for getting a vio with which to flush the lower layer + * prior to reaping. + * @waiter: The journal as a flush waiter. + * @context: The newly acquired flush vio. + */ +static void flush_for_reaping(struct waiter *waiter, void *context) +{ + struct slab_journal *journal = + container_of(waiter, struct slab_journal, flush_waiter); + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + + vio->completion.parent = journal; + submit_flush_vio(vio, flush_endio, handle_flush_error); +} + +/** + * reap_slab_journal() - Conduct a reap on a slab journal to reclaim unreferenced blocks. + * @journal: The slab journal. + */ +static void reap_slab_journal(struct slab_journal *journal) +{ + bool reaped = false; + + if (is_reaping(journal)) { + /* We already have a reap in progress so wait for it to finish. */ + return; + } + + if ((journal->slab->status != VDO_SLAB_REBUILT) || + !vdo_is_state_normal(&journal->slab->state) || + vdo_is_read_only(journal->slab->allocator->depot->vdo)) { + /* + * We must not reap in the first two cases, and there's no point in read-only mode. + */ + return; + } + + /* + * Start reclaiming blocks only when the journal head has no references. Then stop when a + * block is referenced or reap reaches the most recently written block, referenced by the + * slab summary, which has the sequence number just before the tail. + */ + while ((journal->unreapable < journal->tail) && (journal->reap_lock->count == 0)) { + reaped = true; + journal->unreapable++; + journal->reap_lock++; + if (journal->reap_lock == &journal->locks[journal->size]) + journal->reap_lock = &journal->locks[0]; + } + + if (!reaped) + return; + + /* + * It is never safe to reap a slab journal block without first issuing a flush, regardless + * of whether a user flush has been received or not. In the absence of the flush, the + * reference block write which released the locks allowing the slab journal to reap may not + * be persisted. Although slab summary writes will eventually issue flushes, multiple slab + * journal block writes can be issued while previous slab summary updates have not yet been + * made. Even though those slab journal block writes will be ignored if the slab summary + * update is not persisted, they may still overwrite the to-be-reaped slab journal block + * resulting in a loss of reference count updates (VDO-2912). + */ + journal->flush_waiter.callback = flush_for_reaping; + acquire_vio_from_pool(journal->slab->allocator->vio_pool, + &journal->flush_waiter); +} + +/** + * adjust_slab_journal_block_reference() - Adjust the reference count for a slab journal block. + * @journal: The slab journal. + * @sequence_number: The journal sequence number of the referenced block. + * @adjustment: Amount to adjust the reference counter. + * + * Note that when the adjustment is negative, the slab journal will be reaped. + */ +static void adjust_slab_journal_block_reference(struct slab_journal *journal, + sequence_number_t sequence_number, + int adjustment) +{ + struct journal_lock *lock; + + if (sequence_number == 0) + return; + + if (journal->slab->status == VDO_SLAB_REPLAYING) { + /* Locks should not be used during offline replay. */ + return; + } + + ASSERT_LOG_ONLY((adjustment != 0), "adjustment must be non-zero"); + lock = get_lock(journal, sequence_number); + if (adjustment < 0) { + ASSERT_LOG_ONLY((-adjustment <= lock->count), + "adjustment %d of lock count %u for slab journal block %llu must not underflow", + adjustment, lock->count, + (unsigned long long) sequence_number); + } + + lock->count += adjustment; + if (lock->count == 0) + reap_slab_journal(journal); +} + +/** + * release_journal_locks() - Callback invoked after a slab summary update completes. + * @waiter: The slab summary waiter that has just been notified. + * @context: The result code of the update. + * + * Registered in the constructor on behalf of update_tail_block_location(). + * + * Implements waiter_callback_fn. + */ +static void release_journal_locks(struct waiter *waiter, void *context) +{ + sequence_number_t first, i; + struct slab_journal *journal = + container_of(waiter, struct slab_journal, slab_summary_waiter); + int result = *((int *) context); + + if (result != VDO_SUCCESS) { + if (result != VDO_READ_ONLY) { + /* + * Don't bother logging what might be lots of errors if we are already in + * read-only mode. + */ + uds_log_error_strerror(result, "failed slab summary update %llu", + (unsigned long long) journal->summarized); + } + + journal->updating_slab_summary = false; + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); + check_if_slab_drained(journal->slab); + return; + } + + if (journal->partial_write_in_progress && (journal->summarized == journal->tail)) { + journal->partial_write_in_progress = false; + add_entries(journal); + } + + first = journal->last_summarized; + journal->last_summarized = journal->summarized; + for (i = journal->summarized - 1; i >= first; i--) { + /* + * Release the lock the summarized block held on the recovery journal. (During + * replay, recovery_start will always be 0.) + */ + if (journal->recovery_journal != NULL) { + zone_count_t zone_number = journal->slab->allocator->zone_number; + struct journal_lock *lock = get_lock(journal, i); + + vdo_release_recovery_journal_block_reference(journal->recovery_journal, + lock->recovery_start, + VDO_ZONE_TYPE_PHYSICAL, + zone_number); + } + + /* + * Release our own lock against reaping for blocks that are committed. (This + * function will not change locks during replay.) + */ + adjust_slab_journal_block_reference(journal, i, -1); + } + + journal->updating_slab_summary = false; + + reap_slab_journal(journal); + + /* Check if the slab summary needs to be updated again. */ + update_tail_block_location(journal); +} + +/** + * update_tail_block_location() - Update the tail block location in the slab summary, if necessary. + * @journal: The slab journal that is updating its tail block location. + */ +static void update_tail_block_location(struct slab_journal *journal) +{ + block_count_t free_block_count; + struct vdo_slab *slab = journal->slab; + + if (journal->updating_slab_summary || + vdo_is_read_only(journal->slab->allocator->depot->vdo) || + (journal->last_summarized >= journal->next_commit)) { + check_if_slab_drained(slab); + return; + } + + if (slab->status != VDO_SLAB_REBUILT) { + u8 hint = slab->allocator->summary_entries[slab->slab_number].fullness_hint; + + free_block_count = ((block_count_t) hint) << slab->allocator->depot->hint_shift; + } else { + free_block_count = slab->free_blocks; + } + + journal->summarized = journal->next_commit; + journal->updating_slab_summary = true; + + /* + * Update slab summary as dirty. + * vdo_slab journal can only reap past sequence number 1 when all the ref counts for this + * slab have been written to the layer. Therefore, indicate that the ref counts must be + * loaded when the journal head has reaped past sequence number 1. + */ + update_slab_summary_entry(slab, &journal->slab_summary_waiter, + journal->summarized % journal->size, + (journal->head > 1), false, free_block_count); +} + +/** + * reopen_slab_journal() - Reopen a slab's journal by emptying it and then adding pending entries. + */ +static void reopen_slab_journal(struct vdo_slab *slab) +{ + struct slab_journal *journal = &slab->journal; + sequence_number_t block; + + ASSERT_LOG_ONLY(journal->tail_header.entry_count == 0, + "vdo_slab journal's active block empty before reopening"); + journal->head = journal->tail; + initialize_journal_state(journal); + + /* Ensure no locks are spuriously held on an empty journal. */ + for (block = 1; block <= journal->size; block++) { + ASSERT_LOG_ONLY((get_lock(journal, block)->count == 0), + "Scrubbed journal's block %llu is not locked", + (unsigned long long) block); + } + + add_entries(journal); +} + +static sequence_number_t get_committing_sequence_number(const struct pooled_vio *vio) +{ + const struct packed_slab_journal_block *block = + (const struct packed_slab_journal_block *) vio->vio.data; + + return __le64_to_cpu(block->header.sequence_number); +} + +/** + * complete_write() - Handle post-commit processing. + * @completion: The write vio as a completion. + * + * This is the callback registered by write_slab_journal_block(). + */ +static void complete_write(struct vdo_completion *completion) +{ + int result = completion->result; + struct pooled_vio *pooled = vio_as_pooled_vio(as_vio(completion)); + struct slab_journal *journal = completion->parent; + sequence_number_t committed = get_committing_sequence_number(pooled); + + list_del_init(&pooled->list_entry); + return_vio_to_pool(journal->slab->allocator->vio_pool, uds_forget(pooled)); + + if (result != VDO_SUCCESS) { + vio_record_metadata_io_error(as_vio(completion)); + uds_log_error_strerror(result, "cannot write slab journal block %llu", + (unsigned long long) committed); + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); + check_if_slab_drained(journal->slab); + return; + } + + WRITE_ONCE(journal->events->blocks_written, journal->events->blocks_written + 1); + + if (list_empty(&journal->uncommitted_blocks)) { + /* If no blocks are outstanding, then the commit point is at the tail. */ + journal->next_commit = journal->tail; + } else { + /* The commit point is always the beginning of the oldest incomplete block. */ + pooled = container_of(journal->uncommitted_blocks.next, + struct pooled_vio, list_entry); + journal->next_commit = get_committing_sequence_number(pooled); + } + + update_tail_block_location(journal); +} + +static void write_slab_journal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, complete_write, journal->slab->allocator->thread_id); +} + +/** + * write_slab_journal_block() - Write a slab journal block. + * @waiter: The vio pool waiter which was just notified. + * @context: The vio pool entry for the write. + * + * Callback from acquire_vio_from_pool() registered in commit_tail(). + */ +static void write_slab_journal_block(struct waiter *waiter, void *context) +{ + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + struct slab_journal *journal = + container_of(waiter, struct slab_journal, resource_waiter); + struct slab_journal_block_header *header = &journal->tail_header; + int unused_entries = journal->entries_per_block - header->entry_count; + physical_block_number_t block_number; + const struct admin_state_code *operation; + + header->head = journal->head; + list_add_tail(&pooled->list_entry, &journal->uncommitted_blocks); + vdo_pack_slab_journal_block_header(header, &journal->block->header); + + /* Copy the tail block into the vio. */ + memcpy(pooled->vio.data, journal->block, VDO_BLOCK_SIZE); + + ASSERT_LOG_ONLY(unused_entries >= 0, "vdo_slab journal block is not overfull"); + if (unused_entries > 0) { + /* + * Release the per-entry locks for any unused entries in the block we are about to + * write. + */ + adjust_slab_journal_block_reference(journal, header->sequence_number, + -unused_entries); + journal->partial_write_in_progress = !block_is_full(journal); + } + + block_number = journal->slab->journal_origin + + (header->sequence_number % journal->size); + vio->completion.parent = journal; + + /* + * This block won't be read in recovery until the slab summary is updated to refer to it. + * The slab summary update does a flush which is sufficient to protect us from VDO-2331. + */ + submit_metadata_vio(uds_forget(vio), block_number, write_slab_journal_endio, + complete_write, REQ_OP_WRITE); + + /* Since the write is submitted, the tail block structure can be reused. */ + journal->tail++; + initialize_tail_block(journal); + journal->waiting_to_commit = false; + + operation = vdo_get_admin_state_code(&journal->slab->state); + if (operation == VDO_ADMIN_STATE_WAITING_FOR_RECOVERY) { + vdo_finish_operation(&journal->slab->state, + (vdo_is_read_only(journal->slab->allocator->depot->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS)); + return; + } + + add_entries(journal); +} + +/** + * commit_tail() - Commit the tail block of the slab journal. + * @journal: The journal whose tail block should be committed. + */ +static void commit_tail(struct slab_journal *journal) +{ + if ((journal->tail_header.entry_count == 0) && must_make_entries_to_flush(journal)) { + /* + * There are no entries at the moment, but there are some waiters, so defer + * initiating the flush until those entries are ready to write. + */ + return; + } + + if (vdo_is_read_only(journal->slab->allocator->depot->vdo) || + journal->waiting_to_commit || + (journal->tail_header.entry_count == 0)) { + /* + * There is nothing to do since the tail block is empty, or writing, or the journal + * is in read-only mode. + */ + return; + } + + /* + * Since we are about to commit the tail block, this journal no longer needs to be on the + * ring of journals which the recovery journal might ask to commit. + */ + mark_slab_journal_clean(journal); + + journal->waiting_to_commit = true; + + journal->resource_waiter.callback = write_slab_journal_block; + acquire_vio_from_pool(journal->slab->allocator->vio_pool, + &journal->resource_waiter); +} + +/** + * encode_slab_journal_entry() - Encode a slab journal entry. + * @tail_header: The unpacked header for the block. + * @payload: The journal block payload to hold the entry. + * @sbn: The slab block number of the entry to encode. + * @operation: The type of the entry. + * @increment: True if this is an increment. + * + * Exposed for unit tests. + */ +static void encode_slab_journal_entry(struct slab_journal_block_header *tail_header, + slab_journal_payload *payload, + slab_block_number sbn, + enum journal_operation operation, + bool increment) +{ + journal_entry_count_t entry_number = tail_header->entry_count++; + + if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + if (!tail_header->has_block_map_increments) { + memset(payload->full_entries.entry_types, 0, + VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE); + tail_header->has_block_map_increments = true; + } + + payload->full_entries.entry_types[entry_number / 8] |= + ((u8)1 << (entry_number % 8)); + } + + vdo_pack_slab_journal_entry(&payload->entries[entry_number], sbn, increment); +} + +/** + * expand_journal_point() - Convert a recovery journal journal_point which refers to both an + * increment and a decrement to a single point which refers to one or the + * other. + * @recovery_point: The journal point to convert. + * @increment: Whether the current entry is an increment. + * + * Return: The expanded journal point + * + * Because each data_vio has but a single recovery journal point, but may need to make both + * increment and decrement entries in the same slab journal. In order to distinguish the two + * entries, the entry count of the expanded journal point is twice the actual recovery journal + * entry count for increments, and one more than that for decrements. + */ +static struct journal_point expand_journal_point(struct journal_point recovery_point, + bool increment) +{ + recovery_point.entry_count *= 2; + if (!increment) + recovery_point.entry_count++; + + return recovery_point; +} + +/** + * add_entry() - Actually add an entry to the slab journal, potentially firing off a write if a + * block becomes full. + * @journal: The slab journal to append to. + * @pbn: The pbn being adjusted. + * @operation: The type of entry to make. + * @increment: True if this is an increment. + * @recovery_point: The expanded recovery point. + * + * This function is synchronous. + */ +static void add_entry(struct slab_journal *journal, physical_block_number_t pbn, + enum journal_operation operation, bool increment, + struct journal_point recovery_point) +{ + struct packed_slab_journal_block *block = journal->block; + int result; + + result = ASSERT(vdo_before_journal_point(&journal->tail_header.recovery_point, + &recovery_point), + "recovery journal point is monotonically increasing, recovery point: %llu.%u, block recovery point: %llu.%u", + (unsigned long long) recovery_point.sequence_number, + recovery_point.entry_count, + (unsigned long long) journal->tail_header.recovery_point.sequence_number, + journal->tail_header.recovery_point.entry_count); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); + return; + } + + if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + result = ASSERT((journal->tail_header.entry_count < + journal->full_entries_per_block), + "block has room for full entries"); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, + result); + return; + } + } + + encode_slab_journal_entry(&journal->tail_header, &block->payload, + pbn - journal->slab->start, operation, increment); + journal->tail_header.recovery_point = recovery_point; + if (block_is_full(journal)) + commit_tail(journal); +} + +static inline block_count_t journal_length(const struct slab_journal *journal) +{ + return journal->tail - journal->head; +} + +/** + * vdo_attempt_replay_into_slab() - Replay a recovery journal entry into a slab's journal. + * @slab: The slab to play into. + * @pbn: The PBN for the entry. + * @operation: The type of entry to add. + * @increment: True if this entry is an increment. + * @recovery_point: The recovery journal point corresponding to this entry. + * @parent: The completion to notify when there is space to add the entry if the entry could not be + * added immediately. + * + * Return: true if the entry was added immediately. + */ +bool vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn, + enum journal_operation operation, bool increment, + struct journal_point *recovery_point, + struct vdo_completion *parent) +{ + struct slab_journal *journal = &slab->journal; + struct slab_journal_block_header *header = &journal->tail_header; + struct journal_point expanded = expand_journal_point(*recovery_point, increment); + + /* Only accept entries after the current recovery point. */ + if (!vdo_before_journal_point(&journal->tail_header.recovery_point, &expanded)) + return true; + + if ((header->entry_count >= journal->full_entries_per_block) && + (header->has_block_map_increments || (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING))) { + /* + * The tail block does not have room for the entry we are attempting to add so + * commit the tail block now. + */ + commit_tail(journal); + } + + if (journal->waiting_to_commit) { + vdo_start_operation_with_waiter(&journal->slab->state, + VDO_ADMIN_STATE_WAITING_FOR_RECOVERY, + parent, NULL); + return false; + } + + if (journal_length(journal) >= journal->size) { + /* + * We must have reaped the current head before the crash, since the blocked + * threshold keeps us from having more entries than fit in a slab journal; hence we + * can just advance the head (and unreapable block), as needed. + */ + journal->head++; + journal->unreapable++; + } + + if (journal->slab->status == VDO_SLAB_REBUILT) + journal->slab->status = VDO_SLAB_REPLAYING; + + add_entry(journal, pbn, operation, increment, expanded); + return true; +} + +/** + * requires_reaping() - Check whether the journal must be reaped before adding new entries. + * @journal: The journal to check. + * + * Return: true if the journal must be reaped. + */ +static bool requires_reaping(const struct slab_journal *journal) +{ + return (journal_length(journal) >= journal->blocking_threshold); +} + +/** finish_summary_update() - A waiter callback that resets the writing state of a slab. */ +static void finish_summary_update(struct waiter *waiter, void *context) +{ + struct vdo_slab *slab = container_of(waiter, struct vdo_slab, summary_waiter); + int result = *((int *) context); + + slab->active_count--; + + if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { + uds_log_error_strerror(result, "failed to update slab summary"); + vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); + } + + check_if_slab_drained(slab); +} + +static void write_reference_block(struct waiter *waiter, void *context); + +/** + * launch_reference_block_write() - Launch the write of a dirty reference block by first acquiring + * a VIO for it from the pool. + * @waiter: The waiter of the block which is starting to write. + * @context: The parent slab of the block. + * + * This can be asynchronous since the writer will have to wait if all VIOs in the pool are + * currently in use. + */ +static void launch_reference_block_write(struct waiter *waiter, void *context) +{ + struct vdo_slab *slab = context; + + if (vdo_is_read_only(slab->allocator->depot->vdo)) + return; + + slab->active_count++; + container_of(waiter, struct reference_block, waiter)->is_writing = true; + waiter->callback = write_reference_block; + acquire_vio_from_pool(slab->allocator->vio_pool, waiter); +} + +static void save_dirty_reference_blocks(struct vdo_slab *slab) +{ + vdo_notify_all_waiters(&slab->dirty_blocks, launch_reference_block_write, slab); + check_if_slab_drained(slab); +} + +/** + * finish_reference_block_write() - After a reference block has written, clean it, release its + * locks, and return its VIO to the pool. + * @completion: The VIO that just finished writing. + */ +static void finish_reference_block_write(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = vio_as_pooled_vio(vio); + struct reference_block *block = completion->parent; + struct vdo_slab *slab = block->slab; + tail_block_offset_t offset; + + slab->active_count--; + + /* Release the slab journal lock. */ + adjust_slab_journal_block_reference(&slab->journal, + block->slab_journal_lock_to_release, -1); + return_vio_to_pool(slab->allocator->vio_pool, pooled); + + /* + * We can't clear the is_writing flag earlier as releasing the slab journal lock may cause + * us to be dirtied again, but we don't want to double enqueue. + */ + block->is_writing = false; + + if (vdo_is_read_only(completion->vdo)) { + check_if_slab_drained(slab); + return; + } + + /* Re-queue the block if it was re-dirtied while it was writing. */ + if (block->is_dirty) { + vdo_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); + if (vdo_is_state_draining(&slab->state)) { + /* We must be saving, and this block will otherwise not be relaunched. */ + save_dirty_reference_blocks(slab); + } + + return; + } + + /* + * Mark the slab as clean in the slab summary if there are no dirty or writing blocks + * and no summary update in progress. + */ + if ((slab->active_count > 0) || vdo_has_waiters(&slab->dirty_blocks)) { + check_if_slab_drained(slab); + return; + } + + offset = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; + slab->active_count++; + slab->summary_waiter.callback = finish_summary_update; + update_slab_summary_entry(slab, &slab->summary_waiter, offset, + true, true, slab->free_blocks); +} + +/** + * get_reference_counters_for_block() - Find the reference counters for a given block. + * @block: The reference_block in question. + * + * Return: A pointer to the reference counters for this block. + */ +static vdo_refcount_t * __must_check get_reference_counters_for_block(struct reference_block *block) +{ + size_t block_index = block - block->slab->reference_blocks; + + return &block->slab->counters[block_index * COUNTS_PER_BLOCK]; +} + +/** + * pack_reference_block() - Copy data from a reference block to a buffer ready to be written out. + * @block: The block to copy. + * @buffer: The char buffer to fill with the packed block. + */ +static void pack_reference_block(struct reference_block *block, void *buffer) +{ + struct packed_reference_block *packed = buffer; + vdo_refcount_t *counters = get_reference_counters_for_block(block); + sector_count_t i; + struct packed_journal_point commit_point; + + vdo_pack_journal_point(&block->slab->slab_journal_point, &commit_point); + + for (i = 0; i < VDO_SECTORS_PER_BLOCK; i++) { + packed->sectors[i].commit_point = commit_point; + memcpy(packed->sectors[i].counts, counters + (i * COUNTS_PER_SECTOR), + (sizeof(vdo_refcount_t) * COUNTS_PER_SECTOR)); + } +} + +static void write_reference_block_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct reference_block *block = vio->completion.parent; + thread_id_t thread_id = block->slab->allocator->thread_id; + + continue_vio_after_io(vio, finish_reference_block_write, thread_id); +} + +/** + * handle_io_error() - Handle an I/O error reading or writing a reference count block. + * @completion: The VIO doing the I/O as a completion. + */ +static void handle_io_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct vio *vio = as_vio(completion); + struct vdo_slab *slab = ((struct reference_block *) completion->parent)->slab; + + vio_record_metadata_io_error(vio); + return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio)); + slab->active_count--; + vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); + check_if_slab_drained(slab); +} + +/** + * write_reference_block() - After a dirty block waiter has gotten a VIO from the VIO pool, copy + * its counters and associated data into the VIO, and launch the write. + * @waiter: The waiter of the dirty block. + * @context: The VIO returned by the pool. + */ +static void write_reference_block(struct waiter *waiter, void *context) +{ + size_t block_offset; + physical_block_number_t pbn; + struct pooled_vio *pooled = context; + struct vdo_completion *completion = &pooled->vio.completion; + struct reference_block *block = container_of(waiter, struct reference_block, + waiter); + + pack_reference_block(block, pooled->vio.data); + block_offset = (block - block->slab->reference_blocks); + pbn = (block->slab->ref_counts_origin + block_offset); + block->slab_journal_lock_to_release = block->slab_journal_lock; + completion->parent = block; + + /* + * Mark the block as clean, since we won't be committing any updates that happen after this + * moment. As long as VIO order is preserved, two VIOs updating this block at once will not + * cause complications. + */ + block->is_dirty = false; + + /* + * Flush before writing to ensure that the recovery journal and slab journal entries which + * cover this reference update are stable (VDO-2331). + */ + WRITE_ONCE(block->slab->allocator->ref_counts_statistics.blocks_written, + block->slab->allocator->ref_counts_statistics.blocks_written + 1); + + completion->callback_thread_id = ((struct block_allocator *) pooled->context)->thread_id; + submit_metadata_vio(&pooled->vio, pbn, write_reference_block_endio, + handle_io_error, REQ_OP_WRITE | REQ_PREFLUSH); +} + +static void reclaim_journal_space(struct slab_journal *journal) +{ + block_count_t length = journal_length(journal); + struct vdo_slab *slab = journal->slab; + block_count_t write_count = vdo_count_waiters(&slab->dirty_blocks); + block_count_t written; + + if ((length < journal->flushing_threshold) || (write_count == 0)) + return; + + /* The slab journal is over the first threshold, schedule some reference block writes. */ + WRITE_ONCE(journal->events->flush_count, journal->events->flush_count + 1); + if (length < journal->flushing_deadline) { + /* Schedule more writes the closer to the deadline we get. */ + write_count /= journal->flushing_deadline - length + 1; + write_count = max_t(block_count_t, write_count, 1); + } + + for (written = 0; written < write_count; written++) { + vdo_notify_next_waiter(&slab->dirty_blocks, + launch_reference_block_write, slab); + } +} + +/** + * reference_count_to_status() - Convert a reference count to a reference status. + * @count: The count to convert. + * + * Return: The appropriate reference status. + */ +static enum reference_status __must_check reference_count_to_status(vdo_refcount_t count) +{ + if (count == EMPTY_REFERENCE_COUNT) + return RS_FREE; + else if (count == 1) + return RS_SINGLE; + else if (count == PROVISIONAL_REFERENCE_COUNT) + return RS_PROVISIONAL; + else + return RS_SHARED; +} + +/** + * dirty_block() - Mark a reference count block as dirty, potentially adding it to the dirty queue + * if it wasn't already dirty. + * @block: The reference block to mark as dirty. + */ +static void dirty_block(struct reference_block *block) +{ + if (block->is_dirty) + return; + + block->is_dirty = true; + if (!block->is_writing) + vdo_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); +} + +/** + * get_reference_block() - Get the reference block that covers the given block index. + */ +static struct reference_block * __must_check get_reference_block(struct vdo_slab *slab, + slab_block_number index) +{ + return &slab->reference_blocks[index / COUNTS_PER_BLOCK]; +} + +/** + * slab_block_number_from_pbn() - Determine the index within the slab of a particular physical + * block number. + * @slab: The slab. + * @physical_block_number: The physical block number. + * @slab_block_number_ptr: A pointer to the slab block number. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check slab_block_number_from_pbn(struct vdo_slab *slab, + physical_block_number_t pbn, + slab_block_number *slab_block_number_ptr) +{ + u64 slab_block_number; + + if (pbn < slab->start) + return VDO_OUT_OF_RANGE; + + slab_block_number = pbn - slab->start; + if (slab_block_number >= slab->allocator->depot->slab_config.data_blocks) + return VDO_OUT_OF_RANGE; + + *slab_block_number_ptr = slab_block_number; + return VDO_SUCCESS; +} + +/** + * get_reference_counter() - Get the reference counter that covers the given physical block number. + * @slab: The slab to query. + * @pbn: The physical block number. + * @counter_ptr: A pointer to the reference counter. + */ +static int __must_check get_reference_counter(struct vdo_slab *slab, + physical_block_number_t pbn, + vdo_refcount_t **counter_ptr) +{ + slab_block_number index; + int result = slab_block_number_from_pbn(slab, pbn, &index); + + if (result != VDO_SUCCESS) + return result; + + *counter_ptr = &slab->counters[index]; + + return VDO_SUCCESS; +} + +static unsigned int calculate_slab_priority(struct vdo_slab *slab) +{ + block_count_t free_blocks = slab->free_blocks; + unsigned int unopened_slab_priority = slab->allocator->unopened_slab_priority; + unsigned int priority; + + /* + * Wholly full slabs must be the only ones with lowest priority, 0. + * + * Slabs that have never been opened (empty, newly initialized, and never been written to) + * have lower priority than previously opened slabs that have a significant number of free + * blocks. This ranking causes VDO to avoid writing physical blocks for the first time + * unless there are very few free blocks that have been previously written to. + * + * Since VDO doesn't discard blocks currently, reusing previously written blocks makes VDO + * a better client of any underlying storage that is thinly-provisioned (though discarding + * would be better). + * + * For all other slabs, the priority is derived from the logarithm of the number of free + * blocks. Slabs with the same order of magnitude of free blocks have the same priority. + * With 2^23 blocks, the priority will range from 1 to 25. The reserved + * unopened_slab_priority divides the range and is skipped by the logarithmic mapping. + */ + + if (free_blocks == 0) + return 0; + + if (is_slab_journal_blank(slab)) + return unopened_slab_priority; + + priority = (1 + ilog2(free_blocks)); + return ((priority < unopened_slab_priority) ? priority : priority + 1); +} + +/* + * Slabs are essentially prioritized by an approximation of the number of free blocks in the slab + * so slabs with lots of free blocks with be opened for allocation before slabs that have few free + * blocks. + */ +static void prioritize_slab(struct vdo_slab *slab) +{ + ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), + "a slab must not already be on a ring when prioritizing"); + slab->priority = calculate_slab_priority(slab); + vdo_priority_table_enqueue(slab->allocator->prioritized_slabs, + slab->priority, &slab->allocq_entry); +} + +/** + * adjust_free_block_count() - Adjust the free block count and (if needed) reprioritize the slab. + * @increment: should be true if the free block count went up. + */ +static void adjust_free_block_count(struct vdo_slab *slab, bool increment) +{ + struct block_allocator *allocator = slab->allocator; + + WRITE_ONCE(allocator->allocated_blocks, + allocator->allocated_blocks + (increment ? -1 : 1)); + + /* The open slab doesn't need to be reprioritized until it is closed. */ + if (slab == allocator->open_slab) + return; + + /* Don't bother adjusting the priority table if unneeded. */ + if (slab->priority == calculate_slab_priority(slab)) + return; + + /* + * Reprioritize the slab to reflect the new free block count by removing it from the table + * and re-enqueuing it with the new priority. + */ + vdo_priority_table_remove(allocator->prioritized_slabs, &slab->allocq_entry); + prioritize_slab(slab); +} + +/** + * increment_for_data() - Increment the reference count for a data block. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @old_status: The reference status of the data block before this increment. + * @lock: The pbn_lock associated with this increment (may be NULL). + * @counter_ptr: A pointer to the count for the data block (in, out). + * @adjust_block_count: Whether to update the allocator's free block count. + * + * Return: VDO_SUCCESS or an error. + */ +static int increment_for_data(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + enum reference_status old_status, + struct pbn_lock *lock, vdo_refcount_t *counter_ptr, + bool adjust_block_count) +{ + switch (old_status) { + case RS_FREE: + *counter_ptr = 1; + block->allocated_count++; + slab->free_blocks--; + if (adjust_block_count) + adjust_free_block_count(slab, false); + + break; + + case RS_PROVISIONAL: + *counter_ptr = 1; + break; + + default: + /* Single or shared */ + if (*counter_ptr >= MAXIMUM_REFERENCE_COUNT) { + return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + "Incrementing a block already having 254 references (slab %u, offset %u)", + slab->slab_number, block_number); + } + (*counter_ptr)++; + } + + if (lock != NULL) + vdo_unassign_pbn_lock_provisional_reference(lock); + return VDO_SUCCESS; +} + +/** + * decrement_for_data() - Decrement the reference count for a data block. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @old_status: The reference status of the data block before this decrement. + * @updater: The reference updater doing this operation in case we need to look up the pbn lock. + * @lock: The pbn_lock associated with the block being decremented (may be NULL). + * @counter_ptr: A pointer to the count for the data block (in, out). + * @adjust_block_count: Whether to update the allocator's free block count. + * + * Return: VDO_SUCCESS or an error. + */ +static int decrement_for_data(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + enum reference_status old_status, + struct reference_updater *updater, + vdo_refcount_t *counter_ptr, bool adjust_block_count) +{ + switch (old_status) { + case RS_FREE: + return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + "Decrementing free block at offset %u in slab %u", + block_number, slab->slab_number); + + case RS_PROVISIONAL: + case RS_SINGLE: + if (updater->zpbn.zone != NULL) { + struct pbn_lock *lock = vdo_get_physical_zone_pbn_lock(updater->zpbn.zone, + updater->zpbn.pbn); + + if (lock != NULL) { + /* + * There is a read lock on this block, so the block must not become + * unreferenced. + */ + *counter_ptr = PROVISIONAL_REFERENCE_COUNT; + vdo_assign_pbn_lock_provisional_reference(lock); + break; + } + } + + *counter_ptr = EMPTY_REFERENCE_COUNT; + block->allocated_count--; + slab->free_blocks++; + if (adjust_block_count) + adjust_free_block_count(slab, true); + + break; + + default: + /* Shared */ + (*counter_ptr)--; + } + + return VDO_SUCCESS; +} + +/** + * increment_for_block_map() - Increment the reference count for a block map page. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @old_status: The reference status of the block before this increment. + * @lock: The pbn_lock associated with this increment (may be NULL). + * @normal_operation: Whether we are in normal operation vs. recovery or rebuild. + * @counter_ptr: A pointer to the count for the block (in, out). + * @adjust_block_count: Whether to update the allocator's free block count. + * + * All block map increments should be from provisional to MAXIMUM_REFERENCE_COUNT. Since block map + * blocks never dedupe they should never be adjusted from any other state. The adjustment always + * results in MAXIMUM_REFERENCE_COUNT as this value is used to prevent dedupe against block map + * blocks. + * + * Return: VDO_SUCCESS or an error. + */ +static int increment_for_block_map(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + enum reference_status old_status, + struct pbn_lock *lock, bool normal_operation, + vdo_refcount_t *counter_ptr, bool adjust_block_count) +{ + switch (old_status) { + case RS_FREE: + if (normal_operation) { + return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + "Incrementing unallocated block map block (slab %u, offset %u)", + slab->slab_number, block_number); + } + + *counter_ptr = MAXIMUM_REFERENCE_COUNT; + block->allocated_count++; + slab->free_blocks--; + if (adjust_block_count) + adjust_free_block_count(slab, false); + + return VDO_SUCCESS; + + case RS_PROVISIONAL: + if (!normal_operation) + return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + "Block map block had provisional reference during replay (slab %u, offset %u)", + slab->slab_number, block_number); + + *counter_ptr = MAXIMUM_REFERENCE_COUNT; + if (lock != NULL) + vdo_unassign_pbn_lock_provisional_reference(lock); + return VDO_SUCCESS; + + default: + return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + "Incrementing a block map block which is already referenced %u times (slab %u, offset %u)", + *counter_ptr, slab->slab_number, + block_number); + } +} + +static bool __must_check is_valid_journal_point(const struct journal_point *point) +{ + return ((point != NULL) && (point->sequence_number > 0)); +} + +/** + * update_reference_count() - Update the reference count of a block. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @slab_journal_point: The slab journal point at which this update is journaled. + * @updater: The reference updater. + * @normal_operation: Whether we are in normal operation vs. recovery or rebuild. + * @adjust_block_count: Whether to update the slab's free block count. + * @provisional_decrement_ptr: A pointer which will be set to true if this update was a decrement + * of a provisional reference. + * + * Return: VDO_SUCCESS or an error. + */ +static int update_reference_count(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + const struct journal_point *slab_journal_point, + struct reference_updater *updater, + bool normal_operation, bool adjust_block_count, + bool *provisional_decrement_ptr) +{ + vdo_refcount_t *counter_ptr = &slab->counters[block_number]; + enum reference_status old_status = reference_count_to_status(*counter_ptr); + int result; + + if (!updater->increment) { + result = decrement_for_data(slab, block, block_number, old_status, + updater, counter_ptr, adjust_block_count); + if ((result == VDO_SUCCESS) && (old_status == RS_PROVISIONAL)) { + if (provisional_decrement_ptr != NULL) + *provisional_decrement_ptr = true; + return VDO_SUCCESS; + } + } else if (updater->operation == VDO_JOURNAL_DATA_REMAPPING) { + result = increment_for_data(slab, block, block_number, old_status, + updater->lock, counter_ptr, adjust_block_count); + } else { + result = increment_for_block_map(slab, block, block_number, old_status, + updater->lock, normal_operation, + counter_ptr, adjust_block_count); + } + + if (result != VDO_SUCCESS) + return result; + + if (is_valid_journal_point(slab_journal_point)) + slab->slab_journal_point = *slab_journal_point; + + return VDO_SUCCESS; +} + +static int __must_check adjust_reference_count(struct vdo_slab *slab, + struct reference_updater *updater, + const struct journal_point *slab_journal_point) +{ + slab_block_number block_number; + int result; + struct reference_block *block; + bool provisional_decrement = false; + + if (!is_slab_open(slab)) + return VDO_INVALID_ADMIN_STATE; + + result = slab_block_number_from_pbn(slab, updater->zpbn.pbn, &block_number); + if (result != VDO_SUCCESS) + return result; + + block = get_reference_block(slab, block_number); + result = update_reference_count(slab, block, block_number, slab_journal_point, + updater, NORMAL_OPERATION, true, + &provisional_decrement); + if ((result != VDO_SUCCESS) || provisional_decrement) + return result; + + if (block->is_dirty && (block->slab_journal_lock > 0)) { + sequence_number_t entry_lock = slab_journal_point->sequence_number; + /* + * This block is already dirty and a slab journal entry has been made for it since + * the last time it was clean. We must release the per-entry slab journal lock for + * the entry associated with the update we are now doing. + */ + result = ASSERT(is_valid_journal_point(slab_journal_point), + "Reference count adjustments need slab journal points."); + if (result != VDO_SUCCESS) + return result; + + adjust_slab_journal_block_reference(&slab->journal, entry_lock, -1); + return VDO_SUCCESS; + } + + /* + * This may be the first time we are applying an update for which there is a slab journal + * entry to this block since the block was cleaned. Therefore, we convert the per-entry + * slab journal lock to an uncommitted reference block lock, if there is a per-entry lock. + */ + if (is_valid_journal_point(slab_journal_point)) + block->slab_journal_lock = slab_journal_point->sequence_number; + else + block->slab_journal_lock = 0; + + dirty_block(block); + return VDO_SUCCESS; +} + +/** + * add_entry_from_waiter() - Add an entry to the slab journal. + * @waiter: The vio which should make an entry now. + * @context: The slab journal to make an entry in. + * + * This callback is invoked by add_entries() once it has determined that we are ready to make + * another entry in the slab journal. Implements waiter_callback_fn. + */ +static void add_entry_from_waiter(struct waiter *waiter, void *context) +{ + int result; + struct reference_updater *updater = + container_of(waiter, struct reference_updater, waiter); + struct data_vio *data_vio = data_vio_from_reference_updater(updater); + struct slab_journal *journal = context; + struct slab_journal_block_header *header = &journal->tail_header; + struct journal_point slab_journal_point = { + .sequence_number = header->sequence_number, + .entry_count = header->entry_count, + }; + sequence_number_t recovery_block = data_vio->recovery_journal_point.sequence_number; + + if (header->entry_count == 0) { + /* + * This is the first entry in the current tail block, so get a lock on the recovery + * journal which we will hold until this tail block is committed. + */ + get_lock(journal, header->sequence_number)->recovery_start = recovery_block; + if (journal->recovery_journal != NULL) { + zone_count_t zone_number = journal->slab->allocator->zone_number; + + vdo_acquire_recovery_journal_block_reference(journal->recovery_journal, + recovery_block, + VDO_ZONE_TYPE_PHYSICAL, + zone_number); + } + + mark_slab_journal_dirty(journal, recovery_block); + reclaim_journal_space(journal); + } + + add_entry(journal, updater->zpbn.pbn, updater->operation, updater->increment, + expand_journal_point(data_vio->recovery_journal_point, + updater->increment)); + + if (journal->slab->status != VDO_SLAB_REBUILT) { + /* + * If the slab is unrecovered, scrubbing will take care of the count since the + * update is now recorded in the journal. + */ + adjust_slab_journal_block_reference(journal, + slab_journal_point.sequence_number, -1); + result = VDO_SUCCESS; + } else { + /* Now that an entry has been made in the slab journal, update the counter. */ + result = adjust_reference_count(journal->slab, updater, + &slab_journal_point); + } + + if (updater->increment) + continue_data_vio_with_error(data_vio, result); + else + vdo_continue_completion(&data_vio->decrement_completion, result); +} + +/** + * is_next_entry_a_block_map_increment() - Check whether the next entry to be made is a block map + * increment. + * @journal: The journal. + * + * Return: true if the first entry waiter's operation is a block map increment. + */ +static inline bool is_next_entry_a_block_map_increment(struct slab_journal *journal) +{ + struct waiter *waiter = vdo_get_first_waiter(&journal->entry_waiters); + struct reference_updater *updater = container_of(waiter, + struct reference_updater, + waiter); + + return (updater->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING); +} + +/** + * add_entries() - Add as many entries as possible from the queue of vios waiting to make entries. + * @journal: The journal to which entries may be added. + * + * By processing the queue in order, we ensure that slab journal entries are made in the same order + * as recovery journal entries for the same increment or decrement. + */ +static void add_entries(struct slab_journal *journal) +{ + if (journal->adding_entries) { + /* Protect against re-entrancy. */ + return; + } + + journal->adding_entries = true; + while (vdo_has_waiters(&journal->entry_waiters)) { + struct slab_journal_block_header *header = &journal->tail_header; + + if (journal->partial_write_in_progress || + (journal->slab->status == VDO_SLAB_REBUILDING)) { + /* + * Don't add entries while rebuilding or while a partial write is + * outstanding (VDO-2399). + */ + break; + } + + if (journal->waiting_to_commit) { + /* + * If we are waiting for resources to write the tail block, and the tail + * block is full, we can't make another entry. + */ + WRITE_ONCE(journal->events->tail_busy_count, + journal->events->tail_busy_count + 1); + break; + } else if (is_next_entry_a_block_map_increment(journal) && + (header->entry_count >= journal->full_entries_per_block)) { + /* + * The tail block does not have room for a block map increment, so commit + * it now. + */ + commit_tail(journal); + if (journal->waiting_to_commit) { + WRITE_ONCE(journal->events->tail_busy_count, + journal->events->tail_busy_count + 1); + break; + } + } + + /* If the slab is over the blocking threshold, make the vio wait. */ + if (requires_reaping(journal)) { + WRITE_ONCE(journal->events->blocked_count, + journal->events->blocked_count + 1); + save_dirty_reference_blocks(journal->slab); + break; + } + + if (header->entry_count == 0) { + struct journal_lock *lock = + get_lock(journal, header->sequence_number); + + /* + * Check if the on disk slab journal is full. Because of the blocking and + * scrubbing thresholds, this should never happen. + */ + if (lock->count > 0) { + ASSERT_LOG_ONLY((journal->head + journal->size) == journal->tail, + "New block has locks, but journal is not full"); + + /* + * The blocking threshold must let the journal fill up if the new + * block has locks; if the blocking threshold is smaller than the + * journal size, the new block cannot possibly have locks already. + */ + ASSERT_LOG_ONLY((journal->blocking_threshold >= journal->size), + "New block can have locks already iff blocking threshold is at the end of the journal"); + + WRITE_ONCE(journal->events->disk_full_count, + journal->events->disk_full_count + 1); + save_dirty_reference_blocks(journal->slab); + break; + } + + /* + * Don't allow the new block to be reaped until all of the reference count + * blocks are written and the journal block has been fully committed as + * well. + */ + lock->count = journal->entries_per_block + 1; + + if (header->sequence_number == 1) { + struct vdo_slab *slab = journal->slab; + block_count_t i; + + /* + * This is the first entry in this slab journal, ever. Dirty all of + * the reference count blocks. Each will acquire a lock on the tail + * block so that the journal won't be reaped until the reference + * counts are initialized. The lock acquisition must be done by the + * ref_counts since here we don't know how many reference blocks + * the ref_counts has. + */ + for (i = 0; i < slab->reference_block_count; i++) { + slab->reference_blocks[i].slab_journal_lock = 1; + dirty_block(&slab->reference_blocks[i]); + } + + adjust_slab_journal_block_reference(journal, 1, + slab->reference_block_count); + } + } + + vdo_notify_next_waiter(&journal->entry_waiters, + add_entry_from_waiter, journal); + } + + journal->adding_entries = false; + + /* If there are no waiters, and we are flushing or saving, commit the tail block. */ + if (vdo_is_state_draining(&journal->slab->state) && + !vdo_is_state_suspending(&journal->slab->state) && + !vdo_has_waiters(&journal->entry_waiters)) + commit_tail(journal); +} + +/** + * reset_search_cursor() - Reset the free block search back to the first reference counter in the + * first reference block of a slab. + */ +static void reset_search_cursor(struct vdo_slab *slab) +{ + struct search_cursor *cursor = &slab->search_cursor; + + cursor->block = cursor->first_block; + cursor->index = 0; + /* Unit tests have slabs with only one reference block (and it's a runt). */ + cursor->end_index = min_t(u32, COUNTS_PER_BLOCK, slab->block_count); +} + +/** + * advance_search_cursor() - Advance the search cursor to the start of the next reference block in + * a slab, + * + * Wraps around to the first reference block if the current block is the last reference block. + * + * Return: true unless the cursor was at the last reference block. + */ +static bool advance_search_cursor(struct vdo_slab *slab) +{ + struct search_cursor *cursor = &slab->search_cursor; + + /* + * If we just finished searching the last reference block, then wrap back around to the + * start of the array. + */ + if (cursor->block == cursor->last_block) { + reset_search_cursor(slab); + return false; + } + + /* We're not already at the end, so advance to cursor to the next block. */ + cursor->block++; + cursor->index = cursor->end_index; + + if (cursor->block == cursor->last_block) { + /* The last reference block will usually be a runt. */ + cursor->end_index = slab->block_count; + } else { + cursor->end_index += COUNTS_PER_BLOCK; + } + + return true; +} + +/** + * replay_reference_count_change() - Replay the reference count adjustment from a slab journal + * entry into the reference count for a block. + * @slab: The slab. + * @entry_point: The slab journal point for the entry. + * @entry: The slab journal entry being replayed. + * + * The adjustment will be ignored if it was already recorded in the reference count. + * + * Return: VDO_SUCCESS or an error code. + */ +static int replay_reference_count_change(struct vdo_slab *slab, + const struct journal_point *entry_point, + struct slab_journal_entry entry) +{ + int result; + struct reference_block *block = get_reference_block(slab, entry.sbn); + sector_count_t sector = (entry.sbn % COUNTS_PER_BLOCK) / COUNTS_PER_SECTOR; + struct reference_updater updater = { + .operation = entry.operation, + .increment = entry.increment, + }; + + if (!vdo_before_journal_point(&block->commit_points[sector], entry_point)) { + /* This entry is already reflected in the existing counts, so do nothing. */ + return VDO_SUCCESS; + } + + /* This entry is not yet counted in the reference counts. */ + result = update_reference_count(slab, block, entry.sbn, entry_point, + &updater, !NORMAL_OPERATION, false, NULL); + if (result != VDO_SUCCESS) + return result; + + dirty_block(block); + return VDO_SUCCESS; +} + +/** + * find_zero_byte_in_word() - Find the array index of the first zero byte in word-sized range of + * reference counters. + * @word_ptr: A pointer to the eight counter bytes to check. + * @start_index: The array index corresponding to word_ptr[0]. + * @fail_index: The array index to return if no zero byte is found. + * + * The search does no bounds checking; the function relies on the array being sufficiently padded. + * + * Return: The array index of the first zero byte in the word, or the value passed as fail_index if + * no zero byte was found. + */ +static inline slab_block_number find_zero_byte_in_word(const u8 *word_ptr, + slab_block_number start_index, + slab_block_number fail_index) +{ + u64 word = get_unaligned_le64(word_ptr); + + /* This looks like a loop, but GCC will unroll the eight iterations for us. */ + unsigned int offset; + + for (offset = 0; offset < BYTES_PER_WORD; offset++) { + /* Assumes little-endian byte order, which we have on X86. */ + if ((word & 0xFF) == 0) + return (start_index + offset); + word >>= 8; + } + + return fail_index; +} + +/** + * vdo_find_free_block() - Find the first block with a reference count of zero in the specified + * range of reference counter indexes. + * @slab: The slab counters to scan. + * @index_ptr: A pointer to hold the array index of the free block. + * + * Exposed for unit testing. + * + * Return: true if a free block was found in the specified range. + */ +static bool find_free_block(const struct vdo_slab *slab, slab_block_number *index_ptr) +{ + slab_block_number zero_index; + slab_block_number next_index = slab->search_cursor.index; + slab_block_number end_index = slab->search_cursor.end_index; + u8 *next_counter = &slab->counters[next_index]; + u8 *end_counter = &slab->counters[end_index]; + + /* + * Search every byte of the first unaligned word. (Array is padded so reading past end is + * safe.) + */ + zero_index = find_zero_byte_in_word(next_counter, next_index, end_index); + if (zero_index < end_index) { + *index_ptr = zero_index; + return true; + } + + /* + * On architectures where unaligned word access is expensive, this would be a good place to + * advance to an alignment boundary. + */ + next_index += BYTES_PER_WORD; + next_counter += BYTES_PER_WORD; + + /* + * Now we're word-aligned; check an word at a time until we find a word containing a zero. + * (Array is padded so reading past end is safe.) + */ + while (next_counter < end_counter) { + /* + * The following code is currently an exact copy of the code preceding the loop, + * but if you try to merge them by using a do loop, it runs slower because a jump + * instruction gets added at the start of the iteration. + */ + zero_index = find_zero_byte_in_word(next_counter, next_index, end_index); + if (zero_index < end_index) { + *index_ptr = zero_index; + return true; + } + + next_index += BYTES_PER_WORD; + next_counter += BYTES_PER_WORD; + } + + return false; +} + +/** + * search_current_reference_block() - Search the reference block currently saved in the search + * cursor for a reference count of zero, starting at the saved + * counter index. + * @slab: The slab to search. + * @free_index_ptr: A pointer to receive the array index of the zero reference count. + * + * Return: true if an unreferenced counter was found. + */ +static bool search_current_reference_block(const struct vdo_slab *slab, + slab_block_number *free_index_ptr) +{ + /* Don't bother searching if the current block is known to be full. */ + return ((slab->search_cursor.block->allocated_count < COUNTS_PER_BLOCK) && + find_free_block(slab, free_index_ptr)); +} + +/** + * search_reference_blocks() - Search each reference block for a reference count of zero. + * @slab: The slab to search. + * @free_index_ptr: A pointer to receive the array index of the zero reference count. + * + * Searches each reference block for a reference count of zero, starting at the reference block and + * counter index saved in the search cursor and searching up to the end of the last reference + * block. The search does not wrap. + * + * Return: true if an unreferenced counter was found. + */ +static bool search_reference_blocks(struct vdo_slab *slab, + slab_block_number *free_index_ptr) +{ + /* Start searching at the saved search position in the current block. */ + if (search_current_reference_block(slab, free_index_ptr)) + return true; + + /* Search each reference block up to the end of the slab. */ + while (advance_search_cursor(slab)) { + if (search_current_reference_block(slab, free_index_ptr)) + return true; + } + + return false; +} + +/** + * make_provisional_reference() - Do the bookkeeping for making a provisional reference. + */ +static void make_provisional_reference(struct vdo_slab *slab, + slab_block_number block_number) +{ + struct reference_block *block = get_reference_block(slab, block_number); + + /* + * Make the initial transition from an unreferenced block to a + * provisionally allocated block. + */ + slab->counters[block_number] = PROVISIONAL_REFERENCE_COUNT; + + /* Account for the allocation. */ + block->allocated_count++; + slab->free_blocks--; +} + +/** + * dirty_all_reference_blocks() - Mark all reference count blocks in a slab as dirty. + */ +static void dirty_all_reference_blocks(struct vdo_slab *slab) +{ + block_count_t i; + + for (i = 0; i < slab->reference_block_count; i++) + dirty_block(&slab->reference_blocks[i]); +} + +/** + * clear_provisional_references() - Clear the provisional reference counts from a reference block. + * @block: The block to clear. + */ +static void clear_provisional_references(struct reference_block *block) +{ + vdo_refcount_t *counters = get_reference_counters_for_block(block); + block_count_t j; + + for (j = 0; j < COUNTS_PER_BLOCK; j++) { + if (counters[j] == PROVISIONAL_REFERENCE_COUNT) { + counters[j] = EMPTY_REFERENCE_COUNT; + block->allocated_count--; + } + } +} + +static inline bool journal_points_equal(struct journal_point first, + struct journal_point second) +{ + return ((first.sequence_number == second.sequence_number) && + (first.entry_count == second.entry_count)); +} + +/** + * unpack_reference_block() - Unpack reference counts blocks into the internal memory structure. + * @packed: The written reference block to be unpacked. + * @block: The internal reference block to be loaded. + */ +static void unpack_reference_block(struct packed_reference_block *packed, + struct reference_block *block) +{ + block_count_t index; + sector_count_t i; + struct vdo_slab *slab = block->slab; + vdo_refcount_t *counters = get_reference_counters_for_block(block); + + for (i = 0; i < VDO_SECTORS_PER_BLOCK; i++) { + struct packed_reference_sector *sector = &packed->sectors[i]; + + vdo_unpack_journal_point(§or->commit_point, &block->commit_points[i]); + memcpy(counters + (i * COUNTS_PER_SECTOR), sector->counts, + (sizeof(vdo_refcount_t) * COUNTS_PER_SECTOR)); + /* The slab_journal_point must be the latest point found in any sector. */ + if (vdo_before_journal_point(&slab->slab_journal_point, + &block->commit_points[i])) + slab->slab_journal_point = block->commit_points[i]; + + if ((i > 0) && + !journal_points_equal(block->commit_points[0], + block->commit_points[i])) { + size_t block_index = block - block->slab->reference_blocks; + + uds_log_warning("Torn write detected in sector %u of reference block %zu of slab %u", + i, block_index, block->slab->slab_number); + } + } + + block->allocated_count = 0; + for (index = 0; index < COUNTS_PER_BLOCK; index++) { + if (counters[index] != EMPTY_REFERENCE_COUNT) + block->allocated_count++; + } +} + +/** + * finish_reference_block_load() - After a reference block has been read, unpack it. + * @completion: The VIO that just finished reading. + */ +static void finish_reference_block_load(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = vio_as_pooled_vio(vio); + struct reference_block *block = completion->parent; + struct vdo_slab *slab = block->slab; + + unpack_reference_block((struct packed_reference_block *) vio->data, block); + return_vio_to_pool(slab->allocator->vio_pool, pooled); + slab->active_count--; + clear_provisional_references(block); + + slab->free_blocks -= block->allocated_count; + check_if_slab_drained(slab); +} + +static void load_reference_block_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct reference_block *block = vio->completion.parent; + + continue_vio_after_io(vio, finish_reference_block_load, + block->slab->allocator->thread_id); +} + +/** + * load_reference_block() - After a block waiter has gotten a VIO from the VIO pool, load the + * block. + * @waiter: The waiter of the block to load. + * @context: The VIO returned by the pool. + */ +static void load_reference_block(struct waiter *waiter, void *context) +{ + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + struct reference_block *block = + container_of(waiter, struct reference_block, waiter); + size_t block_offset = (block - block->slab->reference_blocks); + + vio->completion.parent = block; + submit_metadata_vio(vio, block->slab->ref_counts_origin + block_offset, + load_reference_block_endio, handle_io_error, + REQ_OP_READ); +} + +/** + * load_reference_blocks() - Load a slab's reference blocks from the underlying storage into a + * pre-allocated reference counter. + */ +static void load_reference_blocks(struct vdo_slab *slab) +{ + block_count_t i; + + slab->free_blocks = slab->block_count; + slab->active_count = slab->reference_block_count; + for (i = 0; i < slab->reference_block_count; i++) { + struct waiter *waiter = &slab->reference_blocks[i].waiter; + + waiter->callback = load_reference_block; + acquire_vio_from_pool(slab->allocator->vio_pool, waiter); + } +} + +/** + * drain_slab() - Drain all reference count I/O. + * + * Depending upon the type of drain being performed (as recorded in the ref_count's vdo_slab), the + * reference blocks may be loaded from disk or dirty reference blocks may be written out. + */ +static void drain_slab(struct vdo_slab *slab) +{ + bool save; + bool load; + const struct admin_state_code *state = vdo_get_admin_state_code(&slab->state); + + if (state == VDO_ADMIN_STATE_SUSPENDING) + return; + + if ((state != VDO_ADMIN_STATE_REBUILDING) && + (state != VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING)) + commit_tail(&slab->journal); + + if ((state == VDO_ADMIN_STATE_RECOVERING) || (slab->counters == NULL)) + return; + + save = false; + load = slab->allocator->summary_entries[slab->slab_number].load_ref_counts; + if (state == VDO_ADMIN_STATE_SCRUBBING) { + if (load) { + load_reference_blocks(slab); + return; + } + } else if (state == VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING) { + if (!load) { + /* These reference counts were never written, so mark them all dirty. */ + dirty_all_reference_blocks(slab); + } + save = true; + } else if (state == VDO_ADMIN_STATE_REBUILDING) { + /* + * Write out the counters if the slab has written them before, or it has any + * non-zero reference counts, or there are any slab journal blocks. + */ + block_count_t data_blocks = slab->allocator->depot->slab_config.data_blocks; + + if (load || (slab->free_blocks != data_blocks) || + !is_slab_journal_blank(slab)) { + dirty_all_reference_blocks(slab); + save = true; + } + } else if (state == VDO_ADMIN_STATE_SAVING) { + save = (slab->status == VDO_SLAB_REBUILT); + } else { + vdo_finish_draining_with_result(&slab->state, VDO_SUCCESS); + return; + } + + if (save) + save_dirty_reference_blocks(slab); +} + +static int allocate_slab_counters(struct vdo_slab *slab) +{ + int result; + size_t index, bytes; + + result = ASSERT(slab->reference_blocks == NULL, + "vdo_slab %u doesn't allocate refcounts twice", + slab->slab_number); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(slab->reference_block_count, struct reference_block, + __func__, &slab->reference_blocks); + if (result != VDO_SUCCESS) + return result; + + /* + * Allocate such that the runt slab has a full-length memory array, plus a little padding + * so we can word-search even at the very end. + */ + bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD); + result = uds_allocate(bytes, vdo_refcount_t, "ref counts array", + &slab->counters); + if (result != UDS_SUCCESS) { + uds_free(uds_forget(slab->reference_blocks)); + return result; + } + + slab->search_cursor.first_block = slab->reference_blocks; + slab->search_cursor.last_block = &slab->reference_blocks[slab->reference_block_count - 1]; + reset_search_cursor(slab); + + for (index = 0; index < slab->reference_block_count; index++) { + slab->reference_blocks[index] = (struct reference_block) { + .slab = slab, + }; + } + + return VDO_SUCCESS; +} + +static int allocate_counters_if_clean(struct vdo_slab *slab) +{ + if (vdo_is_state_clean_load(&slab->state)) + return allocate_slab_counters(slab); + + return VDO_SUCCESS; +} + +static void finish_loading_journal(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct slab_journal *journal = completion->parent; + struct vdo_slab *slab = journal->slab; + struct packed_slab_journal_block *block = (struct packed_slab_journal_block *) vio->data; + struct slab_journal_block_header header; + + vdo_unpack_slab_journal_block_header(&block->header, &header); + + /* FIXME: should it be an error if the following conditional fails? */ + if ((header.metadata_type == VDO_METADATA_SLAB_JOURNAL) && + (header.nonce == slab->allocator->nonce)) { + journal->tail = header.sequence_number + 1; + + /* + * If the slab is clean, this implies the slab journal is empty, so advance the + * head appropriately. + */ + journal->head = (slab->allocator->summary_entries[slab->slab_number].is_dirty ? + header.head : journal->tail); + journal->tail_header = header; + initialize_journal_state(journal); + } + + return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio)); + vdo_finish_loading_with_result(&slab->state, allocate_counters_if_clean(slab)); +} + +static void read_slab_journal_tail_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, finish_loading_journal, + journal->slab->allocator->thread_id); +} + +static void handle_load_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct slab_journal *journal = completion->parent; + struct vio *vio = as_vio(completion); + + vio_record_metadata_io_error(vio); + return_vio_to_pool(journal->slab->allocator->vio_pool, vio_as_pooled_vio(vio)); + vdo_finish_loading_with_result(&journal->slab->state, result); +} + +/** + * read_slab_journal_tail() - Read the slab journal tail block by using a vio acquired from the vio + * pool. + * @waiter: The vio pool waiter which has just been notified. + * @context: The vio pool entry given to the waiter. + * + * This is the success callback from acquire_vio_from_pool() when loading a slab journal. + */ +static void read_slab_journal_tail(struct waiter *waiter, void *context) +{ + struct slab_journal *journal = + container_of(waiter, struct slab_journal, resource_waiter); + struct vdo_slab *slab = journal->slab; + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + tail_block_offset_t last_commit_point = + slab->allocator->summary_entries[slab->slab_number].tail_block_offset; + + /* + * Slab summary keeps the commit point offset, so the tail block is the block before that. + * Calculation supports small journals in unit tests. + */ + tail_block_offset_t tail_block = ((last_commit_point == 0) ? + (tail_block_offset_t)(journal->size - 1) : + (last_commit_point - 1)); + + vio->completion.parent = journal; + vio->completion.callback_thread_id = slab->allocator->thread_id; + submit_metadata_vio(vio, slab->journal_origin + tail_block, + read_slab_journal_tail_endio, handle_load_error, + REQ_OP_READ); +} + +/** + * load_slab_journal() - Load a slab's journal by reading the journal's tail. + */ +static void load_slab_journal(struct vdo_slab *slab) +{ + struct slab_journal *journal = &slab->journal; + tail_block_offset_t last_commit_point; + + last_commit_point = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; + if ((last_commit_point == 0) && + !slab->allocator->summary_entries[slab->slab_number].load_ref_counts) { + /* + * This slab claims that it has a tail block at (journal->size - 1), but a head of + * 1. This is impossible, due to the scrubbing threshold, on a real system, so + * don't bother reading the (bogus) data off disk. + */ + ASSERT_LOG_ONLY(((journal->size < 16) || + (journal->scrubbing_threshold < (journal->size - 1))), + "Scrubbing threshold protects against reads of unwritten slab journal blocks"); + vdo_finish_loading_with_result(&slab->state, + allocate_counters_if_clean(slab)); + return; + } + + journal->resource_waiter.callback = read_slab_journal_tail; + acquire_vio_from_pool(slab->allocator->vio_pool, &journal->resource_waiter); +} + +static void free_slab(struct vdo_slab *slab) +{ + if (slab == NULL) + return; + + list_del(&slab->allocq_entry); + uds_free(uds_forget(slab->journal.block)); + uds_free(uds_forget(slab->journal.locks)); + uds_free(uds_forget(slab->counters)); + uds_free(uds_forget(slab->reference_blocks)); + uds_free(slab); +} + +static int initialize_slab_journal(struct vdo_slab *slab) +{ + struct slab_journal *journal = &slab->journal; + const struct slab_config *slab_config = &slab->allocator->depot->slab_config; + int result; + + result = uds_allocate(slab_config->slab_journal_blocks, struct journal_lock, + __func__, &journal->locks); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(VDO_BLOCK_SIZE, char, "struct packed_slab_journal_block", + (char **) &journal->block); + if (result != VDO_SUCCESS) + return result; + + journal->slab = slab; + journal->size = slab_config->slab_journal_blocks; + journal->flushing_threshold = slab_config->slab_journal_flushing_threshold; + journal->blocking_threshold = slab_config->slab_journal_blocking_threshold; + journal->scrubbing_threshold = slab_config->slab_journal_scrubbing_threshold; + journal->entries_per_block = VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK; + journal->full_entries_per_block = VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK; + journal->events = &slab->allocator->slab_journal_statistics; + journal->recovery_journal = slab->allocator->depot->vdo->recovery_journal; + journal->tail = 1; + journal->head = 1; + + journal->flushing_deadline = journal->flushing_threshold; + /* + * Set there to be some time between the deadline and the blocking threshold, so that + * hopefully all are done before blocking. + */ + if ((journal->blocking_threshold - journal->flushing_threshold) > 5) + journal->flushing_deadline = journal->blocking_threshold - 5; + + journal->slab_summary_waiter.callback = release_journal_locks; + + INIT_LIST_HEAD(&journal->dirty_entry); + INIT_LIST_HEAD(&journal->uncommitted_blocks); + + journal->tail_header.nonce = slab->allocator->nonce; + journal->tail_header.metadata_type = VDO_METADATA_SLAB_JOURNAL; + initialize_journal_state(journal); + return VDO_SUCCESS; +} + +/** + * make_slab() - Construct a new, empty slab. + * @slab_origin: The physical block number within the block allocator partition of the first block + * in the slab. + * @allocator: The block allocator to which the slab belongs. + * @slab_number: The slab number of the slab. + * @is_new: true if this slab is being allocated as part of a resize. + * @slab_ptr: A pointer to receive the new slab. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check make_slab(physical_block_number_t slab_origin, + struct block_allocator *allocator, + slab_count_t slab_number, bool is_new, + struct vdo_slab **slab_ptr) +{ + const struct slab_config *slab_config = &allocator->depot->slab_config; + struct vdo_slab *slab; + int result; + + result = uds_allocate(1, struct vdo_slab, __func__, &slab); + if (result != VDO_SUCCESS) + return result; + + *slab = (struct vdo_slab) { + .allocator = allocator, + .start = slab_origin, + .end = slab_origin + slab_config->slab_blocks, + .slab_number = slab_number, + .ref_counts_origin = slab_origin + slab_config->data_blocks, + .journal_origin = + vdo_get_slab_journal_start_block(slab_config, slab_origin), + .block_count = slab_config->data_blocks, + .free_blocks = slab_config->data_blocks, + .reference_block_count = + vdo_get_saved_reference_count_size(slab_config->data_blocks), + }; + INIT_LIST_HEAD(&slab->allocq_entry); + + result = initialize_slab_journal(slab); + if (result != VDO_SUCCESS) { + free_slab(slab); + return result; + } + + if (is_new) { + vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NEW); + result = allocate_slab_counters(slab); + if (result != VDO_SUCCESS) { + free_slab(slab); + return result; + } + } else { + vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + } + + *slab_ptr = slab; + return VDO_SUCCESS; +} diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h new file mode 100644 index 000000000000..98a2c4135e05 --- /dev/null +++ b/drivers/md/dm-vdo/slab-depot.h @@ -0,0 +1,252 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_SLAB_DEPOT_H +#define VDO_SLAB_DEPOT_H + +#include +#include +#include + +#include "numeric.h" + +#include "admin-state.h" +#include "completion.h" +#include "data-vio.h" +#include "encodings.h" +#include "physical-zone.h" +#include "priority-table.h" +#include "recovery-journal.h" +#include "statistics.h" +#include "types.h" +#include "vio.h" +#include "wait-queue.h" + +/* + * Represents the possible status of a block. + */ +enum reference_status { + RS_FREE, /* this block is free */ + RS_SINGLE, /* this block is singly-referenced */ + RS_SHARED, /* this block is shared */ + RS_PROVISIONAL /* this block is provisionally allocated */ +}; + +struct vdo_slab; + +struct journal_lock { + u16 count; + sequence_number_t recovery_start; +}; + +struct slab_journal { + /* A waiter object for getting a VIO pool entry */ + struct waiter resource_waiter; + /* A waiter object for updating the slab summary */ + struct waiter slab_summary_waiter; + /* A waiter object for getting a vio with which to flush */ + struct waiter flush_waiter; + /* The queue of VIOs waiting to make an entry */ + struct wait_queue entry_waiters; + /* The parent slab reference of this journal */ + struct vdo_slab *slab; + + /* Whether a tail block commit is pending */ + bool waiting_to_commit; + /* Whether the journal is updating the slab summary */ + bool updating_slab_summary; + /* Whether the journal is adding entries from the entry_waiters queue */ + bool adding_entries; + /* Whether a partial write is in progress */ + bool partial_write_in_progress; + + /* The oldest block in the journal on disk */ + sequence_number_t head; + /* The oldest block in the journal which may not be reaped */ + sequence_number_t unreapable; + /* The end of the half-open interval of the active journal */ + sequence_number_t tail; + /* The next journal block to be committed */ + sequence_number_t next_commit; + /* The tail sequence number that is written in the slab summary */ + sequence_number_t summarized; + /* The tail sequence number that was last summarized in slab summary */ + sequence_number_t last_summarized; + + /* The sequence number of the recovery journal lock */ + sequence_number_t recovery_lock; + + /* + * The number of entries which fit in a single block. Can't use the constant because unit + * tests change this number. + */ + journal_entry_count_t entries_per_block; + /* + * The number of full entries which fit in a single block. Can't use the constant because + * unit tests change this number. + */ + journal_entry_count_t full_entries_per_block; + + /* The recovery journal of the VDO (slab journal holds locks on it) */ + struct recovery_journal *recovery_journal; + + /* The statistics shared by all slab journals in our physical zone */ + struct slab_journal_statistics *events; + /* A list of the VIO pool entries for outstanding journal block writes */ + struct list_head uncommitted_blocks; + + /* + * The current tail block header state. This will be packed into the block just before it + * is written. + */ + struct slab_journal_block_header tail_header; + /* A pointer to a block-sized buffer holding the packed block data */ + struct packed_slab_journal_block *block; + + /* The number of blocks in the on-disk journal */ + block_count_t size; + /* The number of blocks at which to start pushing reference blocks */ + block_count_t flushing_threshold; + /* The number of blocks at which all reference blocks should be writing */ + block_count_t flushing_deadline; + /* The number of blocks at which to wait for reference blocks to write */ + block_count_t blocking_threshold; + /* The number of blocks at which to scrub the slab before coming online */ + block_count_t scrubbing_threshold; + + /* This list entry is for block_allocator to keep a queue of dirty journals */ + struct list_head dirty_entry; + + /* The lock for the oldest unreaped block of the journal */ + struct journal_lock *reap_lock; + /* The locks for each on disk block */ + struct journal_lock *locks; +}; + +/* + * Reference_block structure + * + * Blocks are used as a proxy, permitting saves of partial refcounts. + */ +struct reference_block { + /* This block waits on the ref_counts to tell it to write */ + struct waiter waiter; + /* The slab to which this reference_block belongs */ + struct vdo_slab *slab; + /* The number of references in this block that represent allocations */ + block_size_t allocated_count; + /* The slab journal block on which this block must hold a lock */ + sequence_number_t slab_journal_lock; + /* The slab journal block which should be released when this block is committed */ + sequence_number_t slab_journal_lock_to_release; + /* The point up to which each sector is accurate on disk */ + struct journal_point commit_points[VDO_SECTORS_PER_BLOCK]; + /* Whether this block has been modified since it was written to disk */ + bool is_dirty; + /* Whether this block is currently writing */ + bool is_writing; +}; + +/* The search_cursor represents the saved position of a free block search. */ +struct search_cursor { + /* The reference block containing the current search index */ + struct reference_block *block; + /* The position at which to start searching for the next free counter */ + slab_block_number index; + /* The position just past the last valid counter in the current block */ + slab_block_number end_index; + + /* A pointer to the first reference block in the slab */ + struct reference_block *first_block; + /* A pointer to the last reference block in the slab */ + struct reference_block *last_block; +}; + +enum slab_rebuild_status { + VDO_SLAB_REBUILT, + VDO_SLAB_REPLAYING, + VDO_SLAB_REQUIRES_SCRUBBING, + VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING, + VDO_SLAB_REBUILDING, +}; + +/* + * This is the type declaration for the vdo_slab type. A vdo_slab currently consists of a run of + * 2^23 data blocks, but that will soon change to dedicate a small number of those blocks for + * metadata storage for the reference counts and slab journal for the slab. + * + * A reference count is maintained for each physical block number. The vast majority of blocks have + * a very small reference count (usually 0 or 1). For references less than or equal to MAXIMUM_REFS + * (254) the reference count is stored in counters[pbn]. + */ +struct vdo_slab { + /* A list entry to queue this slab in a block_allocator list */ + struct list_head allocq_entry; + + /* The struct block_allocator that owns this slab */ + struct block_allocator *allocator; + + /* The journal for this slab */ + struct slab_journal journal; + + /* The slab number of this slab */ + slab_count_t slab_number; + /* The offset in the allocator partition of the first block in this slab */ + physical_block_number_t start; + /* The offset of the first block past the end of this slab */ + physical_block_number_t end; + /* The starting translated PBN of the slab journal */ + physical_block_number_t journal_origin; + /* The starting translated PBN of the reference counts */ + physical_block_number_t ref_counts_origin; + + /* The administrative state of the slab */ + struct admin_state state; + /* The status of the slab */ + enum slab_rebuild_status status; + /* Whether the slab was ever queued for scrubbing */ + bool was_queued_for_scrubbing; + + /* The priority at which this slab has been queued for allocation */ + u8 priority; + + /* Fields beyond this point are the reference counts for the data blocks in this slab. */ + /* The size of the counters array */ + u32 block_count; + /* The number of free blocks */ + u32 free_blocks; + /* The array of reference counts */ + vdo_refcount_t *counters; /* use uds_allocate() to align data ptr */ + + /* The saved block pointer and array indexes for the free block search */ + struct search_cursor search_cursor; + + /* A list of the dirty blocks waiting to be written out */ + struct wait_queue dirty_blocks; + /* The number of blocks which are currently writing */ + size_t active_count; + + /* A waiter object for updating the slab summary */ + struct waiter summary_waiter; + + /* The latest slab journal for which there has been a reference count update */ + struct journal_point slab_journal_point; + + /* The number of reference count blocks */ + u32 reference_block_count; + /* reference count block array */ + struct reference_block *reference_blocks; +}; + +bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, + physical_block_number_t pbn, + enum journal_operation operation, + bool increment, + struct journal_point *recovery_point, + struct vdo_completion *parent); + +void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion); + +#endif /* VDO_SLAB_DEPOT_H */ From 883069e30e0be28d792db04b97a1785848155d6b Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:57:50 -0500 Subject: [PATCH 0574/1038] dm vdo: add the slab summary The slab depot maintains an additional small data structure, the "slab summary," which is used to reduce the amount of work needed to come back online after a crash. The slab summary maintains an entry for each slab indicating whether or not the slab has ever been used, whether it is clean (i.e. all of its reference count updates have been persisted to storage), and approximately how full it is. During recovery, each physical zone will attempt to recover at least one slab, stopping whenever it has recovered a slab which has some free blocks. Once each zone has some space (or has determined that none is available), the target can resume normal operation in a degraded mode. Read and write requests can be serviced, perhaps with degraded performance, while the remainder of the dirty slabs are recovered. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/slab-depot.c | 320 +++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/slab-depot.h | 74 ++++++++ 2 files changed, 394 insertions(+) diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index af3136f83e4f..89af2e8a2621 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -193,6 +193,199 @@ static void check_if_slab_drained(struct vdo_slab *slab) (read_only ? VDO_READ_ONLY : VDO_SUCCESS)); } +/* FULLNESS HINT COMPUTATION */ + +/** + * compute_fullness_hint() - Translate a slab's free block count into a 'fullness hint' that can be + * stored in a slab_summary_entry's 7 bits that are dedicated to its free + * count. + * @depot: The depot whose summary being updated. + * @free_blocks: The number of free blocks. + * + * Note: the number of free blocks must be strictly less than 2^23 blocks, even though + * theoretically slabs could contain precisely 2^23 blocks; there is an assumption that at least + * one block is used by metadata. This assumption is necessary; otherwise, the fullness hint might + * overflow. The fullness hint formula is roughly (fullness >> 16) & 0x7f, but (2^23 >> 16) & 0x7f + * is 0, which would make it impossible to distinguish completely full from completely empty. + * + * Return: A fullness hint, which can be stored in 7 bits. + */ +static u8 __must_check compute_fullness_hint(struct slab_depot *depot, + block_count_t free_blocks) +{ + block_count_t hint; + + ASSERT_LOG_ONLY((free_blocks < (1 << 23)), "free blocks must be less than 2^23"); + + if (free_blocks == 0) + return 0; + + hint = free_blocks >> depot->hint_shift; + return ((hint == 0) ? 1 : hint); +} + +/** + * check_summary_drain_complete() - Check whether an allocators summary has finished draining. + */ +static void check_summary_drain_complete(struct block_allocator *allocator) +{ + struct vdo *vdo = allocator->depot->vdo; + + if (!vdo_is_state_draining(&allocator->summary_state) || + (allocator->summary_write_count > 0)) + return; + + vdo_finish_operation(&allocator->summary_state, + (vdo_is_read_only(vdo) ? VDO_READ_ONLY : VDO_SUCCESS)); +} + +/** + * notify_summary_waiters() - Wake all the waiters in a given queue. + * @allocator: The block allocator summary which owns the queue. + * @queue: The queue to notify. + */ +static void notify_summary_waiters(struct block_allocator *allocator, + struct wait_queue *queue) +{ + int result = (vdo_is_read_only(allocator->depot->vdo) ? VDO_READ_ONLY : VDO_SUCCESS); + + vdo_notify_all_waiters(queue, NULL, &result); +} + +static void launch_write(struct slab_summary_block *summary_block); + +/** + * finish_updating_slab_summary_block() - Finish processing a block which attempted to write, + * whether or not the attempt succeeded. + * @block: The block. + */ +static void finish_updating_slab_summary_block(struct slab_summary_block *block) +{ + notify_summary_waiters(block->allocator, &block->current_update_waiters); + block->writing = false; + block->allocator->summary_write_count--; + if (vdo_has_waiters(&block->next_update_waiters)) + launch_write(block); + else + check_summary_drain_complete(block->allocator); +} + +/** + * finish_update() - This is the callback for a successful summary block write. + * @completion: The write vio. + */ +static void finish_update(struct vdo_completion *completion) +{ + struct slab_summary_block *block = + container_of(as_vio(completion), struct slab_summary_block, vio); + + atomic64_inc(&block->allocator->depot->summary_statistics.blocks_written); + finish_updating_slab_summary_block(block); +} + +/** + * handle_write_error() - Handle an error writing a slab summary block. + * @completion: The write VIO. + */ +static void handle_write_error(struct vdo_completion *completion) +{ + struct slab_summary_block *block = + container_of(as_vio(completion), struct slab_summary_block, vio); + + vio_record_metadata_io_error(as_vio(completion)); + vdo_enter_read_only_mode(completion->vdo, completion->result); + finish_updating_slab_summary_block(block); +} + +static void write_slab_summary_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_summary_block *block = + container_of(vio, struct slab_summary_block, vio); + + continue_vio_after_io(vio, finish_update, block->allocator->thread_id); +} + +/** + * launch_write() - Write a slab summary block unless it is currently out for writing. + * @block: The block that needs to be committed. + */ +static void launch_write(struct slab_summary_block *block) +{ + struct block_allocator *allocator = block->allocator; + struct slab_depot *depot = allocator->depot; + physical_block_number_t pbn; + + if (block->writing) + return; + + allocator->summary_write_count++; + vdo_transfer_all_waiters(&block->next_update_waiters, + &block->current_update_waiters); + block->writing = true; + + if (vdo_is_read_only(depot->vdo)) { + finish_updating_slab_summary_block(block); + return; + } + + memcpy(block->outgoing_entries, block->entries, VDO_BLOCK_SIZE); + + /* + * Flush before writing to ensure that the slab journal tail blocks and reference updates + * covered by this summary update are stable (VDO-2332). + */ + pbn = (depot->summary_origin + + (VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * allocator->zone_number) + + block->index); + submit_metadata_vio(&block->vio, pbn, write_slab_summary_endio, + handle_write_error, REQ_OP_WRITE | REQ_PREFLUSH); +} + +/** + * update_slab_summary_entry() - Update the entry for a slab. + * @slab: The slab whose entry is to be updated + * @waiter: The waiter that is updating the summary. + * @tail_block_offset: The offset of the slab journal's tail block. + * @load_ref_counts: Whether the reference counts must be loaded from disk on the vdo load. + * @is_clean: Whether the slab is clean. + * @free_blocks: The number of free blocks. + */ +static void update_slab_summary_entry(struct vdo_slab *slab, struct waiter *waiter, + tail_block_offset_t tail_block_offset, + bool load_ref_counts, bool is_clean, + block_count_t free_blocks) +{ + u8 index = slab->slab_number / VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK; + struct block_allocator *allocator = slab->allocator; + struct slab_summary_block *block = &allocator->summary_blocks[index]; + int result; + struct slab_summary_entry *entry; + + if (vdo_is_read_only(block->vio.completion.vdo)) { + result = VDO_READ_ONLY; + waiter->callback(waiter, &result); + return; + } + + if (vdo_is_state_draining(&allocator->summary_state) || + vdo_is_state_quiescent(&allocator->summary_state)) { + result = VDO_INVALID_ADMIN_STATE; + waiter->callback(waiter, &result); + return; + } + + entry = &allocator->summary_entries[slab->slab_number]; + *entry = (struct slab_summary_entry) { + .tail_block_offset = tail_block_offset, + .load_ref_counts = (entry->load_ref_counts || load_ref_counts), + .is_dirty = !is_clean, + .fullness_hint = compute_fullness_hint(allocator->depot, free_blocks), + }; + vdo_enqueue_waiter(&block->next_update_waiters, waiter); + launch_write(block); +} + /** * finish_reaping() - Actually advance the head of the journal now that any necessary flushes are * complete. @@ -2402,3 +2595,130 @@ static int __must_check make_slab(physical_block_number_t slab_origin, *slab_ptr = slab; return VDO_SUCCESS; } + +/** + * finish_combining_zones() - Clean up after saving out the combined slab summary. + * @completion: The vio which was used to write the summary data. + */ +static void finish_combining_zones(struct vdo_completion *completion) +{ + int result = completion->result; + struct vdo_completion *parent = completion->parent; + + free_vio(as_vio(uds_forget(completion))); + vdo_fail_completion(parent, result); +} + +static void handle_combining_error(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + finish_combining_zones(completion); +} + +static void write_summary_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + + continue_vio_after_io(vio, finish_combining_zones, + vdo->thread_config.admin_thread); +} + +/** + * combine_summaries() - Treating the current entries buffer as the on-disk value of all zones, + * update every zone to the correct values for every slab. + * @depot: The depot whose summary entries should be combined. + */ +static void combine_summaries(struct slab_depot *depot) +{ + /* + * Combine all the old summary data into the portion of the buffer corresponding to the + * first zone. + */ + zone_count_t zone = 0; + struct slab_summary_entry *entries = depot->summary_entries; + + if (depot->old_zone_count > 1) { + slab_count_t entry_number; + + for (entry_number = 0; entry_number < MAX_VDO_SLABS; entry_number++) { + if (zone != 0) { + memcpy(entries + entry_number, + entries + (zone * MAX_VDO_SLABS) + entry_number, + sizeof(struct slab_summary_entry)); + } + + zone++; + if (zone == depot->old_zone_count) + zone = 0; + } + } + + /* Copy the combined data to each zones's region of the buffer. */ + for (zone = 1; zone < MAX_VDO_PHYSICAL_ZONES; zone++) { + memcpy(entries + (zone * MAX_VDO_SLABS), entries, + MAX_VDO_SLABS * sizeof(struct slab_summary_entry)); + } +} + +/** + * finish_loading_summary() - Finish loading slab summary data. + * @completion: The vio which was used to read the summary data. + * + * Combines the slab summary data from all the previously written zones and copies the combined + * summary to each partition's data region. Then writes the combined summary back out to disk. This + * callback is registered in load_summary_endio(). + */ +static void finish_loading_summary(struct vdo_completion *completion) +{ + struct slab_depot *depot = completion->vdo->depot; + + /* Combine the summary from each zone so each zone is correct for all slabs. */ + combine_summaries(depot); + + /* Write the combined summary back out. */ + submit_metadata_vio(as_vio(completion), depot->summary_origin, + write_summary_endio, handle_combining_error, + REQ_OP_WRITE); +} + +static void load_summary_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + + continue_vio_after_io(vio, finish_loading_summary, + vdo->thread_config.admin_thread); +} + +/** + * load_slab_summary() - The preamble of a load operation. + * + * Implements vdo_action_preamble_fn. + */ +static void load_slab_summary(void *context, struct vdo_completion *parent) +{ + int result; + struct vio *vio; + struct slab_depot *depot = context; + const struct admin_state_code *operation = + vdo_get_current_manager_operation(depot->action_manager); + + result = create_multi_block_metadata_vio(depot->vdo, VIO_TYPE_SLAB_SUMMARY, + VIO_PRIORITY_METADATA, parent, + VDO_SLAB_SUMMARY_BLOCKS, + (char *) depot->summary_entries, &vio); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + if ((operation == VDO_ADMIN_STATE_FORMATTING) || + (operation == VDO_ADMIN_STATE_LOADING_FOR_REBUILD)) { + finish_loading_summary(&vio->completion); + return; + } + + submit_metadata_vio(vio, depot->summary_origin, load_summary_endio, + handle_combining_error, REQ_OP_READ); +} diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h index 98a2c4135e05..298f2b7148d7 100644 --- a/drivers/md/dm-vdo/slab-depot.h +++ b/drivers/md/dm-vdo/slab-depot.h @@ -24,6 +24,23 @@ #include "vio.h" #include "wait-queue.h" +/* + * A slab_depot is responsible for managing all of the slabs and block allocators of a VDO. It has + * a single array of slabs in order to eliminate the need for additional math in order to compute + * which physical zone a PBN is in. It also has a block_allocator per zone. + * + * Load operations are required to be performed on a single thread. Normal operations are assumed + * to be performed in the appropriate zone. Allocations and reference count updates must be done + * from the thread of their physical zone. Requests to commit slab journal tail blocks from the + * recovery journal must be done on the journal zone thread. Save operations are required to be + * launched from the same thread as the original load operation. + */ + +enum { + /* The number of vios in the vio pool is proportional to the throughput of the VDO. */ + BLOCK_ALLOCATOR_VIO_POOL_SIZE = 128, +}; + /* * Represents the possible status of a block. */ @@ -240,6 +257,63 @@ struct vdo_slab { struct reference_block *reference_blocks; }; +/* + * The slab_summary provides hints during load and recovery about the state of the slabs in order + * to avoid the need to read the slab journals in their entirety before a VDO can come online. + * + * The information in the summary for each slab includes the rough number of free blocks (which is + * used to prioritize scrubbing), the cleanliness of a slab (so that clean slabs containing free + * space will be used on restart), and the location of the tail block of the slab's journal. + * + * The slab_summary has its own partition at the end of the volume which is sized to allow for a + * complete copy of the summary for each of up to 16 physical zones. + * + * During resize, the slab_summary moves its backing partition and is saved once moved; the + * slab_summary is not permitted to overwrite the previous recovery journal space. + * + * The slab_summary does not have its own version information, but relies on the VDO volume version + * number. + */ + +/* + * A slab status is a very small structure for use in determining the ordering of slabs in the + * scrubbing process. + */ +struct slab_status { + slab_count_t slab_number; + bool is_clean; + u8 emptiness; +}; + +struct slab_summary_block { + /* The block_allocator to which this block belongs */ + struct block_allocator *allocator; + /* The index of this block in its zone's summary */ + block_count_t index; + /* Whether this block has a write outstanding */ + bool writing; + /* Ring of updates waiting on the outstanding write */ + struct wait_queue current_update_waiters; + /* Ring of updates waiting on the next write */ + struct wait_queue next_update_waiters; + /* The active slab_summary_entry array for this block */ + struct slab_summary_entry *entries; + /* The vio used to write this block */ + struct vio vio; + /* The packed entries, one block long, backing the vio */ + char *outgoing_entries; +}; + +/* + * The statistics for all the slab summary zones owned by this slab summary. These fields are all + * mutated only by their physical zone threads, but are read by other threads when gathering + * statistics for the entire depot. + */ +struct atomic_slab_summary_statistics { + /* Number of blocks written */ + atomic64_t blocks_written; +}; + bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn, enum journal_operation operation, From c9ba9fd33cbb400589a8b3e6a612243cde71d1e6 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 20:58:33 -0500 Subject: [PATCH 0575/1038] dm vdo: add the block allocators and physical zones Each slab is independent of every other. They are assigned to "physical zones" in round-robin fashion. If there are P physical zones, then slab n is assigned to zone n mod P. The set of slabs in each physical zone is managed by a block allocator. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/physical-zone.c | 646 +++++++++++++ drivers/md/dm-vdo/physical-zone.h | 115 +++ drivers/md/dm-vdo/slab-depot.c | 1431 +++++++++++++++++++++++++++++ drivers/md/dm-vdo/slab-depot.h | 147 +++ 4 files changed, 2339 insertions(+) create mode 100644 drivers/md/dm-vdo/physical-zone.c create mode 100644 drivers/md/dm-vdo/physical-zone.h diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c new file mode 100644 index 000000000000..d3fc4666c3c2 --- /dev/null +++ b/drivers/md/dm-vdo/physical-zone.c @@ -0,0 +1,646 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "physical-zone.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "encodings.h" +#include "flush.h" +#include "int-map.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "vdo.h" + +enum { + /* Each user data_vio needs a PBN read lock and write lock. */ + LOCK_POOL_CAPACITY = 2 * MAXIMUM_VDO_USER_VIOS, +}; + +struct pbn_lock_implementation { + enum pbn_lock_type type; + const char *name; + const char *release_reason; +}; + +/* This array must have an entry for every pbn_lock_type value. */ +static const struct pbn_lock_implementation LOCK_IMPLEMENTATIONS[] = { + [VIO_READ_LOCK] = { + .type = VIO_READ_LOCK, + .name = "read", + .release_reason = "candidate duplicate", + }, + [VIO_WRITE_LOCK] = { + .type = VIO_WRITE_LOCK, + .name = "write", + .release_reason = "newly allocated", + }, + [VIO_BLOCK_MAP_WRITE_LOCK] = { + .type = VIO_BLOCK_MAP_WRITE_LOCK, + .name = "block map write", + .release_reason = "block map write", + }, +}; + +static inline bool has_lock_type(const struct pbn_lock *lock, enum pbn_lock_type type) +{ + return (lock->implementation == &LOCK_IMPLEMENTATIONS[type]); +} + +/** + * vdo_is_pbn_read_lock() - Check whether a pbn_lock is a read lock. + * @lock: The lock to check. + * + * Return: true if the lock is a read lock. + */ +bool vdo_is_pbn_read_lock(const struct pbn_lock *lock) +{ + return has_lock_type(lock, VIO_READ_LOCK); +} + +static inline void set_pbn_lock_type(struct pbn_lock *lock, enum pbn_lock_type type) +{ + lock->implementation = &LOCK_IMPLEMENTATIONS[type]; +} + +/** + * vdo_downgrade_pbn_write_lock() - Downgrade a PBN write lock to a PBN read lock. + * @lock: The PBN write lock to downgrade. + * + * The lock holder count is cleared and the caller is responsible for setting the new count. + */ +void vdo_downgrade_pbn_write_lock(struct pbn_lock *lock, bool compressed_write) +{ + ASSERT_LOG_ONLY(!vdo_is_pbn_read_lock(lock), + "PBN lock must not already have been downgraded"); + ASSERT_LOG_ONLY(!has_lock_type(lock, VIO_BLOCK_MAP_WRITE_LOCK), + "must not downgrade block map write locks"); + ASSERT_LOG_ONLY(lock->holder_count == 1, + "PBN write lock should have one holder but has %u", + lock->holder_count); + /* + * data_vio write locks are downgraded in place--the writer retains the hold on the lock. + * If this was a compressed write, the holder has not yet journaled its own inc ref, + * otherwise, it has. + */ + lock->increment_limit = + (compressed_write ? MAXIMUM_REFERENCE_COUNT : MAXIMUM_REFERENCE_COUNT - 1); + set_pbn_lock_type(lock, VIO_READ_LOCK); +} + +/** + * vdo_claim_pbn_lock_increment() - Try to claim one of the available reference count increments on + * a read lock. + * @lock: The PBN read lock from which to claim an increment. + * + * Claims may be attempted from any thread. A claim is only valid until the PBN lock is released. + * + * Return: true if the claim succeeded, guaranteeing one increment can be made without overflowing + * the PBN's reference count. + */ +bool vdo_claim_pbn_lock_increment(struct pbn_lock *lock) +{ + /* + * Claim the next free reference atomically since hash locks from multiple hash zone + * threads might be concurrently deduplicating against a single PBN lock on compressed + * block. As long as hitting the increment limit will lead to the PBN lock being released + * in a sane time-frame, we won't overflow a 32-bit claim counter, allowing a simple add + * instead of a compare-and-swap. + */ + u32 claim_number = (u32) atomic_add_return(1, &lock->increments_claimed); + + return (claim_number <= lock->increment_limit); +} + +/** + * vdo_assign_pbn_lock_provisional_reference() - Inform a PBN lock that it is responsible for a + * provisional reference. + * @lock: The PBN lock. + */ +void vdo_assign_pbn_lock_provisional_reference(struct pbn_lock *lock) +{ + ASSERT_LOG_ONLY(!lock->has_provisional_reference, + "lock does not have a provisional reference"); + lock->has_provisional_reference = true; +} + +/** + * vdo_unassign_pbn_lock_provisional_reference() - Inform a PBN lock that it is no longer + * responsible for a provisional reference. + * @lock: The PBN lock. + */ +void vdo_unassign_pbn_lock_provisional_reference(struct pbn_lock *lock) +{ + lock->has_provisional_reference = false; +} + +/** + * release_pbn_lock_provisional_reference() - If the lock is responsible for a provisional + * reference, release that reference. + * @lock: The lock. + * @locked_pbn: The PBN covered by the lock. + * @allocator: The block allocator from which to release the reference. + * + * This method is called when the lock is released. + */ +static void release_pbn_lock_provisional_reference(struct pbn_lock *lock, + physical_block_number_t locked_pbn, + struct block_allocator *allocator) +{ + int result; + + if (!vdo_pbn_lock_has_provisional_reference(lock)) + return; + + result = vdo_release_block_reference(allocator, locked_pbn); + if (result != VDO_SUCCESS) { + uds_log_error_strerror(result, + "Failed to release reference to %s physical block %llu", + lock->implementation->release_reason, + (unsigned long long) locked_pbn); + } + + vdo_unassign_pbn_lock_provisional_reference(lock); +} + +/** + * union idle_pbn_lock - PBN lock list entries. + * + * Unused (idle) PBN locks are kept in a list. Just like in a malloc implementation, the lock + * structure is unused memory, so we can save a bit of space (and not pollute the lock structure + * proper) by using a union to overlay the lock structure with the free list. + */ +typedef union { + /** @entry: Only used while locks are in the pool. */ + struct list_head entry; + /** @lock: Only used while locks are not in the pool. */ + struct pbn_lock lock; +} idle_pbn_lock; + +/** + * struct pbn_lock_pool - list of PBN locks. + * + * The lock pool is little more than the memory allocated for the locks. + */ +struct pbn_lock_pool { + /** @capacity: The number of locks allocated for the pool. */ + size_t capacity; + /** @borrowed: The number of locks currently borrowed from the pool. */ + size_t borrowed; + /** @idle_list: A list containing all idle PBN lock instances. */ + struct list_head idle_list; + /** @locks: The memory for all the locks allocated by this pool. */ + idle_pbn_lock locks[]; +}; + +/** + * return_pbn_lock_to_pool() - Return a pbn lock to its pool. + * @pool: The pool from which the lock was borrowed. + * @lock: The last reference to the lock being returned. + * + * It must be the last live reference, as if the memory were being freed (the lock memory will + * re-initialized or zeroed). + */ +static void return_pbn_lock_to_pool(struct pbn_lock_pool *pool, struct pbn_lock *lock) +{ + idle_pbn_lock *idle; + + /* A bit expensive, but will promptly catch some use-after-free errors. */ + memset(lock, 0, sizeof(*lock)); + + idle = container_of(lock, idle_pbn_lock, lock); + INIT_LIST_HEAD(&idle->entry); + list_add_tail(&idle->entry, &pool->idle_list); + + ASSERT_LOG_ONLY(pool->borrowed > 0, "shouldn't return more than borrowed"); + pool->borrowed -= 1; +} + +/** + * make_pbn_lock_pool() - Create a new PBN lock pool and all the lock instances it can loan out. + * + * @capacity: The number of PBN locks to allocate for the pool. + * @pool_ptr: A pointer to receive the new pool. + * + * Return: VDO_SUCCESS or an error code. + */ +static int make_pbn_lock_pool(size_t capacity, struct pbn_lock_pool **pool_ptr) +{ + size_t i; + struct pbn_lock_pool *pool; + int result; + + result = uds_allocate_extended(struct pbn_lock_pool, capacity, idle_pbn_lock, + __func__, &pool); + if (result != VDO_SUCCESS) + return result; + + pool->capacity = capacity; + pool->borrowed = capacity; + INIT_LIST_HEAD(&pool->idle_list); + + for (i = 0; i < capacity; i++) + return_pbn_lock_to_pool(pool, &pool->locks[i].lock); + + *pool_ptr = pool; + return VDO_SUCCESS; +} + +/** + * vdo_free_pbn_lock_pool() - Free a PBN lock pool. + * @pool: The lock pool to free. + * + * This also frees all the PBN locks it allocated, so the caller must ensure that all locks have + * been returned to the pool. + */ +static void free_pbn_lock_pool(struct pbn_lock_pool *pool) +{ + if (pool == NULL) + return; + + ASSERT_LOG_ONLY(pool->borrowed == 0, + "All PBN locks must be returned to the pool before it is freed, but %zu locks are still on loan", + pool->borrowed); + uds_free(pool); +} + +/** + * borrow_pbn_lock_from_pool() - Borrow a PBN lock from the pool and initialize it with the + * provided type. + * @pool: The pool from which to borrow. + * @type: The type with which to initialize the lock. + * @lock_ptr: A pointer to receive the borrowed lock. + * + * Pools do not grow on demand or allocate memory, so this will fail if the pool is empty. Borrowed + * locks are still associated with this pool and must be returned to only this pool. + * + * Return: VDO_SUCCESS, or VDO_LOCK_ERROR if the pool is empty. + */ +static int __must_check borrow_pbn_lock_from_pool(struct pbn_lock_pool *pool, + enum pbn_lock_type type, + struct pbn_lock **lock_ptr) +{ + int result; + struct list_head *idle_entry; + idle_pbn_lock *idle; + + if (pool->borrowed >= pool->capacity) + return uds_log_error_strerror(VDO_LOCK_ERROR, + "no free PBN locks left to borrow"); + pool->borrowed += 1; + + result = ASSERT(!list_empty(&pool->idle_list), + "idle list should not be empty if pool not at capacity"); + if (result != VDO_SUCCESS) + return result; + + idle_entry = pool->idle_list.prev; + list_del(idle_entry); + memset(idle_entry, 0, sizeof(*idle_entry)); + + idle = list_entry(idle_entry, idle_pbn_lock, entry); + idle->lock.holder_count = 0; + set_pbn_lock_type(&idle->lock, type); + + *lock_ptr = &idle->lock; + return VDO_SUCCESS; +} + +/** + * initialize_zone() - Initialize a physical zone. + * @vdo: The vdo to which the zone will belong. + * @zones: The physical_zones to which the zone being initialized belongs + * + * Return: VDO_SUCCESS or an error code. + */ +static int initialize_zone(struct vdo *vdo, struct physical_zones *zones) +{ + int result; + zone_count_t zone_number = zones->zone_count; + struct physical_zone *zone = &zones->zones[zone_number]; + + result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->pbn_operations); + if (result != VDO_SUCCESS) + return result; + + result = make_pbn_lock_pool(LOCK_POOL_CAPACITY, &zone->lock_pool); + if (result != VDO_SUCCESS) { + vdo_free_int_map(zone->pbn_operations); + return result; + } + + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.physical_threads[zone_number]; + zone->allocator = &vdo->depot->allocators[zone_number]; + zone->next = &zones->zones[(zone_number + 1) % vdo->thread_config.physical_zone_count]; + result = vdo_make_default_thread(vdo, zone->thread_id); + if (result != VDO_SUCCESS) { + free_pbn_lock_pool(uds_forget(zone->lock_pool)); + vdo_free_int_map(zone->pbn_operations); + return result; + } + return result; +} + +/** + * vdo_make_physical_zones() - Make the physical zones for a vdo. + * @vdo: The vdo being constructed + * @zones_ptr: A pointer to hold the zones + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_physical_zones(struct vdo *vdo, struct physical_zones **zones_ptr) +{ + struct physical_zones *zones; + int result; + zone_count_t zone_count = vdo->thread_config.physical_zone_count; + + if (zone_count == 0) + return VDO_SUCCESS; + + result = uds_allocate_extended(struct physical_zones, zone_count, + struct physical_zone, __func__, &zones); + if (result != VDO_SUCCESS) + return result; + + for (zones->zone_count = 0; zones->zone_count < zone_count; zones->zone_count++) { + result = initialize_zone(vdo, zones); + if (result != VDO_SUCCESS) { + vdo_free_physical_zones(zones); + return result; + } + } + + *zones_ptr = zones; + return VDO_SUCCESS; +} + +/** + * vdo_free_physical_zones() - Destroy the physical zones. + * @zones: The zones to free. + */ +void vdo_free_physical_zones(struct physical_zones *zones) +{ + zone_count_t index; + + if (zones == NULL) + return; + + for (index = 0; index < zones->zone_count; index++) { + struct physical_zone *zone = &zones->zones[index]; + + free_pbn_lock_pool(uds_forget(zone->lock_pool)); + vdo_free_int_map(uds_forget(zone->pbn_operations)); + } + + uds_free(zones); +} + +/** + * vdo_get_physical_zone_pbn_lock() - Get the lock on a PBN if one exists. + * @zone: The physical zone responsible for the PBN. + * @pbn: The physical block number whose lock is desired. + * + * Return: The lock or NULL if the PBN is not locked. + */ +struct pbn_lock *vdo_get_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn) +{ + return ((zone == NULL) ? NULL : vdo_int_map_get(zone->pbn_operations, pbn)); +} + +/** + * vdo_attempt_physical_zone_pbn_lock() - Attempt to lock a physical block in the zone responsible + * for it. + * @zone: The physical zone responsible for the PBN. + * @pbn: The physical block number to lock. + * @type: The type with which to initialize a new lock. + * @lock_ptr: A pointer to receive the lock, existing or new. + * + * If the PBN is already locked, the existing lock will be returned. Otherwise, a new lock instance + * will be borrowed from the pool, initialized, and returned. The lock owner will be NULL for a new + * lock acquired by the caller, who is responsible for setting that field promptly. The lock owner + * will be non-NULL when there is already an existing lock on the PBN. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn, + enum pbn_lock_type type, + struct pbn_lock **lock_ptr) +{ + /* + * Borrow and prepare a lock from the pool so we don't have to do two int_map accesses in + * the common case of no lock contention. + */ + struct pbn_lock *lock, *new_lock = NULL; + int result; + + result = borrow_pbn_lock_from_pool(zone->lock_pool, type, &new_lock); + if (result != VDO_SUCCESS) { + ASSERT_LOG_ONLY(false, "must always be able to borrow a PBN lock"); + return result; + } + + result = vdo_int_map_put(zone->pbn_operations, pbn, new_lock, false, + (void **) &lock); + if (result != VDO_SUCCESS) { + return_pbn_lock_to_pool(zone->lock_pool, new_lock); + return result; + } + + if (lock != NULL) { + /* The lock is already held, so we don't need the borrowed one. */ + return_pbn_lock_to_pool(zone->lock_pool, uds_forget(new_lock)); + result = ASSERT(lock->holder_count > 0, "physical block %llu lock held", + (unsigned long long) pbn); + if (result != VDO_SUCCESS) + return result; + *lock_ptr = lock; + } else { + *lock_ptr = new_lock; + } + return VDO_SUCCESS; +} + +/** + * allocate_and_lock_block() - Attempt to allocate a block from this zone. + * @allocation: The struct allocation of the data_vio attempting to allocate. + * + * If a block is allocated, the recipient will also hold a lock on it. + * + * Return: VDO_SUCCESS if a block was allocated, or an error code. + */ +static int allocate_and_lock_block(struct allocation *allocation) +{ + int result; + struct pbn_lock *lock; + + ASSERT_LOG_ONLY(allocation->lock == NULL, + "must not allocate a block while already holding a lock on one"); + + result = vdo_allocate_block(allocation->zone->allocator, &allocation->pbn); + if (result != VDO_SUCCESS) + return result; + + result = vdo_attempt_physical_zone_pbn_lock(allocation->zone, allocation->pbn, + allocation->write_lock_type, &lock); + if (result != VDO_SUCCESS) + return result; + + if (lock->holder_count > 0) { + /* This block is already locked, which should be impossible. */ + return uds_log_error_strerror(VDO_LOCK_ERROR, + "Newly allocated block %llu was spuriously locked (holder_count=%u)", + (unsigned long long) allocation->pbn, + lock->holder_count); + } + + /* We've successfully acquired a new lock, so mark it as ours. */ + lock->holder_count += 1; + allocation->lock = lock; + vdo_assign_pbn_lock_provisional_reference(lock); + return VDO_SUCCESS; +} + +/** + * retry_allocation() - Retry allocating a block now that we're done waiting for scrubbing. + * @waiter: The allocating_vio that was waiting to allocate. + * @context: The context (unused). + */ +static void retry_allocation(struct waiter *waiter, void *context __always_unused) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + + /* Now that some slab has scrubbed, restart the allocation process. */ + data_vio->allocation.wait_for_clean_slab = false; + data_vio->allocation.first_allocation_zone = data_vio->allocation.zone->zone_number; + continue_data_vio(data_vio); +} + +/** + * continue_allocating() - Continue searching for an allocation by enqueuing to wait for scrubbing + * or switching to the next zone. + * @data_vio: The data_vio attempting to get an allocation. + * + * This method should only be called from the error handler set in data_vio_allocate_data_block. + * + * Return: true if the allocation process has continued in another zone. + */ +static bool continue_allocating(struct data_vio *data_vio) +{ + struct allocation *allocation = &data_vio->allocation; + struct physical_zone *zone = allocation->zone; + struct vdo_completion *completion = &data_vio->vio.completion; + int result = VDO_SUCCESS; + bool was_waiting = allocation->wait_for_clean_slab; + bool tried_all = (allocation->first_allocation_zone == zone->next->zone_number); + + vdo_reset_completion(completion); + + if (tried_all && !was_waiting) { + /* + * We've already looked in all the zones, and found nothing. So go through the + * zones again, and wait for each to scrub before trying to allocate. + */ + allocation->wait_for_clean_slab = true; + allocation->first_allocation_zone = zone->zone_number; + } + + if (allocation->wait_for_clean_slab) { + data_vio->waiter.callback = retry_allocation; + result = vdo_enqueue_clean_slab_waiter(zone->allocator, + &data_vio->waiter); + if (result == VDO_SUCCESS) { + /* We've enqueued to wait for a slab to be scrubbed. */ + return true; + } + + if ((result != VDO_NO_SPACE) || (was_waiting && tried_all)) { + vdo_set_completion_result(completion, result); + return false; + } + } + + allocation->zone = zone->next; + completion->callback_thread_id = allocation->zone->thread_id; + vdo_launch_completion(completion); + return true; +} + +/** + * vdo_allocate_block_in_zone() - Attempt to allocate a block in the current physical zone, and if + * that fails try the next if possible. + * @data_vio: The data_vio needing an allocation. + * + * Return: true if a block was allocated, if not the data_vio will have been dispatched so the + * caller must not touch it. + */ +bool vdo_allocate_block_in_zone(struct data_vio *data_vio) +{ + int result = allocate_and_lock_block(&data_vio->allocation); + + if (result == VDO_SUCCESS) + return true; + + if ((result != VDO_NO_SPACE) || !continue_allocating(data_vio)) + continue_data_vio_with_error(data_vio, result); + + return false; +} + +/** + * vdo_release_physical_zone_pbn_lock() - Release a physical block lock if it is held and return it + * to the lock pool. + * @zone: The physical zone in which the lock was obtained. + * @locked_pbn: The physical block number to unlock. + * @lock: The lock being released. + * + * It must be the last live reference, as if the memory were being freed (the + * lock memory will re-initialized or zeroed). + */ +void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t locked_pbn, + struct pbn_lock *lock) +{ + struct pbn_lock *holder; + + if (lock == NULL) + return; + + ASSERT_LOG_ONLY(lock->holder_count > 0, + "should not be releasing a lock that is not held"); + + lock->holder_count -= 1; + if (lock->holder_count > 0) { + /* The lock was shared and is still referenced, so don't release it yet. */ + return; + } + + holder = vdo_int_map_remove(zone->pbn_operations, locked_pbn); + ASSERT_LOG_ONLY((lock == holder), "physical block lock mismatch for block %llu", + (unsigned long long) locked_pbn); + + release_pbn_lock_provisional_reference(lock, locked_pbn, zone->allocator); + return_pbn_lock_to_pool(zone->lock_pool, lock); +} + +/** + * vdo_dump_physical_zone() - Dump information about a physical zone to the log for debugging. + * @zone: The zone to dump. + */ +void vdo_dump_physical_zone(const struct physical_zone *zone) +{ + vdo_dump_block_allocator(zone->allocator); +} diff --git a/drivers/md/dm-vdo/physical-zone.h b/drivers/md/dm-vdo/physical-zone.h new file mode 100644 index 000000000000..47d874fd5a0b --- /dev/null +++ b/drivers/md/dm-vdo/physical-zone.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_PHYSICAL_ZONE_H +#define VDO_PHYSICAL_ZONE_H + +#include + +#include "types.h" + +/* + * The type of a PBN lock. + */ +enum pbn_lock_type { + VIO_READ_LOCK, + VIO_WRITE_LOCK, + VIO_BLOCK_MAP_WRITE_LOCK, +}; + +struct pbn_lock_implementation; + +/* + * A PBN lock. + */ +struct pbn_lock { + /* The implementation of the lock */ + const struct pbn_lock_implementation *implementation; + + /* The number of VIOs holding or sharing this lock */ + data_vio_count_t holder_count; + /* + * The number of compressed block writers holding a share of this lock while they are + * acquiring a reference to the PBN. + */ + u8 fragment_locks; + + /* Whether the locked PBN has been provisionally referenced on behalf of the lock holder. */ + bool has_provisional_reference; + + /* + * For read locks, the number of references that were known to be available on the locked + * block at the time the lock was acquired. + */ + u8 increment_limit; + + /* + * For read locks, the number of data_vios that have tried to claim one of the available + * increments during the lifetime of the lock. Each claim will first increment this + * counter, so it can exceed the increment limit. + */ + atomic_t increments_claimed; +}; + +struct physical_zone { + /* Which physical zone this is */ + zone_count_t zone_number; + /* The thread ID for this zone */ + thread_id_t thread_id; + /* In progress operations keyed by PBN */ + struct int_map *pbn_operations; + /* Pool of unused pbn_lock instances */ + struct pbn_lock_pool *lock_pool; + /* The block allocator for this zone */ + struct block_allocator *allocator; + /* The next zone from which to attempt an allocation */ + struct physical_zone *next; +}; + +struct physical_zones { + /* The number of zones */ + zone_count_t zone_count; + /* The physical zones themselves */ + struct physical_zone zones[]; +}; + +bool __must_check vdo_is_pbn_read_lock(const struct pbn_lock *lock); +void vdo_downgrade_pbn_write_lock(struct pbn_lock *lock, bool compressed_write); +bool __must_check vdo_claim_pbn_lock_increment(struct pbn_lock *lock); + +/** + * vdo_pbn_lock_has_provisional_reference() - Check whether a PBN lock has a provisional reference. + * @lock: The PBN lock. + */ +static inline bool vdo_pbn_lock_has_provisional_reference(struct pbn_lock *lock) +{ + return ((lock != NULL) && lock->has_provisional_reference); +} + +void vdo_assign_pbn_lock_provisional_reference(struct pbn_lock *lock); +void vdo_unassign_pbn_lock_provisional_reference(struct pbn_lock *lock); + +int __must_check vdo_make_physical_zones(struct vdo *vdo, + struct physical_zones **zones_ptr); + +void vdo_free_physical_zones(struct physical_zones *zones); + +struct pbn_lock * __must_check vdo_get_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn); + +int __must_check vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn, + enum pbn_lock_type type, + struct pbn_lock **lock_ptr); + +bool __must_check vdo_allocate_block_in_zone(struct data_vio *data_vio); + +void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t locked_pbn, + struct pbn_lock *lock); + +void vdo_dump_physical_zone(const struct physical_zone *zone); + +#endif /* VDO_PHYSICAL_ZONE_H */ diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 89af2e8a2621..da602cbb278e 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -1926,6 +1926,38 @@ static bool advance_search_cursor(struct vdo_slab *slab) return true; } +/** + * vdo_adjust_reference_count_for_rebuild() - Adjust the reference count of a block during rebuild. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot, + physical_block_number_t pbn, + enum journal_operation operation) +{ + int result; + slab_block_number block_number; + struct reference_block *block; + struct vdo_slab *slab = vdo_get_slab(depot, pbn); + struct reference_updater updater = { + .operation = operation, + .increment = true, + }; + + result = slab_block_number_from_pbn(slab, pbn, &block_number); + if (result != VDO_SUCCESS) + return result; + + block = get_reference_block(slab, block_number); + result = update_reference_count(slab, block, block_number, NULL, + &updater, !NORMAL_OPERATION, false, NULL); + if (result != VDO_SUCCESS) + return result; + + dirty_block(block); + return VDO_SUCCESS; +} + /** * replay_reference_count_change() - Replay the reference count adjustment from a slab journal * entry into the reference count for a block. @@ -2476,6 +2508,1106 @@ static void load_slab_journal(struct vdo_slab *slab) acquire_vio_from_pool(slab->allocator->vio_pool, &journal->resource_waiter); } +static void register_slab_for_scrubbing(struct vdo_slab *slab, bool high_priority) +{ + struct slab_scrubber *scrubber = &slab->allocator->scrubber; + + ASSERT_LOG_ONLY((slab->status != VDO_SLAB_REBUILT), + "slab to be scrubbed is unrecovered"); + + if (slab->status != VDO_SLAB_REQUIRES_SCRUBBING) + return; + + list_del_init(&slab->allocq_entry); + if (!slab->was_queued_for_scrubbing) { + WRITE_ONCE(scrubber->slab_count, scrubber->slab_count + 1); + slab->was_queued_for_scrubbing = true; + } + + if (high_priority) { + slab->status = VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING; + list_add_tail(&slab->allocq_entry, &scrubber->high_priority_slabs); + return; + } + + list_add_tail(&slab->allocq_entry, &scrubber->slabs); +} + +/* Queue a slab for allocation or scrubbing. */ +static void queue_slab(struct vdo_slab *slab) +{ + struct block_allocator *allocator = slab->allocator; + block_count_t free_blocks; + int result; + + ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), + "a requeued slab must not already be on a ring"); + + if (vdo_is_read_only(allocator->depot->vdo)) + return; + + free_blocks = slab->free_blocks; + result = ASSERT((free_blocks <= allocator->depot->slab_config.data_blocks), + "rebuilt slab %u must have a valid free block count (has %llu, expected maximum %llu)", + slab->slab_number, (unsigned long long) free_blocks, + (unsigned long long) allocator->depot->slab_config.data_blocks); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(allocator->depot->vdo, result); + return; + } + + if (slab->status != VDO_SLAB_REBUILT) { + register_slab_for_scrubbing(slab, false); + return; + } + + if (!vdo_is_state_resuming(&slab->state)) { + /* + * If the slab is resuming, we've already accounted for it here, so don't do it + * again. + * FIXME: under what situation would the slab be resuming here? + */ + WRITE_ONCE(allocator->allocated_blocks, + allocator->allocated_blocks - free_blocks); + if (!is_slab_journal_blank(slab)) { + WRITE_ONCE(allocator->statistics.slabs_opened, + allocator->statistics.slabs_opened + 1); + } + } + + if (allocator->depot->vdo->suspend_type == VDO_ADMIN_STATE_SAVING) + reopen_slab_journal(slab); + + prioritize_slab(slab); +} + +/** + * initiate_slab_action() - Initiate a slab action. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_slab_action(struct admin_state *state) +{ + struct vdo_slab *slab = container_of(state, struct vdo_slab, state); + + if (vdo_is_state_draining(state)) { + const struct admin_state_code *operation = vdo_get_admin_state_code(state); + + if (operation == VDO_ADMIN_STATE_SCRUBBING) + slab->status = VDO_SLAB_REBUILDING; + + drain_slab(slab); + check_if_slab_drained(slab); + return; + } + + if (vdo_is_state_loading(state)) { + load_slab_journal(slab); + return; + } + + if (vdo_is_state_resuming(state)) { + queue_slab(slab); + vdo_finish_resuming(state); + return; + } + + vdo_finish_operation(state, VDO_INVALID_ADMIN_STATE); +} + +/** + * get_next_slab() - Get the next slab to scrub. + * @scrubber: The slab scrubber. + * + * Return: The next slab to scrub or NULL if there are none. + */ +static struct vdo_slab *get_next_slab(struct slab_scrubber *scrubber) +{ + struct vdo_slab *slab; + + slab = list_first_entry_or_null(&scrubber->high_priority_slabs, + struct vdo_slab, allocq_entry); + if (slab != NULL) + return slab; + + return list_first_entry_or_null(&scrubber->slabs, struct vdo_slab, + allocq_entry); +} + +/** + * has_slabs_to_scrub() - Check whether a scrubber has slabs to scrub. + * @scrubber: The scrubber to check. + * + * Return: true if the scrubber has slabs to scrub. + */ +static bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubber) +{ + return (get_next_slab(scrubber) != NULL); +} + +/** + * uninitialize_scrubber_vio() - Clean up the slab_scrubber's vio. + * @scrubber: The scrubber. + */ +static void uninitialize_scrubber_vio(struct slab_scrubber *scrubber) +{ + uds_free(uds_forget(scrubber->vio.data)); + free_vio_components(&scrubber->vio); +} + +/** + * finish_scrubbing() - Stop scrubbing, either because there are no more slabs to scrub or because + * there's been an error. + * @scrubber: The scrubber. + */ +static void finish_scrubbing(struct slab_scrubber *scrubber, int result) +{ + bool notify = vdo_has_waiters(&scrubber->waiters); + bool done = !has_slabs_to_scrub(scrubber); + struct block_allocator *allocator = + container_of(scrubber, struct block_allocator, scrubber); + + if (done) + uninitialize_scrubber_vio(scrubber); + + if (scrubber->high_priority_only) { + scrubber->high_priority_only = false; + vdo_fail_completion(uds_forget(scrubber->vio.completion.parent), result); + } else if (done && (atomic_add_return(-1, &allocator->depot->zones_to_scrub) == 0)) { + /* All of our slabs were scrubbed, and we're the last allocator to finish. */ + enum vdo_state prior_state = + atomic_cmpxchg(&allocator->depot->vdo->state, VDO_RECOVERING, + VDO_DIRTY); + + /* + * To be safe, even if the CAS failed, ensure anything that follows is ordered with + * respect to whatever state change did happen. + */ + smp_mb__after_atomic(); + + /* + * We must check the VDO state here and not the depot's read_only_notifier since + * the compare-swap-above could have failed due to a read-only entry which our own + * thread does not yet know about. + */ + if (prior_state == VDO_DIRTY) + uds_log_info("VDO commencing normal operation"); + else if (prior_state == VDO_RECOVERING) + uds_log_info("Exiting recovery mode"); + } + + /* + * Note that the scrubber has stopped, and inform anyone who might be waiting for that to + * happen. + */ + if (!vdo_finish_draining(&scrubber->admin_state)) + WRITE_ONCE(scrubber->admin_state.current_state, + VDO_ADMIN_STATE_SUSPENDED); + + /* + * We can't notify waiters until after we've finished draining or they'll just requeue. + * Fortunately if there were waiters, we can't have been freed yet. + */ + if (notify) + vdo_notify_all_waiters(&scrubber->waiters, NULL, NULL); +} + +static void scrub_next_slab(struct slab_scrubber *scrubber); + +/** + * slab_scrubbed() - Notify the scrubber that a slab has been scrubbed. + * @completion: The slab rebuild completion. + * + * This callback is registered in apply_journal_entries(). + */ +static void slab_scrubbed(struct vdo_completion *completion) +{ + struct slab_scrubber *scrubber = + container_of(as_vio(completion), struct slab_scrubber, vio); + struct vdo_slab *slab = scrubber->slab; + + slab->status = VDO_SLAB_REBUILT; + queue_slab(slab); + reopen_slab_journal(slab); + WRITE_ONCE(scrubber->slab_count, scrubber->slab_count - 1); + scrub_next_slab(scrubber); +} + +/** + * abort_scrubbing() - Abort scrubbing due to an error. + * @scrubber: The slab scrubber. + * @result: The error. + */ +static void abort_scrubbing(struct slab_scrubber *scrubber, int result) +{ + vdo_enter_read_only_mode(scrubber->vio.completion.vdo, result); + finish_scrubbing(scrubber, result); +} + +/** + * handle_scrubber_error() - Handle errors while rebuilding a slab. + * @completion: The slab rebuild completion. + */ +static void handle_scrubber_error(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + + vio_record_metadata_io_error(vio); + abort_scrubbing(container_of(vio, struct slab_scrubber, vio), + completion->result); +} + +/** + * apply_block_entries() - Apply all the entries in a block to the reference counts. + * @block: A block with entries to apply. + * @entry_count: The number of entries to apply. + * @block_number: The sequence number of the block. + * @slab: The slab to apply the entries to. + * + * Return: VDO_SUCCESS or an error code. + */ +static int apply_block_entries(struct packed_slab_journal_block *block, + journal_entry_count_t entry_count, + sequence_number_t block_number, struct vdo_slab *slab) +{ + struct journal_point entry_point = { + .sequence_number = block_number, + .entry_count = 0, + }; + int result; + slab_block_number max_sbn = slab->end - slab->start; + + while (entry_point.entry_count < entry_count) { + struct slab_journal_entry entry = + vdo_decode_slab_journal_entry(block, entry_point.entry_count); + + if (entry.sbn > max_sbn) { + /* This entry is out of bounds. */ + return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + "vdo_slab journal entry (%llu, %u) had invalid offset %u in slab (size %u blocks)", + (unsigned long long) block_number, + entry_point.entry_count, + entry.sbn, max_sbn); + } + + result = replay_reference_count_change(slab, &entry_point, entry); + if (result != VDO_SUCCESS) { + uds_log_error_strerror(result, + "vdo_slab journal entry (%llu, %u) (%s of offset %u) could not be applied in slab %u", + (unsigned long long) block_number, + entry_point.entry_count, + vdo_get_journal_operation_name(entry.operation), + entry.sbn, slab->slab_number); + return result; + } + entry_point.entry_count++; + } + + return VDO_SUCCESS; +} + +/** + * apply_journal_entries() - Find the relevant vio of the slab journal and apply all valid entries. + * @completion: The metadata read vio completion. + * + * This is a callback registered in start_scrubbing(). + */ +static void apply_journal_entries(struct vdo_completion *completion) +{ + int result; + struct slab_scrubber *scrubber + = container_of(as_vio(completion), struct slab_scrubber, vio); + struct vdo_slab *slab = scrubber->slab; + struct slab_journal *journal = &slab->journal; + + /* Find the boundaries of the useful part of the journal. */ + sequence_number_t tail = journal->tail; + tail_block_offset_t end_index = (tail - 1) % journal->size; + char *end_data = scrubber->vio.data + (end_index * VDO_BLOCK_SIZE); + struct packed_slab_journal_block *end_block = + (struct packed_slab_journal_block *) end_data; + + sequence_number_t head = __le64_to_cpu(end_block->header.head); + tail_block_offset_t head_index = head % journal->size; + block_count_t index = head_index; + + struct journal_point ref_counts_point = slab->slab_journal_point; + struct journal_point last_entry_applied = ref_counts_point; + sequence_number_t sequence; + + for (sequence = head; sequence < tail; sequence++) { + char *block_data = scrubber->vio.data + (index * VDO_BLOCK_SIZE); + struct packed_slab_journal_block *block = + (struct packed_slab_journal_block *) block_data; + struct slab_journal_block_header header; + + vdo_unpack_slab_journal_block_header(&block->header, &header); + + if ((header.nonce != slab->allocator->nonce) || + (header.metadata_type != VDO_METADATA_SLAB_JOURNAL) || + (header.sequence_number != sequence) || + (header.entry_count > journal->entries_per_block) || + (header.has_block_map_increments && + (header.entry_count > journal->full_entries_per_block))) { + /* The block is not what we expect it to be. */ + uds_log_error("vdo_slab journal block for slab %u was invalid", + slab->slab_number); + abort_scrubbing(scrubber, VDO_CORRUPT_JOURNAL); + return; + } + + result = apply_block_entries(block, header.entry_count, sequence, slab); + if (result != VDO_SUCCESS) { + abort_scrubbing(scrubber, result); + return; + } + + last_entry_applied.sequence_number = sequence; + last_entry_applied.entry_count = header.entry_count - 1; + index++; + if (index == journal->size) + index = 0; + } + + /* + * At the end of rebuild, the reference counters should be accurate to the end of the + * journal we just applied. + */ + result = ASSERT(!vdo_before_journal_point(&last_entry_applied, + &ref_counts_point), + "Refcounts are not more accurate than the slab journal"); + if (result != VDO_SUCCESS) { + abort_scrubbing(scrubber, result); + return; + } + + /* Save out the rebuilt reference blocks. */ + vdo_prepare_completion(completion, slab_scrubbed, handle_scrubber_error, + slab->allocator->thread_id, completion->parent); + vdo_start_operation_with_waiter(&slab->state, + VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING, + completion, initiate_slab_action); +} + +static void read_slab_journal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_scrubber *scrubber = container_of(vio, struct slab_scrubber, vio); + + continue_vio_after_io(bio->bi_private, apply_journal_entries, + scrubber->slab->allocator->thread_id); +} + +/** + * start_scrubbing() - Read the current slab's journal from disk now that it has been flushed. + * @completion: The scrubber's vio completion. + * + * This callback is registered in scrub_next_slab(). + */ +static void start_scrubbing(struct vdo_completion *completion) +{ + struct slab_scrubber *scrubber = + container_of(as_vio(completion), struct slab_scrubber, vio); + struct vdo_slab *slab = scrubber->slab; + + if (!slab->allocator->summary_entries[slab->slab_number].is_dirty) { + slab_scrubbed(completion); + return; + } + + submit_metadata_vio(&scrubber->vio, slab->journal_origin, + read_slab_journal_endio, handle_scrubber_error, + REQ_OP_READ); +} + +/** + * scrub_next_slab() - Scrub the next slab if there is one. + * @scrubber: The scrubber. + */ +static void scrub_next_slab(struct slab_scrubber *scrubber) +{ + struct vdo_completion *completion = &scrubber->vio.completion; + struct vdo_slab *slab; + + /* + * Note: this notify call is always safe only because scrubbing can only be started when + * the VDO is quiescent. + */ + vdo_notify_all_waiters(&scrubber->waiters, NULL, NULL); + + if (vdo_is_read_only(completion->vdo)) { + finish_scrubbing(scrubber, VDO_READ_ONLY); + return; + } + + slab = get_next_slab(scrubber); + if ((slab == NULL) || + (scrubber->high_priority_only && list_empty(&scrubber->high_priority_slabs))) { + finish_scrubbing(scrubber, VDO_SUCCESS); + return; + } + + if (vdo_finish_draining(&scrubber->admin_state)) + return; + + list_del_init(&slab->allocq_entry); + scrubber->slab = slab; + vdo_prepare_completion(completion, start_scrubbing, handle_scrubber_error, + slab->allocator->thread_id, completion->parent); + vdo_start_operation_with_waiter(&slab->state, VDO_ADMIN_STATE_SCRUBBING, + completion, initiate_slab_action); +} + +/** + * scrub_slabs() - Scrub all of an allocator's slabs that are eligible for scrubbing. + * @allocator: The block_allocator to scrub. + * @parent: The completion to notify when scrubbing is done, implies high_priority, may be NULL. + */ +static void scrub_slabs(struct block_allocator *allocator, struct vdo_completion *parent) +{ + struct slab_scrubber *scrubber = &allocator->scrubber; + + scrubber->vio.completion.parent = parent; + scrubber->high_priority_only = (parent != NULL); + if (!has_slabs_to_scrub(scrubber)) { + finish_scrubbing(scrubber, VDO_SUCCESS); + return; + } + + if (scrubber->high_priority_only && + vdo_is_priority_table_empty(allocator->prioritized_slabs) && + list_empty(&scrubber->high_priority_slabs)) + register_slab_for_scrubbing(get_next_slab(scrubber), true); + + vdo_resume_if_quiescent(&scrubber->admin_state); + scrub_next_slab(scrubber); +} + +static inline void assert_on_allocator_thread(thread_id_t thread_id, + const char *function_name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == thread_id), + "%s called on correct thread", function_name); +} + +static void register_slab_with_allocator(struct block_allocator *allocator, + struct vdo_slab *slab) +{ + allocator->slab_count++; + allocator->last_slab = slab->slab_number; +} + +static struct slab_iterator get_slab_iterator(const struct block_allocator *allocator) +{ + return get_depot_slab_iterator(allocator->depot, allocator->last_slab, + allocator->zone_number, + allocator->depot->zone_count); +} + +/** + * next_slab() - Get the next slab from a slab_iterator and advance the iterator + * @iterator: The slab_iterator. + * + * Return: The next slab or NULL if the iterator is exhausted. + */ +static struct vdo_slab *next_slab(struct slab_iterator *iterator) +{ + struct vdo_slab *slab = iterator->next; + + if ((slab == NULL) || (slab->slab_number < iterator->end + iterator->stride)) + iterator->next = NULL; + else + iterator->next = iterator->slabs[slab->slab_number - iterator->stride]; + + return slab; +} + +/** + * abort_waiter() - Abort vios waiting to make journal entries when read-only. + * + * This callback is invoked on all vios waiting to make slab journal entries after the VDO has gone + * into read-only mode. Implements waiter_callback_fn. + */ +static void abort_waiter(struct waiter *waiter, void *context __always_unused) +{ + struct reference_updater *updater = + container_of(waiter, struct reference_updater, waiter); + struct data_vio *data_vio = data_vio_from_reference_updater(updater); + + if (updater->increment) { + continue_data_vio_with_error(data_vio, VDO_READ_ONLY); + return; + } + + vdo_continue_completion(&data_vio->decrement_completion, VDO_READ_ONLY); +} + +/* Implements vdo_read_only_notification_fn. */ +static void notify_block_allocator_of_read_only_mode(void *listener, + struct vdo_completion *parent) +{ + struct block_allocator *allocator = listener; + struct slab_iterator iterator; + + assert_on_allocator_thread(allocator->thread_id, __func__); + iterator = get_slab_iterator(allocator); + while (iterator.next != NULL) { + struct vdo_slab *slab = next_slab(&iterator); + + vdo_notify_all_waiters(&slab->journal.entry_waiters, + abort_waiter, &slab->journal); + check_if_slab_drained(slab); + } + + vdo_finish_completion(parent); +} + +/** + * vdo_acquire_provisional_reference() - Acquire a provisional reference on behalf of a PBN lock if + * the block it locks is unreferenced. + * @slab: The slab which contains the block. + * @pbn: The physical block to reference. + * @lock: The lock. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_acquire_provisional_reference(struct vdo_slab *slab, physical_block_number_t pbn, + struct pbn_lock *lock) +{ + slab_block_number block_number; + int result; + + if (vdo_pbn_lock_has_provisional_reference(lock)) + return VDO_SUCCESS; + + if (!is_slab_open(slab)) + return VDO_INVALID_ADMIN_STATE; + + result = slab_block_number_from_pbn(slab, pbn, &block_number); + if (result != VDO_SUCCESS) + return result; + + if (slab->counters[block_number] == EMPTY_REFERENCE_COUNT) { + make_provisional_reference(slab, block_number); + if (lock != NULL) + vdo_assign_pbn_lock_provisional_reference(lock); + } + + if (vdo_pbn_lock_has_provisional_reference(lock)) + adjust_free_block_count(slab, false); + + return VDO_SUCCESS; +} + +static int __must_check allocate_slab_block(struct vdo_slab *slab, + physical_block_number_t *block_number_ptr) +{ + slab_block_number free_index; + + if (!is_slab_open(slab)) + return VDO_INVALID_ADMIN_STATE; + + if (!search_reference_blocks(slab, &free_index)) + return VDO_NO_SPACE; + + ASSERT_LOG_ONLY((slab->counters[free_index] == EMPTY_REFERENCE_COUNT), + "free block must have ref count of zero"); + make_provisional_reference(slab, free_index); + adjust_free_block_count(slab, false); + + /* + * Update the search hint so the next search will start at the array index just past the + * free block we just found. + */ + slab->search_cursor.index = (free_index + 1); + + *block_number_ptr = slab->start + free_index; + return VDO_SUCCESS; +} + +/** + * open_slab() - Prepare a slab to be allocated from. + * @slab: The slab. + */ +static void open_slab(struct vdo_slab *slab) +{ + reset_search_cursor(slab); + if (is_slab_journal_blank(slab)) { + WRITE_ONCE(slab->allocator->statistics.slabs_opened, + slab->allocator->statistics.slabs_opened + 1); + dirty_all_reference_blocks(slab); + } else { + WRITE_ONCE(slab->allocator->statistics.slabs_reopened, + slab->allocator->statistics.slabs_reopened + 1); + } + + slab->allocator->open_slab = slab; +} + + +/* + * The block allocated will have a provisional reference and the reference must be either confirmed + * with a subsequent increment or vacated with a subsequent decrement via + * vdo_release_block_reference(). + */ +int vdo_allocate_block(struct block_allocator *allocator, + physical_block_number_t *block_number_ptr) +{ + int result; + + if (allocator->open_slab != NULL) { + /* Try to allocate the next block in the currently open slab. */ + result = allocate_slab_block(allocator->open_slab, block_number_ptr); + if ((result == VDO_SUCCESS) || (result != VDO_NO_SPACE)) + return result; + + /* Put the exhausted open slab back into the priority table. */ + prioritize_slab(allocator->open_slab); + } + + /* Remove the highest priority slab from the priority table and make it the open slab. */ + open_slab(list_entry(vdo_priority_table_dequeue(allocator->prioritized_slabs), + struct vdo_slab, allocq_entry)); + + /* + * Try allocating again. If we're out of space immediately after opening a slab, then every + * slab must be fully allocated. + */ + return allocate_slab_block(allocator->open_slab, block_number_ptr); +} + +/** + * vdo_enqueue_clean_slab_waiter() - Wait for a clean slab. + * @allocator: The block_allocator on which to wait. + * @waiter: The waiter. + * + * Return: VDO_SUCCESS if the waiter was queued, VDO_NO_SPACE if there are no slabs to scrub, and + * some other error otherwise. + */ +int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, + struct waiter *waiter) +{ + if (vdo_is_read_only(allocator->depot->vdo)) + return VDO_READ_ONLY; + + if (vdo_is_state_quiescent(&allocator->scrubber.admin_state)) + return VDO_NO_SPACE; + + vdo_enqueue_waiter(&allocator->scrubber.waiters, waiter); + return VDO_SUCCESS; +} + +/** + * vdo_modify_reference_count() - Modify the reference count of a block by first making a slab + * journal entry and then updating the reference counter. + * + * @data_vio: The data_vio for which to add the entry. + * @updater: Which of the data_vio's reference updaters is being submitted. + */ +void vdo_modify_reference_count(struct vdo_completion *completion, + struct reference_updater *updater) +{ + struct vdo_slab *slab = vdo_get_slab(completion->vdo->depot, updater->zpbn.pbn); + + if (!is_slab_open(slab)) { + vdo_continue_completion(completion, VDO_INVALID_ADMIN_STATE); + return; + } + + if (vdo_is_read_only(completion->vdo)) { + vdo_continue_completion(completion, VDO_READ_ONLY); + return; + } + + vdo_enqueue_waiter(&slab->journal.entry_waiters, &updater->waiter); + if ((slab->status != VDO_SLAB_REBUILT) && requires_reaping(&slab->journal)) + register_slab_for_scrubbing(slab, true); + + add_entries(&slab->journal); +} + +/* Release an unused provisional reference. */ +int vdo_release_block_reference(struct block_allocator *allocator, + physical_block_number_t pbn) +{ + struct reference_updater updater; + + if (pbn == VDO_ZERO_BLOCK) + return VDO_SUCCESS; + + updater = (struct reference_updater) { + .operation = VDO_JOURNAL_DATA_REMAPPING, + .increment = false, + .zpbn = { + .pbn = pbn, + }, + }; + + return adjust_reference_count(vdo_get_slab(allocator->depot, pbn), + &updater, NULL); +} + +/* + * This is a min_heap callback function orders slab_status structures using the 'is_clean' field as + * the primary key and the 'emptiness' field as the secondary key. + * + * Slabs need to be pushed onto the rings in the same order they are to be popped off. Popping + * should always get the most empty first, so pushing should be from most empty to least empty. + * Thus, the ordering is reversed from the usual sense since min_heap returns smaller elements + * before larger ones. + */ +static bool slab_status_is_less_than(const void *item1, const void *item2) +{ + const struct slab_status *info1 = item1; + const struct slab_status *info2 = item2; + + if (info1->is_clean != info2->is_clean) + return info1->is_clean; + if (info1->emptiness != info2->emptiness) + return info1->emptiness > info2->emptiness; + return info1->slab_number < info2->slab_number; +} + +static void swap_slab_statuses(void *item1, void *item2) +{ + struct slab_status *info1 = item1; + struct slab_status *info2 = item2; + + swap(*info1, *info2); +} + +static const struct min_heap_callbacks slab_status_min_heap = { + .elem_size = sizeof(struct slab_status), + .less = slab_status_is_less_than, + .swp = swap_slab_statuses, +}; + +/* Inform the slab actor that a action has finished on some slab; used by apply_to_slabs(). */ +static void slab_action_callback(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + struct slab_actor *actor = &allocator->slab_actor; + + if (--actor->slab_action_count == 0) { + actor->callback(completion); + return; + } + + vdo_reset_completion(completion); +} + +/* Preserve the error from part of an action and continue. */ +static void handle_operation_error(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + if (allocator->state.waiter != NULL) + vdo_set_completion_result(allocator->state.waiter, completion->result); + completion->callback(completion); +} + +/* Perform an action on each of an allocator's slabs in parallel. */ +static void apply_to_slabs(struct block_allocator *allocator, vdo_action_fn callback) +{ + struct slab_iterator iterator; + + vdo_prepare_completion(&allocator->completion, slab_action_callback, + handle_operation_error, allocator->thread_id, NULL); + allocator->completion.requeue = false; + + /* + * Since we are going to dequeue all of the slabs, the open slab will become invalid, so + * clear it. + */ + allocator->open_slab = NULL; + + /* Ensure that we don't finish before we're done starting. */ + allocator->slab_actor = (struct slab_actor) { + .slab_action_count = 1, + .callback = callback, + }; + + iterator = get_slab_iterator(allocator); + while (iterator.next != NULL) { + const struct admin_state_code *operation = + vdo_get_admin_state_code(&allocator->state); + struct vdo_slab *slab = next_slab(&iterator); + + list_del_init(&slab->allocq_entry); + allocator->slab_actor.slab_action_count++; + vdo_start_operation_with_waiter(&slab->state, operation, + &allocator->completion, + initiate_slab_action); + } + + slab_action_callback(&allocator->completion); +} + +static void finish_loading_allocator(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + const struct admin_state_code *operation = + vdo_get_admin_state_code(&allocator->state); + + if (allocator->eraser != NULL) + dm_kcopyd_client_destroy(uds_forget(allocator->eraser)); + + if (operation == VDO_ADMIN_STATE_LOADING_FOR_RECOVERY) { + void *context = + vdo_get_current_action_context(allocator->depot->action_manager); + + vdo_replay_into_slab_journals(allocator, context); + return; + } + + vdo_finish_loading(&allocator->state); +} + +static void erase_next_slab_journal(struct block_allocator *allocator); + +static void copy_callback(int read_err, unsigned long write_err, void *context) +{ + struct block_allocator *allocator = context; + int result = (((read_err == 0) && (write_err == 0)) ? VDO_SUCCESS : -EIO); + + if (result != VDO_SUCCESS) { + vdo_fail_completion(&allocator->completion, result); + return; + } + + erase_next_slab_journal(allocator); +} + +/* erase_next_slab_journal() - Erase the next slab journal. */ +static void erase_next_slab_journal(struct block_allocator *allocator) +{ + struct vdo_slab *slab; + physical_block_number_t pbn; + struct dm_io_region regions[1]; + struct slab_depot *depot = allocator->depot; + block_count_t blocks = depot->slab_config.slab_journal_blocks; + + if (allocator->slabs_to_erase.next == NULL) { + vdo_finish_completion(&allocator->completion); + return; + } + + slab = next_slab(&allocator->slabs_to_erase); + pbn = slab->journal_origin - depot->vdo->geometry.bio_offset; + regions[0] = (struct dm_io_region) { + .bdev = vdo_get_backing_device(depot->vdo), + .sector = pbn * VDO_SECTORS_PER_BLOCK, + .count = blocks * VDO_SECTORS_PER_BLOCK, + }; + dm_kcopyd_zero(allocator->eraser, 1, regions, 0, copy_callback, allocator); +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_load(struct admin_state *state) +{ + struct block_allocator *allocator = + container_of(state, struct block_allocator, state); + const struct admin_state_code *operation = vdo_get_admin_state_code(state); + + if (operation == VDO_ADMIN_STATE_LOADING_FOR_REBUILD) { + /* + * Must requeue because the kcopyd client cannot be freed in the same stack frame + * as the kcopyd callback, lest it deadlock. + */ + vdo_prepare_completion_for_requeue(&allocator->completion, + finish_loading_allocator, + handle_operation_error, + allocator->thread_id, NULL); + allocator->eraser = dm_kcopyd_client_create(NULL); + if (allocator->eraser == NULL) { + vdo_fail_completion(&allocator->completion, -ENOMEM); + return; + } + allocator->slabs_to_erase = get_slab_iterator(allocator); + + erase_next_slab_journal(allocator); + return; + } + + apply_to_slabs(allocator, finish_loading_allocator); +} + +/** + * vdo_notify_slab_journals_are_recovered() - Inform a block allocator that its slab journals have + * been recovered from the recovery journal. + * @completion The allocator completion + */ +void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + vdo_finish_loading_with_result(&allocator->state, completion->result); +} + +static int get_slab_statuses(struct block_allocator *allocator, + struct slab_status **statuses_ptr) +{ + int result; + struct slab_status *statuses; + struct slab_iterator iterator = get_slab_iterator(allocator); + + result = uds_allocate(allocator->slab_count, struct slab_status, __func__, + &statuses); + if (result != VDO_SUCCESS) + return result; + + *statuses_ptr = statuses; + + while (iterator.next != NULL) { + slab_count_t slab_number = next_slab(&iterator)->slab_number; + + *statuses++ = (struct slab_status) { + .slab_number = slab_number, + .is_clean = !allocator->summary_entries[slab_number].is_dirty, + .emptiness = allocator->summary_entries[slab_number].fullness_hint, + }; + } + + return VDO_SUCCESS; +} + +/* Prepare slabs for allocation or scrubbing. */ +static int __must_check vdo_prepare_slabs_for_allocation(struct block_allocator *allocator) +{ + struct slab_status current_slab_status; + struct min_heap heap; + int result; + struct slab_status *slab_statuses; + struct slab_depot *depot = allocator->depot; + + WRITE_ONCE(allocator->allocated_blocks, + allocator->slab_count * depot->slab_config.data_blocks); + result = get_slab_statuses(allocator, &slab_statuses); + if (result != VDO_SUCCESS) + return result; + + /* Sort the slabs by cleanliness, then by emptiness hint. */ + heap = (struct min_heap) { + .data = slab_statuses, + .nr = allocator->slab_count, + .size = allocator->slab_count, + }; + min_heapify_all(&heap, &slab_status_min_heap); + + while (heap.nr > 0) { + bool high_priority; + struct vdo_slab *slab; + struct slab_journal *journal; + + current_slab_status = slab_statuses[0]; + min_heap_pop(&heap, &slab_status_min_heap); + slab = depot->slabs[current_slab_status.slab_number]; + + if ((depot->load_type == VDO_SLAB_DEPOT_REBUILD_LOAD) || + (!allocator->summary_entries[slab->slab_number].load_ref_counts && + current_slab_status.is_clean)) { + queue_slab(slab); + continue; + } + + slab->status = VDO_SLAB_REQUIRES_SCRUBBING; + journal = &slab->journal; + high_priority = ((current_slab_status.is_clean && + (depot->load_type == VDO_SLAB_DEPOT_NORMAL_LOAD)) || + (journal_length(journal) >= journal->scrubbing_threshold)); + register_slab_for_scrubbing(slab, high_priority); + } + + uds_free(slab_statuses); + return VDO_SUCCESS; +} + +static const char *status_to_string(enum slab_rebuild_status status) +{ + switch (status) { + case VDO_SLAB_REBUILT: + return "REBUILT"; + case VDO_SLAB_REQUIRES_SCRUBBING: + return "SCRUBBING"; + case VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING: + return "PRIORITY_SCRUBBING"; + case VDO_SLAB_REBUILDING: + return "REBUILDING"; + case VDO_SLAB_REPLAYING: + return "REPLAYING"; + default: + return "UNKNOWN"; + } +} + +void vdo_dump_block_allocator(const struct block_allocator *allocator) +{ + unsigned int pause_counter = 0; + struct slab_iterator iterator = get_slab_iterator(allocator); + const struct slab_scrubber *scrubber = &allocator->scrubber; + + uds_log_info("block_allocator zone %u", allocator->zone_number); + while (iterator.next != NULL) { + struct vdo_slab *slab = next_slab(&iterator); + struct slab_journal *journal = &slab->journal; + + if (slab->reference_blocks != NULL) { + /* Terse because there are a lot of slabs to dump and syslog is lossy. */ + uds_log_info("slab %u: P%u, %llu free", slab->slab_number, + slab->priority, + (unsigned long long) slab->free_blocks); + } else { + uds_log_info("slab %u: status %s", slab->slab_number, + status_to_string(slab->status)); + } + + uds_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", + vdo_count_waiters(&journal->entry_waiters), + uds_bool_to_string(journal->waiting_to_commit), + uds_bool_to_string(journal->updating_slab_summary), + (unsigned long long) journal->head, + (unsigned long long) journal->unreapable, + (unsigned long long) journal->tail, + (unsigned long long) journal->next_commit, + (unsigned long long) journal->summarized, + (unsigned long long) journal->last_summarized, + (unsigned long long) journal->recovery_lock, + uds_bool_to_string(journal->recovery_lock != 0)); + /* + * Given the frequency with which the locks are just a tiny bit off, it might be + * worth dumping all the locks, but that might be too much logging. + */ + + if (slab->counters != NULL) { + /* Terse because there are a lot of slabs to dump and syslog is lossy. */ + uds_log_info(" slab: free=%u/%u blocks=%u dirty=%zu active=%zu journal@(%llu,%u)", + slab->free_blocks, slab->block_count, + slab->reference_block_count, + vdo_count_waiters(&slab->dirty_blocks), + slab->active_count, + (unsigned long long) slab->slab_journal_point.sequence_number, + slab->slab_journal_point.entry_count); + } else { + uds_log_info(" no counters"); + } + + /* + * Wait for a while after each batch of 32 slabs dumped, an arbitrary number, + * allowing the kernel log a chance to be flushed instead of being overrun. + */ + if (pause_counter++ == 31) { + pause_counter = 0; + uds_pause_for_logger(); + } + } + + uds_log_info("slab_scrubber slab_count %u waiters %zu %s%s", + READ_ONCE(scrubber->slab_count), + vdo_count_waiters(&scrubber->waiters), + vdo_get_admin_state_code(&scrubber->admin_state)->name, + scrubber->high_priority_only ? ", high_priority_only " : ""); +} + static void free_slab(struct vdo_slab *slab) { if (slab == NULL) @@ -2596,6 +3728,161 @@ static int __must_check make_slab(physical_block_number_t slab_origin, return VDO_SUCCESS; } +/** + * initialize_slab_scrubber() - Initialize an allocator's slab scrubber. + * @allocator: The allocator being initialized + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_slab_scrubber(struct block_allocator *allocator) +{ + struct slab_scrubber *scrubber = &allocator->scrubber; + block_count_t slab_journal_size = + allocator->depot->slab_config.slab_journal_blocks; + char *journal_data; + int result; + + result = uds_allocate(VDO_BLOCK_SIZE * slab_journal_size, + char, __func__, &journal_data); + if (result != VDO_SUCCESS) + return result; + + result = allocate_vio_components(allocator->completion.vdo, + VIO_TYPE_SLAB_JOURNAL, + VIO_PRIORITY_METADATA, + allocator, slab_journal_size, + journal_data, &scrubber->vio); + if (result != VDO_SUCCESS) { + uds_free(journal_data); + return result; + } + + INIT_LIST_HEAD(&scrubber->high_priority_slabs); + INIT_LIST_HEAD(&scrubber->slabs); + vdo_set_admin_state_code(&scrubber->admin_state, VDO_ADMIN_STATE_SUSPENDED); + return VDO_SUCCESS; +} + +/** + * initialize_slab_summary_block() - Initialize a slab_summary_block. + * @allocator: The allocator which owns the block. + * @index: The index of this block in its zone's summary. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check initialize_slab_summary_block(struct block_allocator *allocator, + block_count_t index) +{ + struct slab_summary_block *block = &allocator->summary_blocks[index]; + int result; + + result = uds_allocate(VDO_BLOCK_SIZE, char, __func__, &block->outgoing_entries); + if (result != VDO_SUCCESS) + return result; + + result = allocate_vio_components(allocator->depot->vdo, VIO_TYPE_SLAB_SUMMARY, + VIO_PRIORITY_METADATA, NULL, 1, + block->outgoing_entries, &block->vio); + if (result != VDO_SUCCESS) + return result; + + block->allocator = allocator; + block->entries = &allocator->summary_entries[VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK * index]; + block->index = index; + return VDO_SUCCESS; +} + +static int __must_check initialize_block_allocator(struct slab_depot *depot, + zone_count_t zone) +{ + int result; + block_count_t i; + struct block_allocator *allocator = &depot->allocators[zone]; + struct vdo *vdo = depot->vdo; + block_count_t max_free_blocks = depot->slab_config.data_blocks; + unsigned int max_priority = (2 + ilog2(max_free_blocks)); + + *allocator = (struct block_allocator) { + .depot = depot, + .zone_number = zone, + .thread_id = vdo->thread_config.physical_threads[zone], + .nonce = vdo->states.vdo.nonce, + }; + + INIT_LIST_HEAD(&allocator->dirty_slab_journals); + vdo_set_admin_state_code(&allocator->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + result = vdo_register_read_only_listener(vdo, allocator, + notify_block_allocator_of_read_only_mode, + allocator->thread_id); + if (result != VDO_SUCCESS) + return result; + + vdo_initialize_completion(&allocator->completion, vdo, VDO_BLOCK_ALLOCATOR_COMPLETION); + result = make_vio_pool(vdo, BLOCK_ALLOCATOR_VIO_POOL_SIZE, allocator->thread_id, + VIO_TYPE_SLAB_JOURNAL, VIO_PRIORITY_METADATA, + allocator, &allocator->vio_pool); + if (result != VDO_SUCCESS) + return result; + + result = initialize_slab_scrubber(allocator); + if (result != VDO_SUCCESS) + return result; + + result = vdo_make_priority_table(max_priority, &allocator->prioritized_slabs); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE, + struct slab_summary_block, __func__, + &allocator->summary_blocks); + if (result != VDO_SUCCESS) + return result; + + vdo_set_admin_state_code(&allocator->summary_state, + VDO_ADMIN_STATE_NORMAL_OPERATION); + allocator->summary_entries = depot->summary_entries + (MAX_VDO_SLABS * zone); + + /* Initialize each summary block. */ + for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { + result = initialize_slab_summary_block(allocator, i); + if (result != VDO_SUCCESS) + return result; + } + + /* + * Performing well atop thin provisioned storage requires either that VDO discards freed + * blocks, or that the block allocator try to use slabs that already have allocated blocks + * in preference to slabs that have never been opened. For reasons we have not been able to + * fully understand, some SSD machines have been have been very sensitive (50% reduction in + * test throughput) to very slight differences in the timing and locality of block + * allocation. Assigning a low priority to unopened slabs (max_priority/2, say) would be + * ideal for the story, but anything less than a very high threshold (max_priority - 1) + * hurts on these machines. + * + * This sets the free block threshold for preferring to open an unopened slab to the binary + * floor of 3/4ths the total number of data blocks in a slab, which will generally evaluate + * to about half the slab size. + */ + allocator->unopened_slab_priority = (1 + ilog2((max_free_blocks * 3) / 4)); + + return VDO_SUCCESS; +} + +static void uninitialize_allocator_summary(struct block_allocator *allocator) +{ + block_count_t i; + + if (allocator->summary_blocks == NULL) + return; + + for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { + free_vio_components(&allocator->summary_blocks[i].vio); + uds_free(uds_forget(allocator->summary_blocks[i].outgoing_entries)); + } + + uds_free(uds_forget(allocator->summary_blocks)); +} + /** * finish_combining_zones() - Clean up after saving out the combined slab summary. * @completion: The vio which was used to write the summary data. @@ -2722,3 +4009,147 @@ static void load_slab_summary(void *context, struct vdo_completion *parent) submit_metadata_vio(vio, depot->summary_origin, load_summary_endio, handle_combining_error, REQ_OP_READ); } + +/** + * stop_scrubbing() - Tell the scrubber to stop scrubbing after it finishes the slab it is + * currently working on. + * @scrubber: The scrubber to stop. + * @parent: The completion to notify when scrubbing has stopped. + */ +static void stop_scrubbing(struct block_allocator *allocator) +{ + struct slab_scrubber *scrubber = &allocator->scrubber; + + if (vdo_is_state_quiescent(&scrubber->admin_state)) { + vdo_finish_completion(&allocator->completion); + } else { + vdo_start_draining(&scrubber->admin_state, + VDO_ADMIN_STATE_SUSPENDING, + &allocator->completion, NULL); + } +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_summary_drain(struct admin_state *state) +{ + check_summary_drain_complete(container_of(state, struct block_allocator, + summary_state)); +} + +static void do_drain_step(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + vdo_prepare_completion_for_requeue(&allocator->completion, do_drain_step, + handle_operation_error, allocator->thread_id, + NULL); + switch (++allocator->drain_step) { + case VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER: + stop_scrubbing(allocator); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SLABS: + apply_to_slabs(allocator, do_drain_step); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SUMMARY: + vdo_start_draining(&allocator->summary_state, + vdo_get_admin_state_code(&allocator->state), + completion, initiate_summary_drain); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_FINISHED: + ASSERT_LOG_ONLY(!is_vio_pool_busy(allocator->vio_pool), + "vio pool not busy"); + vdo_finish_draining_with_result(&allocator->state, completion->result); + return; + + default: + vdo_finish_draining_with_result(&allocator->state, UDS_BAD_STATE); + } +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_drain(struct admin_state *state) +{ + struct block_allocator *allocator = + container_of(state, struct block_allocator, state); + + allocator->drain_step = VDO_DRAIN_ALLOCATOR_START; + do_drain_step(&allocator->completion); +} + +/** + * resume_scrubbing() - Tell the scrubber to resume scrubbing if it has been stopped. + * @allocator: The allocator being resumed. + */ +static void resume_scrubbing(struct block_allocator *allocator) +{ + int result; + struct slab_scrubber *scrubber = &allocator->scrubber; + + if (!has_slabs_to_scrub(scrubber)) { + vdo_finish_completion(&allocator->completion); + return; + } + + result = vdo_resume_if_quiescent(&scrubber->admin_state); + if (result != VDO_SUCCESS) { + vdo_fail_completion(&allocator->completion, result); + return; + } + + scrub_next_slab(scrubber); + vdo_finish_completion(&allocator->completion); +} + +static void do_resume_step(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + vdo_prepare_completion_for_requeue(&allocator->completion, do_resume_step, + handle_operation_error, + allocator->thread_id, NULL); + switch (--allocator->drain_step) { + case VDO_DRAIN_ALLOCATOR_STEP_SUMMARY: + vdo_fail_completion(completion, + vdo_resume_if_quiescent(&allocator->summary_state)); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SLABS: + apply_to_slabs(allocator, do_resume_step); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER: + resume_scrubbing(allocator); + return; + + case VDO_DRAIN_ALLOCATOR_START: + vdo_finish_resuming_with_result(&allocator->state, completion->result); + return; + + default: + vdo_finish_resuming_with_result(&allocator->state, UDS_BAD_STATE); + } +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_resume(struct admin_state *state) +{ + struct block_allocator *allocator = + container_of(state, struct block_allocator, state); + + allocator->drain_step = VDO_DRAIN_ALLOCATOR_STEP_FINISHED; + do_resume_step(&allocator->completion); +} + +/* Implements vdo_zone_action_fn. */ +static void resume_allocator(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + vdo_start_resuming(&depot->allocators[zone_number].state, + vdo_get_current_manager_operation(depot->action_manager), + parent, initiate_resume); +} diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h index 298f2b7148d7..2a88b3198ad1 100644 --- a/drivers/md/dm-vdo/slab-depot.h +++ b/drivers/md/dm-vdo/slab-depot.h @@ -257,6 +257,54 @@ struct vdo_slab { struct reference_block *reference_blocks; }; +enum block_allocator_drain_step { + VDO_DRAIN_ALLOCATOR_START, + VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER, + VDO_DRAIN_ALLOCATOR_STEP_SLABS, + VDO_DRAIN_ALLOCATOR_STEP_SUMMARY, + VDO_DRAIN_ALLOCATOR_STEP_FINISHED, +}; + +struct slab_scrubber { + /* The queue of slabs to scrub first */ + struct list_head high_priority_slabs; + /* The queue of slabs to scrub once there are no high_priority_slabs */ + struct list_head slabs; + /* The queue of VIOs waiting for a slab to be scrubbed */ + struct wait_queue waiters; + + /* + * The number of slabs that are unrecovered or being scrubbed. This field is modified by + * the physical zone thread, but is queried by other threads. + */ + slab_count_t slab_count; + + /* The administrative state of the scrubber */ + struct admin_state admin_state; + /* Whether to only scrub high-priority slabs */ + bool high_priority_only; + /* The slab currently being scrubbed */ + struct vdo_slab *slab; + /* The vio for loading slab journal blocks */ + struct vio vio; +}; + +/* A sub-structure for applying actions in parallel to all an allocator's slabs. */ +struct slab_actor { + /* The number of slabs performing a slab action */ + slab_count_t slab_action_count; + /* The method to call when a slab action has been completed by all slabs */ + vdo_action_fn callback; +}; + +/* A slab_iterator is a structure for iterating over a set of slabs. */ +struct slab_iterator { + struct vdo_slab **slabs; + struct vdo_slab *next; + slab_count_t end; + slab_count_t stride; +}; + /* * The slab_summary provides hints during load and recovery about the state of the slabs in order * to avoid the need to read the slab journals in their entirety before a VDO can come online. @@ -314,6 +362,81 @@ struct atomic_slab_summary_statistics { atomic64_t blocks_written; }; +struct block_allocator { + struct vdo_completion completion; + /* The slab depot for this allocator */ + struct slab_depot *depot; + /* The nonce of the VDO */ + nonce_t nonce; + /* The physical zone number of this allocator */ + zone_count_t zone_number; + /* The thread ID for this allocator's physical zone */ + thread_id_t thread_id; + /* The number of slabs in this allocator */ + slab_count_t slab_count; + /* The number of the last slab owned by this allocator */ + slab_count_t last_slab; + /* The reduced priority level used to preserve unopened slabs */ + unsigned int unopened_slab_priority; + /* The state of this allocator */ + struct admin_state state; + /* The actor for applying an action to all slabs */ + struct slab_actor slab_actor; + + /* The slab from which blocks are currently being allocated */ + struct vdo_slab *open_slab; + /* A priority queue containing all slabs available for allocation */ + struct priority_table *prioritized_slabs; + /* The slab scrubber */ + struct slab_scrubber scrubber; + /* What phase of the close operation the allocator is to perform */ + enum block_allocator_drain_step drain_step; + + /* + * These statistics are all mutated only by the physical zone thread, but are read by other + * threads when gathering statistics for the entire depot. + */ + /* + * The count of allocated blocks in this zone. Not in block_allocator_statistics for + * historical reasons. + */ + u64 allocated_blocks; + /* Statistics for this block allocator */ + struct block_allocator_statistics statistics; + /* Cumulative statistics for the slab journals in this zone */ + struct slab_journal_statistics slab_journal_statistics; + /* Cumulative statistics for the reference counters in this zone */ + struct ref_counts_statistics ref_counts_statistics; + + /* + * This is the head of a queue of slab journals which have entries in their tail blocks + * which have not yet started to commit. When the recovery journal is under space pressure, + * slab journals which have uncommitted entries holding a lock on the recovery journal head + * are forced to commit their blocks early. This list is kept in order, with the tail + * containing the slab journal holding the most recent recovery journal lock. + */ + struct list_head dirty_slab_journals; + + /* The vio pool for reading and writing block allocator metadata */ + struct vio_pool *vio_pool; + /* The dm_kcopyd client for erasing slab journals */ + struct dm_kcopyd_client *eraser; + /* Iterator over the slabs to be erased */ + struct slab_iterator slabs_to_erase; + + /* The portion of the slab summary managed by this allocator */ + /* The state of the slab summary */ + struct admin_state summary_state; + /* The number of outstanding summary writes */ + block_count_t summary_write_count; + /* The array (owned by the blocks) of all entries */ + struct slab_summary_entry *summary_entries; + /* The array of slab_summary_blocks */ + struct slab_summary_block *summary_blocks; +}; + +struct reference_updater; + bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn, enum journal_operation operation, @@ -321,6 +444,30 @@ bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, struct journal_point *recovery_point, struct vdo_completion *parent); +static inline struct block_allocator *vdo_as_block_allocator(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_BLOCK_ALLOCATOR_COMPLETION); + return container_of(completion, struct block_allocator, completion); +} + +int __must_check vdo_acquire_provisional_reference(struct vdo_slab *slab, + physical_block_number_t pbn, + struct pbn_lock *lock); + +int __must_check vdo_allocate_block(struct block_allocator *allocator, + physical_block_number_t *block_number_ptr); + +int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, + struct waiter *waiter); + +void vdo_modify_reference_count(struct vdo_completion *completion, + struct reference_updater *updater); + +int __must_check vdo_release_block_reference(struct block_allocator *allocator, + physical_block_number_t pbn); + void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion); +void vdo_dump_block_allocator(const struct block_allocator *allocator); + #endif /* VDO_SLAB_DEPOT_H */ From 7ce49449ffb9409935d78712550be97cd5e68bb2 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:00:27 -0500 Subject: [PATCH 0576/1038] dm vdo: add the slab depot Add the data and methods that implement the slab_depot that manages the allocation of slabs of blocks added by the preceding patches. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/slab-depot.c | 940 +++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/slab-depot.h | 122 +++++ 2 files changed, 1062 insertions(+) diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index da602cbb278e..1ad97d2bf80b 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -2997,6 +2997,31 @@ static void register_slab_with_allocator(struct block_allocator *allocator, allocator->last_slab = slab->slab_number; } +/** + * get_depot_slab_iterator() - Return a slab_iterator over the slabs in a slab_depot. + * @depot: The depot over which to iterate. + * @start: The number of the slab to start iterating from. + * @end: The number of the last slab which may be returned. + * @stride: The difference in slab number between successive slabs. + * + * Iteration always occurs from higher to lower numbered slabs. + * + * Return: An initialized iterator structure. + */ +static struct slab_iterator get_depot_slab_iterator(struct slab_depot *depot, + slab_count_t start, slab_count_t end, + slab_count_t stride) +{ + struct vdo_slab **slabs = depot->slabs; + + return (struct slab_iterator) { + .slabs = slabs, + .next = (((slabs == NULL) || (start < end)) ? NULL : slabs[start]), + .end = end, + .stride = stride, + }; +} + static struct slab_iterator get_slab_iterator(const struct block_allocator *allocator) { return get_depot_slab_iterator(allocator->depot, allocator->last_slab, @@ -3728,6 +3753,164 @@ static int __must_check make_slab(physical_block_number_t slab_origin, return VDO_SUCCESS; } +/** + * allocate_slabs() - Allocate a new slab pointer array. + * @depot: The depot. + * @slab_count: The number of slabs the depot should have in the new array. + * + * Any existing slab pointers will be copied into the new array, and slabs will be allocated as + * needed. The newly allocated slabs will not be distributed for use by the block allocators. + * + * Return: VDO_SUCCESS or an error code. + */ +static int allocate_slabs(struct slab_depot *depot, slab_count_t slab_count) +{ + block_count_t slab_size; + bool resizing = false; + physical_block_number_t slab_origin; + int result; + + result = uds_allocate(slab_count, struct vdo_slab *, + "slab pointer array", &depot->new_slabs); + if (result != VDO_SUCCESS) + return result; + + if (depot->slabs != NULL) { + memcpy(depot->new_slabs, depot->slabs, + depot->slab_count * sizeof(struct vdo_slab *)); + resizing = true; + } + + slab_size = depot->slab_config.slab_blocks; + slab_origin = depot->first_block + (depot->slab_count * slab_size); + + for (depot->new_slab_count = depot->slab_count; + depot->new_slab_count < slab_count; + depot->new_slab_count++, slab_origin += slab_size) { + struct block_allocator *allocator = + &depot->allocators[depot->new_slab_count % depot->zone_count]; + struct vdo_slab **slab_ptr = &depot->new_slabs[depot->new_slab_count]; + + result = make_slab(slab_origin, allocator, depot->new_slab_count, + resizing, slab_ptr); + if (result != VDO_SUCCESS) + return result; + } + + return VDO_SUCCESS; +} + +/** + * vdo_abandon_new_slabs() - Abandon any new slabs in this depot, freeing them as needed. + * @depot: The depot. + */ +void vdo_abandon_new_slabs(struct slab_depot *depot) +{ + slab_count_t i; + + if (depot->new_slabs == NULL) + return; + + for (i = depot->slab_count; i < depot->new_slab_count; i++) + free_slab(uds_forget(depot->new_slabs[i])); + depot->new_slab_count = 0; + depot->new_size = 0; + uds_free(uds_forget(depot->new_slabs)); +} + +/** + * get_allocator_thread_id() - Get the ID of the thread on which a given allocator operates. + * + * Implements vdo_zone_thread_getter_fn. + */ +static thread_id_t get_allocator_thread_id(void *context, zone_count_t zone_number) +{ + return ((struct slab_depot *) context)->allocators[zone_number].thread_id; +} + +/** + * release_recovery_journal_lock() - Request the slab journal to release the recovery journal lock + * it may hold on a specified recovery journal block. + * @journal: The slab journal. + * @recovery_lock: The sequence number of the recovery journal block whose locks should be + * released. + * + * Return: true if the journal does hold a lock on the specified block (which it will release). + */ +static bool __must_check release_recovery_journal_lock(struct slab_journal *journal, + sequence_number_t recovery_lock) +{ + if (recovery_lock > journal->recovery_lock) { + ASSERT_LOG_ONLY((recovery_lock < journal->recovery_lock), + "slab journal recovery lock is not older than the recovery journal head"); + return false; + } + + if ((recovery_lock < journal->recovery_lock) || + vdo_is_read_only(journal->slab->allocator->depot->vdo)) + return false; + + /* All locks are held by the block which is in progress; write it. */ + commit_tail(journal); + return true; +} + +/* + * Request a commit of all dirty tail blocks which are locking the recovery journal block the depot + * is seeking to release. + * + * Implements vdo_zone_action_fn. + */ +static void release_tail_block_locks(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_journal *journal, *tmp; + struct slab_depot *depot = context; + struct list_head *list = &depot->allocators[zone_number].dirty_slab_journals; + + list_for_each_entry_safe(journal, tmp, list, dirty_entry) { + if (!release_recovery_journal_lock(journal, + depot->active_release_request)) + break; + } + + vdo_finish_completion(parent); +} + +/** + * prepare_for_tail_block_commit() - Prepare to commit oldest tail blocks. + * + * Implements vdo_action_preamble_fn. + */ +static void prepare_for_tail_block_commit(void *context, struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + depot->active_release_request = depot->new_release_request; + vdo_finish_completion(parent); +} + +/** + * schedule_tail_block_commit() - Schedule a tail block commit if necessary. + * + * This method should not be called directly. Rather, call vdo_schedule_default_action() on the + * depot's action manager. + * + * Implements vdo_action_scheduler_fn. + */ +static bool schedule_tail_block_commit(void *context) +{ + struct slab_depot *depot = context; + + if (depot->new_release_request == depot->active_release_request) + return false; + + return vdo_schedule_action(depot->action_manager, + prepare_for_tail_block_commit, + release_tail_block_locks, + NULL, NULL); +} + /** * initialize_slab_scrubber() - Initialize an allocator's slab scrubber. * @allocator: The allocator being initialized @@ -3868,6 +4051,145 @@ static int __must_check initialize_block_allocator(struct slab_depot *depot, return VDO_SUCCESS; } +static int allocate_components(struct slab_depot *depot, + struct partition *summary_partition) +{ + int result; + zone_count_t zone; + slab_count_t slab_count; + u8 hint; + u32 i; + const struct thread_config *thread_config = &depot->vdo->thread_config; + + result = vdo_make_action_manager(depot->zone_count, get_allocator_thread_id, + thread_config->journal_thread, depot, + schedule_tail_block_commit, + depot->vdo, &depot->action_manager); + if (result != VDO_SUCCESS) + return result; + + depot->origin = depot->first_block; + + /* block size must be a multiple of entry size */ + BUILD_BUG_ON((VDO_BLOCK_SIZE % sizeof(struct slab_summary_entry)) != 0); + + depot->summary_origin = summary_partition->offset; + depot->hint_shift = vdo_get_slab_summary_hint_shift(depot->slab_size_shift); + result = uds_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES, + struct slab_summary_entry, __func__, + &depot->summary_entries); + if (result != VDO_SUCCESS) + return result; + + + /* Initialize all the entries. */ + hint = compute_fullness_hint(depot, depot->slab_config.data_blocks); + for (i = 0; i < MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES; i++) { + /* + * This default tail block offset must be reflected in + * slabJournal.c::read_slab_journal_tail(). + */ + depot->summary_entries[i] = (struct slab_summary_entry) { + .tail_block_offset = 0, + .fullness_hint = hint, + .load_ref_counts = false, + .is_dirty = false, + }; + } + + if (result != VDO_SUCCESS) + return result; + + slab_count = vdo_compute_slab_count(depot->first_block, depot->last_block, + depot->slab_size_shift); + if (thread_config->physical_zone_count > slab_count) { + return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + "%u physical zones exceeds slab count %u", + thread_config->physical_zone_count, + slab_count); + } + + /* Initialize the block allocators. */ + for (zone = 0; zone < depot->zone_count; zone++) { + result = initialize_block_allocator(depot, zone); + if (result != VDO_SUCCESS) + return result; + } + + /* Allocate slabs. */ + result = allocate_slabs(depot, slab_count); + if (result != VDO_SUCCESS) + return result; + + /* Use the new slabs. */ + for (i = depot->slab_count; i < depot->new_slab_count; i++) { + struct vdo_slab *slab = depot->new_slabs[i]; + + register_slab_with_allocator(slab->allocator, slab); + WRITE_ONCE(depot->slab_count, depot->slab_count + 1); + } + + depot->slabs = depot->new_slabs; + depot->new_slabs = NULL; + depot->new_slab_count = 0; + + return VDO_SUCCESS; +} + +/** + * vdo_decode_slab_depot() - Make a slab depot and configure it with the state read from the super + * block. + * @state: The slab depot state from the super block. + * @vdo: The VDO which will own the depot. + * @summary_partition: The partition which holds the slab summary. + * @depot_ptr: A pointer to hold the depot. + * + * Return: A success or error code. + */ +int vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo, + struct partition *summary_partition, + struct slab_depot **depot_ptr) +{ + unsigned int slab_size_shift; + struct slab_depot *depot; + int result; + + /* + * Calculate the bit shift for efficiently mapping block numbers to slabs. Using a shift + * requires that the slab size be a power of two. + */ + block_count_t slab_size = state.slab_config.slab_blocks; + + if (!is_power_of_2(slab_size)) { + return uds_log_error_strerror(UDS_INVALID_ARGUMENT, + "slab size must be a power of two"); + } + slab_size_shift = ilog2(slab_size); + + result = uds_allocate_extended(struct slab_depot, + vdo->thread_config.physical_zone_count, + struct block_allocator, __func__, &depot); + if (result != VDO_SUCCESS) + return result; + + depot->vdo = vdo; + depot->old_zone_count = state.zone_count; + depot->zone_count = vdo->thread_config.physical_zone_count; + depot->slab_config = state.slab_config; + depot->first_block = state.first_block; + depot->last_block = state.last_block; + depot->slab_size_shift = slab_size_shift; + + result = allocate_components(depot, summary_partition); + if (result != VDO_SUCCESS) { + vdo_free_slab_depot(depot); + return result; + } + + *depot_ptr = depot; + return VDO_SUCCESS; +} + static void uninitialize_allocator_summary(struct block_allocator *allocator) { block_count_t i; @@ -3883,6 +4205,233 @@ static void uninitialize_allocator_summary(struct block_allocator *allocator) uds_free(uds_forget(allocator->summary_blocks)); } +/** + * vdo_free_slab_depot() - Destroy a slab depot. + * @depot: The depot to destroy. + */ +void vdo_free_slab_depot(struct slab_depot *depot) +{ + zone_count_t zone = 0; + + if (depot == NULL) + return; + + vdo_abandon_new_slabs(depot); + + for (zone = 0; zone < depot->zone_count; zone++) { + struct block_allocator *allocator = &depot->allocators[zone]; + + if (allocator->eraser != NULL) + dm_kcopyd_client_destroy(uds_forget(allocator->eraser)); + + uninitialize_allocator_summary(allocator); + uninitialize_scrubber_vio(&allocator->scrubber); + free_vio_pool(uds_forget(allocator->vio_pool)); + vdo_free_priority_table(uds_forget(allocator->prioritized_slabs)); + } + + if (depot->slabs != NULL) { + slab_count_t i; + + for (i = 0; i < depot->slab_count; i++) + free_slab(uds_forget(depot->slabs[i])); + } + + uds_free(uds_forget(depot->slabs)); + uds_free(uds_forget(depot->action_manager)); + uds_free(uds_forget(depot->summary_entries)); + uds_free(depot); +} + +/** + * vdo_record_slab_depot() - Record the state of a slab depot for encoding into the super block. + * @depot: The depot to encode. + * + * Return: The depot state. + */ +struct slab_depot_state_2_0 vdo_record_slab_depot(const struct slab_depot *depot) +{ + /* + * If this depot is currently using 0 zones, it must have been synchronously loaded by a + * tool and is now being saved. We did not load and combine the slab summary, so we still + * need to do that next time we load with the old zone count rather than 0. + */ + struct slab_depot_state_2_0 state; + zone_count_t zones_to_record = depot->zone_count; + + if (depot->zone_count == 0) + zones_to_record = depot->old_zone_count; + + state = (struct slab_depot_state_2_0) { + .slab_config = depot->slab_config, + .first_block = depot->first_block, + .last_block = depot->last_block, + .zone_count = zones_to_record, + }; + + return state; +} + +/** + * vdo_allocate_reference_counters() - Allocate the reference counters for all slabs in the depot. + * + * Context: This method may be called only before entering normal operation from the load thread. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_allocate_reference_counters(struct slab_depot *depot) +{ + struct slab_iterator iterator = + get_depot_slab_iterator(depot, depot->slab_count - 1, 0, 1); + + while (iterator.next != NULL) { + int result = allocate_slab_counters(next_slab(&iterator)); + + if (result != VDO_SUCCESS) + return result; + } + + return VDO_SUCCESS; +} + +/** + * get_slab_number() - Get the number of the slab that contains a specified block. + * @depot: The slab depot. + * @pbn: The physical block number. + * @slab_number_ptr: A pointer to hold the slab number. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check get_slab_number(const struct slab_depot *depot, + physical_block_number_t pbn, + slab_count_t *slab_number_ptr) +{ + slab_count_t slab_number; + + if (pbn < depot->first_block) + return VDO_OUT_OF_RANGE; + + slab_number = (pbn - depot->first_block) >> depot->slab_size_shift; + if (slab_number >= depot->slab_count) + return VDO_OUT_OF_RANGE; + + *slab_number_ptr = slab_number; + return VDO_SUCCESS; +} + +/** + * vdo_get_slab() - Get the slab object for the slab that contains a specified block. + * @depot: The slab depot. + * @pbn: The physical block number. + * + * Will put the VDO in read-only mode if the PBN is not a valid data block nor the zero block. + * + * Return: The slab containing the block, or NULL if the block number is the zero block or + * otherwise out of range. + */ +struct vdo_slab *vdo_get_slab(const struct slab_depot *depot, + physical_block_number_t pbn) +{ + slab_count_t slab_number; + int result; + + if (pbn == VDO_ZERO_BLOCK) + return NULL; + + result = get_slab_number(depot, pbn, &slab_number); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(depot->vdo, result); + return NULL; + } + + return depot->slabs[slab_number]; +} + +/** + * vdo_get_increment_limit() - Determine how many new references a block can acquire. + * @depot: The slab depot. + * @pbn: The physical block number that is being queried. + * + * Context: This method must be called from the physical zone thread of the PBN. + * + * Return: The number of available references. + */ +u8 vdo_get_increment_limit(struct slab_depot *depot, physical_block_number_t pbn) +{ + struct vdo_slab *slab = vdo_get_slab(depot, pbn); + vdo_refcount_t *counter_ptr = NULL; + int result; + + if ((slab == NULL) || (slab->status != VDO_SLAB_REBUILT)) + return 0; + + result = get_reference_counter(slab, pbn, &counter_ptr); + if (result != VDO_SUCCESS) + return 0; + + if (*counter_ptr == PROVISIONAL_REFERENCE_COUNT) + return (MAXIMUM_REFERENCE_COUNT - 1); + + return (MAXIMUM_REFERENCE_COUNT - *counter_ptr); +} + +/** + * vdo_is_physical_data_block() - Determine whether the given PBN refers to a data block. + * @depot: The depot. + * @pbn: The physical block number to ask about. + * + * Return: True if the PBN corresponds to a data block. + */ +bool vdo_is_physical_data_block(const struct slab_depot *depot, + physical_block_number_t pbn) +{ + slab_count_t slab_number; + slab_block_number sbn; + + return ((pbn == VDO_ZERO_BLOCK) || + ((get_slab_number(depot, pbn, &slab_number) == VDO_SUCCESS) && + (slab_block_number_from_pbn(depot->slabs[slab_number], pbn, &sbn) == + VDO_SUCCESS))); +} + +/** + * vdo_get_slab_depot_allocated_blocks() - Get the total number of data blocks allocated across all + * the slabs in the depot. + * @depot: The slab depot. + * + * This is the total number of blocks with a non-zero reference count. + * + * Context: This may be called from any thread. + * + * Return: The total number of blocks with a non-zero reference count. + */ +block_count_t vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot) +{ + block_count_t total = 0; + zone_count_t zone; + + for (zone = 0; zone < depot->zone_count; zone++) { + /* The allocators are responsible for thread safety. */ + total += READ_ONCE(depot->allocators[zone].allocated_blocks); + } + + return total; +} + +/** + * vdo_get_slab_depot_data_blocks() - Get the total number of data blocks in all the slabs in the + * depot. + * @depot: The slab depot. + * + * Context: This may be called from any thread. + * + * Return: The total number of data blocks in all slabs. + */ +block_count_t vdo_get_slab_depot_data_blocks(const struct slab_depot *depot) +{ + return (READ_ONCE(depot->slab_count) * depot->slab_config.data_blocks); +} + /** * finish_combining_zones() - Clean up after saving out the combined slab summary. * @completion: The vio which was used to write the summary data. @@ -4010,6 +4559,188 @@ static void load_slab_summary(void *context, struct vdo_completion *parent) handle_combining_error, REQ_OP_READ); } +/* Implements vdo_zone_action_fn. */ +static void load_allocator(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + vdo_start_loading(&depot->allocators[zone_number].state, + vdo_get_current_manager_operation(depot->action_manager), + parent, initiate_load); +} + +/** + * vdo_load_slab_depot() - Asynchronously load any slab depot state that isn't included in the + * super_block component. + * @depot: The depot to load. + * @operation: The type of load to perform. + * @parent: The completion to notify when the load is complete. + * @context: Additional context for the load operation; may be NULL. + * + * This method may be called only before entering normal operation from the load thread. + */ +void vdo_load_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent, void *context) +{ + if (!vdo_assert_load_operation(operation, parent)) + return; + + vdo_schedule_operation_with_context(depot->action_manager, operation, + load_slab_summary, load_allocator, + NULL, context, parent); +} + +/* Implements vdo_zone_action_fn. */ +static void prepare_to_allocate(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + struct block_allocator *allocator = &depot->allocators[zone_number]; + int result; + + result = vdo_prepare_slabs_for_allocation(allocator); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + scrub_slabs(allocator, parent); +} + +/** + * vdo_prepare_slab_depot_to_allocate() - Prepare the slab depot to come online and start + * allocating blocks. + * @depot: The depot to prepare. + * @load_type: The load type. + * @parent: The completion to notify when the operation is complete. + * + * This method may be called only before entering normal operation from the load thread. It must be + * called before allocation may proceed. + */ +void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot, + enum slab_depot_load_type load_type, + struct vdo_completion *parent) +{ + depot->load_type = load_type; + atomic_set(&depot->zones_to_scrub, depot->zone_count); + vdo_schedule_action(depot->action_manager, NULL, + prepare_to_allocate, NULL, parent); +} + +/** + * vdo_update_slab_depot_size() - Update the slab depot to reflect its new size in memory. + * @depot: The depot to update. + * + * This size is saved to disk as part of the super block. + */ +void vdo_update_slab_depot_size(struct slab_depot *depot) +{ + depot->last_block = depot->new_last_block; +} + +/** + * vdo_prepare_to_grow_slab_depot() - Allocate new memory needed for a resize of a slab depot to + * the given size. + * @depot: The depot to prepare to resize. + * @partition: The new depot partition + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, + const struct partition *partition) +{ + struct slab_depot_state_2_0 new_state; + int result; + slab_count_t new_slab_count; + + if ((partition->count >> depot->slab_size_shift) <= depot->slab_count) + return VDO_INCREMENT_TOO_SMALL; + + /* Generate the depot configuration for the new block count. */ + ASSERT_LOG_ONLY(depot->first_block == partition->offset, + "New slab depot partition doesn't change origin"); + result = vdo_configure_slab_depot(partition, depot->slab_config, + depot->zone_count, &new_state); + if (result != VDO_SUCCESS) + return result; + + new_slab_count = vdo_compute_slab_count(depot->first_block, + new_state.last_block, + depot->slab_size_shift); + if (new_slab_count <= depot->slab_count) + return uds_log_error_strerror(VDO_INCREMENT_TOO_SMALL, + "Depot can only grow"); + if (new_slab_count == depot->new_slab_count) { + /* Check it out, we've already got all the new slabs allocated! */ + return VDO_SUCCESS; + } + + vdo_abandon_new_slabs(depot); + result = allocate_slabs(depot, new_slab_count); + if (result != VDO_SUCCESS) { + vdo_abandon_new_slabs(depot); + return result; + } + + depot->new_size = partition->count; + depot->old_last_block = depot->last_block; + depot->new_last_block = new_state.last_block; + + return VDO_SUCCESS; +} + +/** + * finish_registration() - Finish registering new slabs now that all of the allocators have + * received their new slabs. + * + * Implements vdo_action_conclusion_fn. + */ +static int finish_registration(void *context) +{ + struct slab_depot *depot = context; + + WRITE_ONCE(depot->slab_count, depot->new_slab_count); + uds_free(depot->slabs); + depot->slabs = depot->new_slabs; + depot->new_slabs = NULL; + depot->new_slab_count = 0; + return VDO_SUCCESS; +} + +/* Implements vdo_zone_action_fn. */ +static void register_new_slabs(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + struct block_allocator *allocator = &depot->allocators[zone_number]; + slab_count_t i; + + for (i = depot->slab_count; i < depot->new_slab_count; i++) { + struct vdo_slab *slab = depot->new_slabs[i]; + + if (slab->allocator == allocator) + register_slab_with_allocator(allocator, slab); + } + + vdo_finish_completion(parent); +} + +/** + * vdo_use_new_slabs() - Use the new slabs allocated for resize. + * @depot: The depot. + * @parent: The object to notify when complete. + */ +void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent) +{ + ASSERT_LOG_ONLY(depot->new_slabs != NULL, "Must have new slabs to use"); + vdo_schedule_operation(depot->action_manager, + VDO_ADMIN_STATE_SUSPENDED_OPERATION, + NULL, register_new_slabs, + finish_registration, parent); +} + /** * stop_scrubbing() - Tell the scrubber to stop scrubbing after it finishes the slab it is * currently working on. @@ -4079,6 +4810,39 @@ static void initiate_drain(struct admin_state *state) do_drain_step(&allocator->completion); } +/* + * Drain all allocator I/O. Depending upon the type of drain, some or all dirty metadata may be + * written to disk. The type of drain will be determined from the state of the allocator's depot. + * + * Implements vdo_zone_action_fn. + */ +static void drain_allocator(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + vdo_start_draining(&depot->allocators[zone_number].state, + vdo_get_current_manager_operation(depot->action_manager), + parent, initiate_drain); +} + +/** + * vdo_drain_slab_depot() - Drain all slab depot I/O. + * @depot: The depot to drain. + * @operation: The drain operation (flush, rebuild, suspend, or save). + * @parent: The completion to finish when the drain is complete. + * + * If saving, or flushing, all dirty depot metadata will be written out. If saving or suspending, + * the depot will be left in a suspended state. + */ +void vdo_drain_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + vdo_schedule_operation(depot->action_manager, operation, + NULL, drain_allocator, NULL, parent); +} + /** * resume_scrubbing() - Tell the scrubber to resume scrubbing if it has been stopped. * @allocator: The allocator being resumed. @@ -4153,3 +4917,179 @@ static void resume_allocator(void *context, zone_count_t zone_number, vdo_get_current_manager_operation(depot->action_manager), parent, initiate_resume); } + +/** + * vdo_resume_slab_depot() - Resume a suspended slab depot. + * @depot: The depot to resume. + * @parent: The completion to finish when the depot has resumed. + */ +void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent) +{ + if (vdo_is_read_only(depot->vdo)) { + vdo_continue_completion(parent, VDO_READ_ONLY); + return; + } + + vdo_schedule_operation(depot->action_manager, VDO_ADMIN_STATE_RESUMING, + NULL, resume_allocator, NULL, parent); +} + +/** + * vdo_commit_oldest_slab_journal_tail_blocks() - Commit all dirty tail blocks which are locking a + * given recovery journal block. + * @depot: The depot. + * @recovery_block_number: The sequence number of the recovery journal block whose locks should be + * released. + * + * Context: This method must be called from the journal zone thread. + */ +void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot, + sequence_number_t recovery_block_number) +{ + if (depot == NULL) + return; + + depot->new_release_request = recovery_block_number; + vdo_schedule_default_action(depot->action_manager); +} + +/* Implements vdo_zone_action_fn. */ +static void scrub_all_unrecovered_slabs(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + scrub_slabs(&depot->allocators[zone_number], NULL); + vdo_launch_completion(parent); +} + +/** + * vdo_scrub_all_unrecovered_slabs() - Scrub all unrecovered slabs. + * @depot: The depot to scrub. + * @parent: The object to notify when scrubbing has been launched for all zones. + */ +void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot, + struct vdo_completion *parent) +{ + vdo_schedule_action(depot->action_manager, NULL, + scrub_all_unrecovered_slabs, + NULL, parent); +} + +/** + * get_block_allocator_statistics() - Get the total of the statistics from all the block allocators + * in the depot. + * @depot: The slab depot. + * + * Return: The statistics from all block allocators in the depot. + */ +static struct block_allocator_statistics __must_check +get_block_allocator_statistics(const struct slab_depot *depot) +{ + struct block_allocator_statistics totals; + zone_count_t zone; + + memset(&totals, 0, sizeof(totals)); + + for (zone = 0; zone < depot->zone_count; zone++) { + const struct block_allocator *allocator = &depot->allocators[zone]; + const struct block_allocator_statistics *stats = &allocator->statistics; + + totals.slab_count += allocator->slab_count; + totals.slabs_opened += READ_ONCE(stats->slabs_opened); + totals.slabs_reopened += READ_ONCE(stats->slabs_reopened); + } + + return totals; +} + +/** + * get_ref_counts_statistics() - Get the cumulative ref_counts statistics for the depot. + * @depot: The slab depot. + * + * Return: The cumulative statistics for all ref_counts in the depot. + */ +static struct ref_counts_statistics __must_check +get_ref_counts_statistics(const struct slab_depot *depot) +{ + struct ref_counts_statistics totals; + zone_count_t zone; + + memset(&totals, 0, sizeof(totals)); + + for (zone = 0; zone < depot->zone_count; zone++) { + totals.blocks_written += + READ_ONCE(depot->allocators[zone].ref_counts_statistics.blocks_written); + } + + return totals; +} + +/** + * get_depot_slab_journal_statistics() - Get the aggregated slab journal statistics for the depot. + * @depot: The slab depot. + * + * Return: The aggregated statistics for all slab journals in the depot. + */ +static struct slab_journal_statistics __must_check +get_slab_journal_statistics(const struct slab_depot *depot) +{ + struct slab_journal_statistics totals; + zone_count_t zone; + + memset(&totals, 0, sizeof(totals)); + + for (zone = 0; zone < depot->zone_count; zone++) { + const struct slab_journal_statistics *stats = + &depot->allocators[zone].slab_journal_statistics; + + totals.disk_full_count += READ_ONCE(stats->disk_full_count); + totals.flush_count += READ_ONCE(stats->flush_count); + totals.blocked_count += READ_ONCE(stats->blocked_count); + totals.blocks_written += READ_ONCE(stats->blocks_written); + totals.tail_busy_count += READ_ONCE(stats->tail_busy_count); + } + + return totals; +} + +/** + * vdo_get_slab_depot_statistics() - Get all the vdo_statistics fields that are properties of the + * slab depot. + * @depot: The slab depot. + * @stats: The vdo statistics structure to partially fill. + */ +void vdo_get_slab_depot_statistics(const struct slab_depot *depot, + struct vdo_statistics *stats) +{ + slab_count_t slab_count = READ_ONCE(depot->slab_count); + slab_count_t unrecovered = 0; + zone_count_t zone; + + for (zone = 0; zone < depot->zone_count; zone++) { + /* The allocators are responsible for thread safety. */ + unrecovered += READ_ONCE(depot->allocators[zone].scrubber.slab_count); + } + + stats->recovery_percentage = (slab_count - unrecovered) * 100 / slab_count; + stats->allocator = get_block_allocator_statistics(depot); + stats->ref_counts = get_ref_counts_statistics(depot); + stats->slab_journal = get_slab_journal_statistics(depot); + stats->slab_summary = (struct slab_summary_statistics) { + .blocks_written = atomic64_read(&depot->summary_statistics.blocks_written), + }; +} + +/** + * vdo_dump_slab_depot() - Dump the slab depot, in a thread-unsafe fashion. + * @depot: The slab depot. + */ +void vdo_dump_slab_depot(const struct slab_depot *depot) +{ + uds_log_info("vdo slab depot"); + uds_log_info(" zone_count=%u old_zone_count=%u slabCount=%u active_release_request=%llu new_release_request=%llu", + (unsigned int) depot->zone_count, + (unsigned int) depot->old_zone_count, READ_ONCE(depot->slab_count), + (unsigned long long) depot->active_release_request, + (unsigned long long) depot->new_release_request); +} diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h index 2a88b3198ad1..169021b0811a 100644 --- a/drivers/md/dm-vdo/slab-depot.h +++ b/drivers/md/dm-vdo/slab-depot.h @@ -435,6 +435,66 @@ struct block_allocator { struct slab_summary_block *summary_blocks; }; +enum slab_depot_load_type { + VDO_SLAB_DEPOT_NORMAL_LOAD, + VDO_SLAB_DEPOT_RECOVERY_LOAD, + VDO_SLAB_DEPOT_REBUILD_LOAD +}; + +struct slab_depot { + zone_count_t zone_count; + zone_count_t old_zone_count; + struct vdo *vdo; + struct slab_config slab_config; + struct action_manager *action_manager; + + physical_block_number_t first_block; + physical_block_number_t last_block; + physical_block_number_t origin; + + /* slab_size == (1 << slab_size_shift) */ + unsigned int slab_size_shift; + + /* Determines how slabs should be queued during load */ + enum slab_depot_load_type load_type; + + /* The state for notifying slab journals to release recovery journal */ + sequence_number_t active_release_request; + sequence_number_t new_release_request; + + /* State variables for scrubbing complete handling */ + atomic_t zones_to_scrub; + + /* Array of pointers to individually allocated slabs */ + struct vdo_slab **slabs; + /* The number of slabs currently allocated and stored in 'slabs' */ + slab_count_t slab_count; + + /* Array of pointers to a larger set of slabs (used during resize) */ + struct vdo_slab **new_slabs; + /* The number of slabs currently allocated and stored in 'new_slabs' */ + slab_count_t new_slab_count; + /* The size that 'new_slabs' was allocated for */ + block_count_t new_size; + + /* The last block before resize, for rollback */ + physical_block_number_t old_last_block; + /* The last block after resize, for resize */ + physical_block_number_t new_last_block; + + /* The statistics for the slab summary */ + struct atomic_slab_summary_statistics summary_statistics; + /* The start of the slab summary partition */ + physical_block_number_t summary_origin; + /* The number of bits to shift to get a 7-bit fullness hint */ + unsigned int hint_shift; + /* The slab summary entries for all of the zones the partition can hold */ + struct slab_summary_entry *summary_entries; + + /* The block allocators for this depot */ + struct block_allocator allocators[]; +}; + struct reference_updater; bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, @@ -444,6 +504,10 @@ bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, struct journal_point *recovery_point, struct vdo_completion *parent); +int __must_check vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot, + physical_block_number_t pbn, + enum journal_operation operation); + static inline struct block_allocator *vdo_as_block_allocator(struct vdo_completion *completion) { vdo_assert_completion_type(completion, VDO_BLOCK_ALLOCATOR_COMPLETION); @@ -470,4 +534,62 @@ void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion); void vdo_dump_block_allocator(const struct block_allocator *allocator); +int __must_check vdo_decode_slab_depot(struct slab_depot_state_2_0 state, + struct vdo *vdo, + struct partition *summary_partition, + struct slab_depot **depot_ptr); + +void vdo_free_slab_depot(struct slab_depot *depot); + +struct slab_depot_state_2_0 __must_check vdo_record_slab_depot(const struct slab_depot *depot); + +int __must_check vdo_allocate_reference_counters(struct slab_depot *depot); + +struct vdo_slab * __must_check vdo_get_slab(const struct slab_depot *depot, + physical_block_number_t pbn); + +u8 __must_check vdo_get_increment_limit(struct slab_depot *depot, + physical_block_number_t pbn); + +bool __must_check vdo_is_physical_data_block(const struct slab_depot *depot, + physical_block_number_t pbn); + +block_count_t __must_check vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot); + +block_count_t __must_check vdo_get_slab_depot_data_blocks(const struct slab_depot *depot); + +void vdo_get_slab_depot_statistics(const struct slab_depot *depot, + struct vdo_statistics *stats); + +void vdo_load_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent, void *context); + +void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot, + enum slab_depot_load_type load_type, + struct vdo_completion *parent); + +void vdo_update_slab_depot_size(struct slab_depot *depot); + +int __must_check vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, + const struct partition *partition); + +void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent); + +void vdo_abandon_new_slabs(struct slab_depot *depot); + +void vdo_drain_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent); + +void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent); + +void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot, + sequence_number_t recovery_block_number); + +void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot, + struct vdo_completion *parent); + +void vdo_dump_slab_depot(const struct slab_depot *depot); + #endif /* VDO_SLAB_DEPOT_H */ From ddb12d6714a5c28c87826180a331f5c68a66c5a5 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:05:22 -0500 Subject: [PATCH 0577/1038] dm vdo: add the block map The block map contains the logical to physical mapping. It can be thought of as an array with one entry per logical address. Each entry is 5 bytes: 36 bits contain the physical block number which holds the data for the given logical address, and the remaining 4 bits are used to indicate the nature of the mapping. Of the 16 possible states, one represents a logical address which is unmapped (i.e. it has never been written, or has been discarded), one represents an uncompressed block, and the other 14 states are used to indicate that the mapped data is compressed, and which of the compression slots in the compressed block this logical address maps to. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 2088 +++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/block-map.h | 225 ++++ 2 files changed, 2313 insertions(+) create mode 100644 drivers/md/dm-vdo/block-map.c create mode 100644 drivers/md/dm-vdo/block-map.h diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c new file mode 100644 index 000000000000..bc50564f7316 --- /dev/null +++ b/drivers/md/dm-vdo/block-map.c @@ -0,0 +1,2088 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "block-map.h" + +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "encodings.h" +#include "io-submitter.h" +#include "physical-zone.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +/** + * DOC: Block map eras + * + * The block map era, or maximum age, is used as follows: + * + * Each block map page, when dirty, records the earliest recovery journal block sequence number of + * the changes reflected in that dirty block. Sequence numbers are classified into eras: every + * @maximum_age sequence numbers, we switch to a new era. Block map pages are assigned to eras + * according to the sequence number they record. + * + * In the current (newest) era, block map pages are not written unless there is cache pressure. In + * the next oldest era, each time a new journal block is written 1/@maximum_age of the pages in + * this era are issued for write. In all older eras, pages are issued for write immediately. + */ + +struct page_descriptor { + root_count_t root_index; + height_t height; + page_number_t page_index; + slot_number_t slot; +} __packed; + +union page_key { + struct page_descriptor descriptor; + u64 key; +}; + +struct write_if_not_dirtied_context { + struct block_map_zone *zone; + u8 generation; +}; + +struct block_map_tree_segment { + struct tree_page *levels[VDO_BLOCK_MAP_TREE_HEIGHT]; +}; + +struct block_map_tree { + struct block_map_tree_segment *segments; +} block_map_tree; + +struct forest { + struct block_map *map; + size_t segments; + struct boundary *boundaries; + struct tree_page **pages; + struct block_map_tree trees[]; +}; + +struct cursor_level { + page_number_t page_index; + slot_number_t slot; +}; + +struct cursors; + +struct cursor { + struct waiter waiter; + struct block_map_tree *tree; + height_t height; + struct cursors *parent; + struct boundary boundary; + struct cursor_level levels[VDO_BLOCK_MAP_TREE_HEIGHT]; + struct pooled_vio *vio; +}; + +struct cursors { + struct block_map_zone *zone; + struct vio_pool *pool; + vdo_entry_callback_fn entry_callback; + struct vdo_completion *parent; + root_count_t active_roots; + struct cursor cursors[]; +}; + +/* Used to indicate that the page holding the location of a tree root has been "loaded". */ +static const physical_block_number_t VDO_INVALID_PBN = 0xFFFFFFFFFFFFFFFF; + +const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY = { + .mapping_state = VDO_MAPPING_STATE_UNMAPPED & 0x0F, + .pbn_high_nibble = 0, + .pbn_low_word = __cpu_to_le32(VDO_ZERO_BLOCK & UINT_MAX), +}; + +/** + * get_tree_page_by_index() - Get the tree page for a given height and page index. + * + * Return: The requested page. + */ +static struct tree_page * __must_check get_tree_page_by_index(struct forest *forest, + root_count_t root_index, + height_t height, + page_number_t page_index) +{ + page_number_t offset = 0; + size_t segment; + + for (segment = 0; segment < forest->segments; segment++) { + page_number_t border = forest->boundaries[segment].levels[height - 1]; + + if (page_index < border) { + struct block_map_tree *tree = &forest->trees[root_index]; + + return &(tree->segments[segment].levels[height - 1][page_index - offset]); + } + + offset = border; + } + + return NULL; +} + +/* Get the page referred to by the lock's tree slot at its current height. */ +static inline struct tree_page *get_tree_page(const struct block_map_zone *zone, + const struct tree_lock *lock) +{ + return get_tree_page_by_index(zone->block_map->forest, lock->root_index, + lock->height, + lock->tree_slots[lock->height].page_index); +} + +/** vdo_copy_valid_page() - Validate and copy a buffer to a page. */ +bool vdo_copy_valid_page(char *buffer, nonce_t nonce, + physical_block_number_t pbn, + struct block_map_page *page) +{ + struct block_map_page *loaded = (struct block_map_page *) buffer; + enum block_map_page_validity validity = + vdo_validate_block_map_page(loaded, nonce, pbn); + + if (validity == VDO_BLOCK_MAP_PAGE_VALID) { + memcpy(page, loaded, VDO_BLOCK_SIZE); + return true; + } + + if (validity == VDO_BLOCK_MAP_PAGE_BAD) { + uds_log_error_strerror(VDO_BAD_PAGE, + "Expected page %llu but got page %llu instead", + (unsigned long long) pbn, + (unsigned long long) vdo_get_block_map_page_pbn(loaded)); + } + + return false; +} + +/** + * in_cyclic_range() - Check whether the given value is between the lower and upper bounds, within + * a cyclic range of values from 0 to (modulus - 1). + * @lower: The lowest value to accept. + * @value: The value to check. + * @upper: The highest value to accept. + * @modulus: The size of the cyclic space, no more than 2^15. + * + * The value and both bounds must be smaller than the modulus. + * + * Return: true if the value is in range. + */ +static bool in_cyclic_range(u16 lower, u16 value, u16 upper, u16 modulus) +{ + if (value < lower) + value += modulus; + if (upper < lower) + upper += modulus; + return (value <= upper); +} + +/** + * is_not_older() - Check whether a generation is strictly older than some other generation in the + * context of a zone's current generation range. + * @zone: The zone in which to do the comparison. + * @a: The generation in question. + * @b: The generation to compare to. + * + * Return: true if generation @a is not strictly older than generation @b in the context of @zone + */ +static bool __must_check is_not_older(struct block_map_zone *zone, u8 a, u8 b) +{ + int result; + + result = ASSERT((in_cyclic_range(zone->oldest_generation, a, zone->generation, 1 << 8) && + in_cyclic_range(zone->oldest_generation, b, zone->generation, 1 << 8)), + "generation(s) %u, %u are out of range [%u, %u]", + a, b, zone->oldest_generation, zone->generation); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + return true; + } + + return in_cyclic_range(b, a, zone->generation, 1 << 8); +} + +static void release_generation(struct block_map_zone *zone, u8 generation) +{ + int result; + + result = ASSERT((zone->dirty_page_counts[generation] > 0), + "dirty page count underflow for generation %u", generation); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + return; + } + + zone->dirty_page_counts[generation]--; + while ((zone->dirty_page_counts[zone->oldest_generation] == 0) && + (zone->oldest_generation != zone->generation)) + zone->oldest_generation++; +} + +static void set_generation(struct block_map_zone *zone, struct tree_page *page, + u8 new_generation) +{ + u32 new_count; + int result; + bool decrement_old = vdo_is_waiting(&page->waiter); + u8 old_generation = page->generation; + + if (decrement_old && (old_generation == new_generation)) + return; + + page->generation = new_generation; + new_count = ++zone->dirty_page_counts[new_generation]; + result = ASSERT((new_count != 0), "dirty page count overflow for generation %u", + new_generation); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + return; + } + + if (decrement_old) + release_generation(zone, old_generation); +} + +static void write_page(struct tree_page *tree_page, struct pooled_vio *vio); + +/* Implements waiter_callback_fn */ +static void write_page_callback(struct waiter *waiter, void *context) +{ + write_page(container_of(waiter, struct tree_page, waiter), context); +} + +static void acquire_vio(struct waiter *waiter, struct block_map_zone *zone) +{ + waiter->callback = write_page_callback; + acquire_vio_from_pool(zone->vio_pool, waiter); +} + +/* Return: true if all possible generations were not already active */ +static bool attempt_increment(struct block_map_zone *zone) +{ + u8 generation = zone->generation + 1; + + if (zone->oldest_generation == generation) + return false; + + zone->generation = generation; + return true; +} + +/* Launches a flush if one is not already in progress. */ +static void enqueue_page(struct tree_page *page, struct block_map_zone *zone) +{ + if ((zone->flusher == NULL) && attempt_increment(zone)) { + zone->flusher = page; + acquire_vio(&page->waiter, zone); + return; + } + + vdo_enqueue_waiter(&zone->flush_waiters, &page->waiter); +} + +static void write_page_if_not_dirtied(struct waiter *waiter, void *context) +{ + struct tree_page *page = container_of(waiter, struct tree_page, waiter); + struct write_if_not_dirtied_context *write_context = context; + + if (page->generation == write_context->generation) { + acquire_vio(waiter, write_context->zone); + return; + } + + enqueue_page(page, write_context->zone); +} + +static void return_to_pool(struct block_map_zone *zone, struct pooled_vio *vio) +{ + return_vio_to_pool(zone->vio_pool, vio); + check_for_drain_complete(zone); +} + +/* This callback is registered in write_initialized_page(). */ +static void finish_page_write(struct vdo_completion *completion) +{ + bool dirty; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct tree_page *page = completion->parent; + struct block_map_zone *zone = pooled->context; + + vdo_release_recovery_journal_block_reference(zone->block_map->journal, + page->writing_recovery_lock, + VDO_ZONE_TYPE_LOGICAL, + zone->zone_number); + + dirty = (page->writing_generation != page->generation); + release_generation(zone, page->writing_generation); + page->writing = false; + + if (zone->flusher == page) { + struct write_if_not_dirtied_context context = { + .zone = zone, + .generation = page->writing_generation, + }; + + vdo_notify_all_waiters(&zone->flush_waiters, + write_page_if_not_dirtied, &context); + if (dirty && attempt_increment(zone)) { + write_page(page, pooled); + return; + } + + zone->flusher = NULL; + } + + if (dirty) { + enqueue_page(page, zone); + } else if ((zone->flusher == NULL) && vdo_has_waiters(&zone->flush_waiters) && + attempt_increment(zone)) { + zone->flusher = + container_of(vdo_dequeue_next_waiter(&zone->flush_waiters), + struct tree_page, waiter); + write_page(zone->flusher, pooled); + return; + } + + return_to_pool(zone, pooled); +} + +static void handle_write_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct block_map_zone *zone = pooled->context; + + vio_record_metadata_io_error(vio); + enter_zone_read_only_mode(zone, result); + return_to_pool(zone, pooled); +} + +static void write_page_endio(struct bio *bio); + +static void write_initialized_page(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct block_map_zone *zone = pooled->context; + struct tree_page *tree_page = completion->parent; + struct block_map_page *page = (struct block_map_page *) vio->data; + unsigned int operation = REQ_OP_WRITE | REQ_PRIO; + + /* + * Now that we know the page has been written at least once, mark the copy we are writing + * as initialized. + */ + page->header.initialized = true; + + if (zone->flusher == tree_page) + operation |= REQ_PREFLUSH; + + submit_metadata_vio(vio, vdo_get_block_map_page_pbn(page), + write_page_endio, handle_write_error, + operation); +} + +static void write_page_endio(struct bio *bio) +{ + struct pooled_vio *vio = bio->bi_private; + struct block_map_zone *zone = vio->context; + struct block_map_page *page = (struct block_map_page *) vio->vio.data; + + continue_vio_after_io(&vio->vio, + (page->header.initialized ? + finish_page_write : write_initialized_page), + zone->thread_id); +} + +static void write_page(struct tree_page *tree_page, struct pooled_vio *vio) +{ + struct vdo_completion *completion = &vio->vio.completion; + struct block_map_zone *zone = vio->context; + struct block_map_page *page = vdo_as_block_map_page(tree_page); + + if ((zone->flusher != tree_page) && + is_not_older(zone, tree_page->generation, zone->generation)) { + /* + * This page was re-dirtied after the last flush was issued, hence we need to do + * another flush. + */ + enqueue_page(tree_page, zone); + return_to_pool(zone, vio); + return; + } + + completion->parent = tree_page; + memcpy(vio->vio.data, tree_page->page_buffer, VDO_BLOCK_SIZE); + completion->callback_thread_id = zone->thread_id; + + tree_page->writing = true; + tree_page->writing_generation = tree_page->generation; + tree_page->writing_recovery_lock = tree_page->recovery_lock; + + /* Clear this now so that we know this page is not on any dirty list. */ + tree_page->recovery_lock = 0; + + /* + * We've already copied the page into the vio which will write it, so if it was not yet + * initialized, the first write will indicate that (for torn write protection). It is now + * safe to mark it as initialized in memory since if the write fails, the in memory state + * will become irrelevant. + */ + if (page->header.initialized) { + write_initialized_page(completion); + return; + } + + page->header.initialized = true; + submit_metadata_vio(&vio->vio, vdo_get_block_map_page_pbn(page), + write_page_endio, handle_write_error, + REQ_OP_WRITE | REQ_PRIO); +} + +/* Release a lock on a page which was being loaded or allocated. */ +static void release_page_lock(struct data_vio *data_vio, char *what) +{ + struct block_map_zone *zone; + struct tree_lock *lock_holder; + struct tree_lock *lock = &data_vio->tree_lock; + + ASSERT_LOG_ONLY(lock->locked, + "release of unlocked block map page %s for key %llu in tree %u", + what, (unsigned long long) lock->key, lock->root_index); + + zone = data_vio->logical.zone->block_map_zone; + lock_holder = vdo_int_map_remove(zone->loading_pages, lock->key); + ASSERT_LOG_ONLY((lock_holder == lock), + "block map page %s mismatch for key %llu in tree %u", + what, (unsigned long long) lock->key, lock->root_index); + lock->locked = false; +} + +static void finish_lookup(struct data_vio *data_vio, int result) +{ + data_vio->tree_lock.height = 0; + + --data_vio->logical.zone->block_map_zone->active_lookups; + + set_data_vio_logical_callback(data_vio, continue_data_vio_with_block_map_slot); + data_vio->vio.completion.error_handler = handle_data_vio_error; + continue_data_vio_with_error(data_vio, result); +} + +static void abort_lookup_for_waiter(struct waiter *waiter, void *context) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + int result = *((int *) context); + + if (!data_vio->write) { + if (result == VDO_NO_SPACE) + result = VDO_SUCCESS; + } else if (result != VDO_NO_SPACE) { + result = VDO_READ_ONLY; + } + + finish_lookup(data_vio, result); +} + +static void abort_lookup(struct data_vio *data_vio, int result, char *what) +{ + if (result != VDO_NO_SPACE) + enter_zone_read_only_mode(data_vio->logical.zone->block_map_zone, result); + + if (data_vio->tree_lock.locked) { + release_page_lock(data_vio, what); + vdo_notify_all_waiters(&data_vio->tree_lock.waiters, + abort_lookup_for_waiter, &result); + } + + finish_lookup(data_vio, result); +} + +static void abort_load(struct data_vio *data_vio, int result) +{ + abort_lookup(data_vio, result, "load"); +} + +static bool __must_check is_invalid_tree_entry(const struct vdo *vdo, + const struct data_location *mapping, + height_t height) +{ + if (!vdo_is_valid_location(mapping) || + vdo_is_state_compressed(mapping->state) || + (vdo_is_mapped_location(mapping) && (mapping->pbn == VDO_ZERO_BLOCK))) + return true; + + /* Roots aren't physical data blocks, so we can't check their PBNs. */ + if (height == VDO_BLOCK_MAP_TREE_HEIGHT) + return false; + + return !vdo_is_physical_data_block(vdo->depot, mapping->pbn); +} + +static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio); +static void allocate_block_map_page(struct block_map_zone *zone, + struct data_vio *data_vio); + +static void continue_with_loaded_page(struct data_vio *data_vio, + struct block_map_page *page) +{ + struct tree_lock *lock = &data_vio->tree_lock; + struct block_map_tree_slot slot = lock->tree_slots[lock->height]; + struct data_location mapping = + vdo_unpack_block_map_entry(&page->entries[slot.block_map_slot.slot]); + + if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { + uds_log_error_strerror(VDO_BAD_MAPPING, + "Invalid block map tree PBN: %llu with state %u for page index %u at height %u", + (unsigned long long) mapping.pbn, mapping.state, + lock->tree_slots[lock->height - 1].page_index, + lock->height - 1); + abort_load(data_vio, VDO_BAD_MAPPING); + return; + } + + if (!vdo_is_mapped_location(&mapping)) { + /* The page we need is unallocated */ + allocate_block_map_page(data_vio->logical.zone->block_map_zone, + data_vio); + return; + } + + lock->tree_slots[lock->height - 1].block_map_slot.pbn = mapping.pbn; + if (lock->height == 1) { + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + /* We know what page we need to load next */ + load_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); +} + +static void continue_load_for_waiter(struct waiter *waiter, void *context) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + + data_vio->tree_lock.height--; + continue_with_loaded_page(data_vio, context); +} + +static void finish_block_map_page_load(struct vdo_completion *completion) +{ + physical_block_number_t pbn; + struct tree_page *tree_page; + struct block_map_page *page; + nonce_t nonce; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = vio_as_pooled_vio(vio); + struct data_vio *data_vio = completion->parent; + struct block_map_zone *zone = pooled->context; + struct tree_lock *tree_lock = &data_vio->tree_lock; + + tree_lock->height--; + pbn = tree_lock->tree_slots[tree_lock->height].block_map_slot.pbn; + tree_page = get_tree_page(zone, tree_lock); + page = (struct block_map_page *) tree_page->page_buffer; + nonce = zone->block_map->nonce; + + if (!vdo_copy_valid_page(vio->data, nonce, pbn, page)) + vdo_format_block_map_page(page, nonce, pbn, false); + return_vio_to_pool(zone->vio_pool, pooled); + + /* Release our claim to the load and wake any waiters */ + release_page_lock(data_vio, "load"); + vdo_notify_all_waiters(&tree_lock->waiters, continue_load_for_waiter, page); + continue_with_loaded_page(data_vio, page); +} + +static void handle_io_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct data_vio *data_vio = completion->parent; + struct block_map_zone *zone = pooled->context; + + vio_record_metadata_io_error(vio); + return_vio_to_pool(zone->vio_pool, pooled); + abort_load(data_vio, result); +} + +static void load_page_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct data_vio *data_vio = vio->completion.parent; + + continue_vio_after_io(vio, finish_block_map_page_load, + data_vio->logical.zone->thread_id); +} + +static void load_page(struct waiter *waiter, void *context) +{ + struct pooled_vio *pooled = context; + struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct tree_lock *lock = &data_vio->tree_lock; + physical_block_number_t pbn = lock->tree_slots[lock->height - 1].block_map_slot.pbn; + + pooled->vio.completion.parent = data_vio; + submit_metadata_vio(&pooled->vio, pbn, load_page_endio, + handle_io_error, REQ_OP_READ | REQ_PRIO); +} + +/* + * If the page is already locked, queue up to wait for the lock to be released. If the lock is + * acquired, @data_vio->tree_lock.locked will be true. + */ +static int attempt_page_lock(struct block_map_zone *zone, struct data_vio *data_vio) +{ + int result; + struct tree_lock *lock_holder; + struct tree_lock *lock = &data_vio->tree_lock; + height_t height = lock->height; + struct block_map_tree_slot tree_slot = lock->tree_slots[height]; + union page_key key; + + key.descriptor = (struct page_descriptor) { + .root_index = lock->root_index, + .height = height, + .page_index = tree_slot.page_index, + .slot = tree_slot.block_map_slot.slot, + }; + lock->key = key.key; + + result = vdo_int_map_put(zone->loading_pages, lock->key, + lock, false, (void **) &lock_holder); + if (result != VDO_SUCCESS) + return result; + + if (lock_holder == NULL) { + /* We got the lock */ + data_vio->tree_lock.locked = true; + return VDO_SUCCESS; + } + + /* Someone else is loading or allocating the page we need */ + vdo_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter); + return VDO_SUCCESS; +} + +/* Load a block map tree page from disk, for the next level in the data vio tree lock. */ +static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio) +{ + int result; + + result = attempt_page_lock(zone, data_vio); + if (result != VDO_SUCCESS) { + abort_load(data_vio, result); + return; + } + + if (data_vio->tree_lock.locked) { + data_vio->waiter.callback = load_page; + acquire_vio_from_pool(zone->vio_pool, &data_vio->waiter); + } +} + +static void allocation_failure(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + if (vdo_requeue_completion_if_needed(completion, + data_vio->logical.zone->thread_id)) + return; + + abort_lookup(data_vio, completion->result, "allocation"); +} + +static void continue_allocation_for_waiter(struct waiter *waiter, void *context) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct tree_lock *tree_lock = &data_vio->tree_lock; + physical_block_number_t pbn = *((physical_block_number_t *) context); + + tree_lock->height--; + data_vio->tree_lock.tree_slots[tree_lock->height].block_map_slot.pbn = pbn; + + if (tree_lock->height == 0) { + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + allocate_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); +} + +/** expire_oldest_list() - Expire the oldest list. */ +static void expire_oldest_list(struct dirty_lists *dirty_lists) +{ + block_count_t i = dirty_lists->offset++; + + dirty_lists->oldest_period++; + if (!list_empty(&dirty_lists->eras[i][VDO_TREE_PAGE])) { + list_splice_tail_init(&dirty_lists->eras[i][VDO_TREE_PAGE], + &dirty_lists->expired[VDO_TREE_PAGE]); + } + + if (!list_empty(&dirty_lists->eras[i][VDO_CACHE_PAGE])) { + list_splice_tail_init(&dirty_lists->eras[i][VDO_CACHE_PAGE], + &dirty_lists->expired[VDO_CACHE_PAGE]); + } + + if (dirty_lists->offset == dirty_lists->maximum_age) + dirty_lists->offset = 0; +} + + +/** update_period() - Update the dirty_lists period if necessary. */ +static void update_period(struct dirty_lists *dirty, sequence_number_t period) +{ + while (dirty->next_period <= period) { + if ((dirty->next_period - dirty->oldest_period) == dirty->maximum_age) + expire_oldest_list(dirty); + dirty->next_period++; + } +} + +/** write_expired_elements() - Write out the expired list. */ +static void write_expired_elements(struct block_map_zone *zone) +{ + struct tree_page *page, *ttmp; + struct page_info *info, *ptmp; + struct list_head *expired; + u8 generation = zone->generation; + + expired = &zone->dirty_lists->expired[VDO_TREE_PAGE]; + list_for_each_entry_safe(page, ttmp, expired, entry) { + int result; + + list_del_init(&page->entry); + + result = ASSERT(!vdo_is_waiting(&page->waiter), + "Newly expired page not already waiting to write"); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + continue; + } + + set_generation(zone, page, generation); + if (!page->writing) + enqueue_page(page, zone); + } + + expired = &zone->dirty_lists->expired[VDO_CACHE_PAGE]; + list_for_each_entry_safe(info, ptmp, expired, state_entry) { + list_del_init(&info->state_entry); + schedule_page_save(info); + } + + save_pages(&zone->page_cache); +} + +/** + * add_to_dirty_lists() - Add an element to the dirty lists. + * @zone: The zone in which we are operating. + * @entry: The list entry of the element to add. + * @type: The type of page. + * @old_period: The period in which the element was previously dirtied, or 0 if it was not dirty. + * @new_period: The period in which the element has now been dirtied, or 0 if it does not hold a + * lock. + */ +static void add_to_dirty_lists(struct block_map_zone *zone, + struct list_head *entry, + enum block_map_page_type type, + sequence_number_t old_period, + sequence_number_t new_period) +{ + struct dirty_lists *dirty_lists = zone->dirty_lists; + + if ((old_period == new_period) || ((old_period != 0) && (old_period < new_period))) + return; + + if (new_period < dirty_lists->oldest_period) { + list_move_tail(entry, &dirty_lists->expired[type]); + } else { + update_period(dirty_lists, new_period); + list_move_tail(entry, + &dirty_lists->eras[new_period % dirty_lists->maximum_age][type]); + } + + write_expired_elements(zone); +} + +/* + * Record the allocation in the tree and wake any waiters now that the write lock has been + * released. + */ +static void finish_block_map_allocation(struct vdo_completion *completion) +{ + physical_block_number_t pbn; + struct tree_page *tree_page; + struct block_map_page *page; + sequence_number_t old_lock; + struct data_vio *data_vio = as_data_vio(completion); + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + struct tree_lock *tree_lock = &data_vio->tree_lock; + height_t height = tree_lock->height; + + assert_data_vio_in_logical_zone(data_vio); + + tree_page = get_tree_page(zone, tree_lock); + pbn = tree_lock->tree_slots[height - 1].block_map_slot.pbn; + + /* Record the allocation. */ + page = (struct block_map_page *) tree_page->page_buffer; + old_lock = tree_page->recovery_lock; + vdo_update_block_map_page(page, data_vio, pbn, + VDO_MAPPING_STATE_UNCOMPRESSED, + &tree_page->recovery_lock); + + if (vdo_is_waiting(&tree_page->waiter)) { + /* This page is waiting to be written out. */ + if (zone->flusher != tree_page) { + /* + * The outstanding flush won't cover the update we just made, + * so mark the page as needing another flush. + */ + set_generation(zone, tree_page, zone->generation); + } + } else { + /* Put the page on a dirty list */ + if (old_lock == 0) + INIT_LIST_HEAD(&tree_page->entry); + add_to_dirty_lists(zone, &tree_page->entry, VDO_TREE_PAGE, + old_lock, tree_page->recovery_lock); + } + + tree_lock->height--; + if (height > 1) { + /* Format the interior node we just allocated (in memory). */ + tree_page = get_tree_page(zone, tree_lock); + vdo_format_block_map_page(tree_page->page_buffer, + zone->block_map->nonce, + pbn, false); + } + + /* Release our claim to the allocation and wake any waiters */ + release_page_lock(data_vio, "allocation"); + vdo_notify_all_waiters(&tree_lock->waiters, continue_allocation_for_waiter, + &pbn); + if (tree_lock->height == 0) { + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + allocate_block_map_page(zone, data_vio); +} + +static void release_block_map_write_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + + release_data_vio_allocation_lock(data_vio, true); + launch_data_vio_logical_callback(data_vio, finish_block_map_allocation); +} + +/* + * Newly allocated block map pages are set to have to MAXIMUM_REFERENCES after they are journaled, + * to prevent deduplication against the block after we release the write lock on it, but before we + * write out the page. + */ +static void set_block_map_page_reference_count(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + + completion->callback = release_block_map_write_lock; + vdo_modify_reference_count(completion, &data_vio->increment_updater); +} + +static void journal_block_map_allocation(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_journal_zone(data_vio); + + set_data_vio_allocated_zone_callback(data_vio, + set_block_map_page_reference_count); + vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio); +} + +static void allocate_block(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct tree_lock *lock = &data_vio->tree_lock; + physical_block_number_t pbn; + + assert_data_vio_in_allocated_zone(data_vio); + + if (!vdo_allocate_block_in_zone(data_vio)) + return; + + pbn = data_vio->allocation.pbn; + lock->tree_slots[lock->height - 1].block_map_slot.pbn = pbn; + data_vio->increment_updater = (struct reference_updater) { + .operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING, + .increment = true, + .zpbn = { + .pbn = pbn, + .state = VDO_MAPPING_STATE_UNCOMPRESSED, + }, + .lock = data_vio->allocation.lock, + }; + + launch_data_vio_journal_callback(data_vio, journal_block_map_allocation); +} + +static void allocate_block_map_page(struct block_map_zone *zone, + struct data_vio *data_vio) +{ + int result; + + if (!data_vio->write || data_vio->is_trim) { + /* This is a pure read or a trim, so there's nothing left to do here. */ + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + result = attempt_page_lock(zone, data_vio); + if (result != VDO_SUCCESS) { + abort_lookup(data_vio, result, "allocation"); + return; + } + + if (!data_vio->tree_lock.locked) + return; + + data_vio_allocate_data_block(data_vio, VIO_BLOCK_MAP_WRITE_LOCK, + allocate_block, allocation_failure); +} + +/** + * vdo_find_block_map_slot() - Find the block map slot in which the block map entry for a data_vio + * resides and cache that result in the data_vio. + * + * All ancestors in the tree will be allocated or loaded, as needed. + */ +void vdo_find_block_map_slot(struct data_vio *data_vio) +{ + page_number_t page_index; + struct block_map_tree_slot tree_slot; + struct data_location mapping; + struct block_map_page *page = NULL; + struct tree_lock *lock = &data_vio->tree_lock; + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + + zone->active_lookups++; + if (vdo_is_state_draining(&zone->state)) { + finish_lookup(data_vio, VDO_SHUTTING_DOWN); + return; + } + + lock->tree_slots[0].block_map_slot.slot = + data_vio->logical.lbn % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + page_index = (lock->tree_slots[0].page_index / zone->block_map->root_count); + tree_slot = (struct block_map_tree_slot) { + .page_index = page_index / VDO_BLOCK_MAP_ENTRIES_PER_PAGE, + .block_map_slot = { + .pbn = 0, + .slot = page_index % VDO_BLOCK_MAP_ENTRIES_PER_PAGE, + }, + }; + + for (lock->height = 1; lock->height <= VDO_BLOCK_MAP_TREE_HEIGHT; lock->height++) { + physical_block_number_t pbn; + + lock->tree_slots[lock->height] = tree_slot; + page = (struct block_map_page *) (get_tree_page(zone, lock)->page_buffer); + pbn = vdo_get_block_map_page_pbn(page); + if (pbn != VDO_ZERO_BLOCK) { + lock->tree_slots[lock->height].block_map_slot.pbn = pbn; + break; + } + + /* Calculate the index and slot for the next level. */ + tree_slot.block_map_slot.slot = + tree_slot.page_index % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + tree_slot.page_index = tree_slot.page_index / VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + } + + /* The page at this height has been allocated and loaded. */ + mapping = vdo_unpack_block_map_entry(&page->entries[tree_slot.block_map_slot.slot]); + if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { + uds_log_error_strerror(VDO_BAD_MAPPING, + "Invalid block map tree PBN: %llu with state %u for page index %u at height %u", + (unsigned long long) mapping.pbn, mapping.state, + lock->tree_slots[lock->height - 1].page_index, + lock->height - 1); + abort_load(data_vio, VDO_BAD_MAPPING); + return; + } + + if (!vdo_is_mapped_location(&mapping)) { + /* The page we want one level down has not been allocated, so allocate it. */ + allocate_block_map_page(zone, data_vio); + return; + } + + lock->tree_slots[lock->height - 1].block_map_slot.pbn = mapping.pbn; + if (lock->height == 1) { + /* This is the ultimate block map page, so we're done */ + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + /* We know what page we need to load. */ + load_block_map_page(zone, data_vio); +} + +/* + * Find the PBN of a leaf block map page. This method may only be used after all allocated tree + * pages have been loaded, otherwise, it may give the wrong answer (0). + */ +physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, + page_number_t page_number) +{ + struct data_location mapping; + struct tree_page *tree_page; + struct block_map_page *page; + root_count_t root_index = page_number % map->root_count; + page_number_t page_index = page_number / map->root_count; + slot_number_t slot = page_index % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + page_index /= VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + tree_page = get_tree_page_by_index(map->forest, root_index, 1, page_index); + page = (struct block_map_page *) tree_page->page_buffer; + if (!page->header.initialized) + return VDO_ZERO_BLOCK; + + mapping = vdo_unpack_block_map_entry(&page->entries[slot]); + if (!vdo_is_valid_location(&mapping) || vdo_is_state_compressed(mapping.state)) + return VDO_ZERO_BLOCK; + return mapping.pbn; +} + +/* + * Write a tree page or indicate that it has been re-dirtied if it is already being written. This + * method is used when correcting errors in the tree during read-only rebuild. + */ +void vdo_write_tree_page(struct tree_page *page, struct block_map_zone *zone) +{ + bool waiting = vdo_is_waiting(&page->waiter); + + if (waiting && (zone->flusher == page)) + return; + + set_generation(zone, page, zone->generation); + if (waiting || page->writing) + return; + + enqueue_page(page, zone); +} + +static int make_segment(struct forest *old_forest, block_count_t new_pages, + struct boundary *new_boundary, struct forest *forest) +{ + size_t index = (old_forest == NULL) ? 0 : old_forest->segments; + struct tree_page *page_ptr; + page_count_t segment_sizes[VDO_BLOCK_MAP_TREE_HEIGHT]; + height_t height; + root_count_t root; + int result; + + forest->segments = index + 1; + + result = uds_allocate(forest->segments, struct boundary, + "forest boundary array", &forest->boundaries); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(forest->segments, struct tree_page *, + "forest page pointers", &forest->pages); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(new_pages, struct tree_page, + "new forest pages", &forest->pages[index]); + if (result != VDO_SUCCESS) + return result; + + if (index > 0) { + memcpy(forest->boundaries, old_forest->boundaries, + index * sizeof(struct boundary)); + memcpy(forest->pages, old_forest->pages, + index * sizeof(struct tree_page *)); + } + + memcpy(&(forest->boundaries[index]), new_boundary, sizeof(struct boundary)); + + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT; height++) { + segment_sizes[height] = new_boundary->levels[height]; + if (index > 0) + segment_sizes[height] -= old_forest->boundaries[index - 1].levels[height]; + } + + page_ptr = forest->pages[index]; + for (root = 0; root < forest->map->root_count; root++) { + struct block_map_tree_segment *segment; + struct block_map_tree *tree = &(forest->trees[root]); + height_t height; + + int result = uds_allocate(forest->segments, + struct block_map_tree_segment, + "tree root segments", &tree->segments); + if (result != VDO_SUCCESS) + return result; + + if (index > 0) { + memcpy(tree->segments, old_forest->trees[root].segments, + index * sizeof(struct block_map_tree_segment)); + } + + segment = &(tree->segments[index]); + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT; height++) { + if (segment_sizes[height] == 0) + continue; + + segment->levels[height] = page_ptr; + if (height == (VDO_BLOCK_MAP_TREE_HEIGHT - 1)) { + /* Record the root. */ + struct block_map_page *page = + vdo_format_block_map_page(page_ptr->page_buffer, + forest->map->nonce, + VDO_INVALID_PBN, true); + page->entries[0] = + vdo_pack_block_map_entry(forest->map->root_origin + root, + VDO_MAPPING_STATE_UNCOMPRESSED); + } + page_ptr += segment_sizes[height]; + } + } + + return VDO_SUCCESS; +} + +static void deforest(struct forest *forest, size_t first_page_segment) +{ + root_count_t root; + + if (forest->pages != NULL) { + size_t segment; + + for (segment = first_page_segment; segment < forest->segments; segment++) + uds_free(forest->pages[segment]); + uds_free(forest->pages); + } + + for (root = 0; root < forest->map->root_count; root++) + uds_free(forest->trees[root].segments); + + uds_free(forest->boundaries); + uds_free(forest); +} + +/** + * make_forest() - Make a collection of trees for a block_map, expanding the existing forest if + * there is one. + * @entries: The number of entries the block map will hold. + * + * Return: VDO_SUCCESS or an error. + */ +static int make_forest(struct block_map *map, block_count_t entries) +{ + struct forest *forest, *old_forest = map->forest; + struct boundary new_boundary, *old_boundary = NULL; + block_count_t new_pages; + int result; + + if (old_forest != NULL) + old_boundary = &(old_forest->boundaries[old_forest->segments - 1]); + + new_pages = vdo_compute_new_forest_pages(map->root_count, old_boundary, + entries, &new_boundary); + if (new_pages == 0) { + map->next_entry_count = entries; + return VDO_SUCCESS; + } + + result = uds_allocate_extended(struct forest, map->root_count, + struct block_map_tree, __func__, + &forest); + if (result != VDO_SUCCESS) + return result; + + forest->map = map; + result = make_segment(old_forest, new_pages, &new_boundary, forest); + if (result != VDO_SUCCESS) { + deforest(forest, forest->segments - 1); + return result; + } + + map->next_forest = forest; + map->next_entry_count = entries; + return VDO_SUCCESS; +} + +/** + * replace_forest() - Replace a block_map's forest with the already-prepared larger forest. + */ +static void replace_forest(struct block_map *map) +{ + if (map->next_forest != NULL) { + if (map->forest != NULL) + deforest(map->forest, map->forest->segments); + map->forest = uds_forget(map->next_forest); + } + + map->entry_count = map->next_entry_count; + map->next_entry_count = 0; +} + +/** + * finish_cursor() - Finish the traversal of a single tree. If it was the last cursor, finish the + * traversal. + */ +static void finish_cursor(struct cursor *cursor) +{ + struct cursors *cursors = cursor->parent; + struct vdo_completion *parent = cursors->parent; + + return_vio_to_pool(cursors->pool, uds_forget(cursor->vio)); + if (--cursors->active_roots > 0) + return; + + uds_free(cursors); + + vdo_finish_completion(parent); +} + +static void traverse(struct cursor *cursor); + +/** + * continue_traversal() - Continue traversing a block map tree. + * @completion: The VIO doing a read or write. + */ +static void continue_traversal(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + traverse(completion->parent); +} + +/** + * finish_traversal_load() - Continue traversing a block map tree now that a page has been loaded. + * @completion: The VIO doing the read. + */ +static void finish_traversal_load(struct vdo_completion *completion) +{ + struct cursor *cursor = completion->parent; + height_t height = cursor->height; + struct cursor_level *level = &cursor->levels[height]; + struct tree_page *tree_page = + &(cursor->tree->segments[0].levels[height][level->page_index]); + struct block_map_page *page = (struct block_map_page *) tree_page->page_buffer; + + vdo_copy_valid_page(cursor->vio->vio.data, + cursor->parent->zone->block_map->nonce, + pbn_from_vio_bio(cursor->vio->vio.bio), page); + traverse(cursor); +} + +static void traversal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct cursor *cursor = vio->completion.parent; + + continue_vio_after_io(vio, finish_traversal_load, + cursor->parent->zone->thread_id); +} + +/** + * traverse() - Traverse a single block map tree. + * + * This is the recursive heart of the traversal process. + */ +static void traverse(struct cursor *cursor) +{ + for (; cursor->height < VDO_BLOCK_MAP_TREE_HEIGHT; cursor->height++) { + height_t height = cursor->height; + struct cursor_level *level = &cursor->levels[height]; + struct tree_page *tree_page = + &(cursor->tree->segments[0].levels[height][level->page_index]); + struct block_map_page *page = (struct block_map_page *) tree_page->page_buffer; + + if (!page->header.initialized) + continue; + + for (; level->slot < VDO_BLOCK_MAP_ENTRIES_PER_PAGE; level->slot++) { + struct cursor_level *next_level; + page_number_t entry_index = + (VDO_BLOCK_MAP_ENTRIES_PER_PAGE * level->page_index) + level->slot; + struct data_location location = + vdo_unpack_block_map_entry(&page->entries[level->slot]); + + if (!vdo_is_valid_location(&location)) { + /* This entry is invalid, so remove it from the page. */ + page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_write_tree_page(tree_page, cursor->parent->zone); + continue; + } + + if (!vdo_is_mapped_location(&location)) + continue; + + /* Erase mapped entries past the end of the logical space. */ + if (entry_index >= cursor->boundary.levels[height]) { + page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_write_tree_page(tree_page, cursor->parent->zone); + continue; + } + + if (cursor->height < VDO_BLOCK_MAP_TREE_HEIGHT - 1) { + int result = cursor->parent->entry_callback(location.pbn, + cursor->parent->parent); + + if (result != VDO_SUCCESS) { + page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_write_tree_page(tree_page, + cursor->parent->zone); + continue; + } + } + + if (cursor->height == 0) + continue; + + cursor->height--; + next_level = &cursor->levels[cursor->height]; + next_level->page_index = entry_index; + next_level->slot = 0; + level->slot++; + submit_metadata_vio(&cursor->vio->vio, location.pbn, + traversal_endio, continue_traversal, + REQ_OP_READ | REQ_PRIO); + return; + } + } + + finish_cursor(cursor); +} + +/** + * launch_cursor() - Start traversing a single block map tree now that the cursor has a VIO with + * which to load pages. + * @context: The pooled_vio just acquired. + * + * Implements waiter_callback_fn. + */ +static void launch_cursor(struct waiter *waiter, void *context) +{ + struct cursor *cursor = container_of(waiter, struct cursor, waiter); + struct pooled_vio *pooled = context; + + cursor->vio = pooled; + pooled->vio.completion.parent = cursor; + pooled->vio.completion.callback_thread_id = cursor->parent->zone->thread_id; + traverse(cursor); +} + +/** + * compute_boundary() - Compute the number of pages used at each level of the given root's tree. + * + * Return: The list of page counts as a boundary structure. + */ +static struct boundary compute_boundary(struct block_map *map, root_count_t root_index) +{ + struct boundary boundary; + height_t height; + page_count_t leaf_pages = vdo_compute_block_map_page_count(map->entry_count); + /* + * Compute the leaf pages for this root. If the number of leaf pages does not distribute + * evenly, we must determine if this root gets an extra page. Extra pages are assigned to + * roots starting from tree 0. + */ + page_count_t last_tree_root = (leaf_pages - 1) % map->root_count; + page_count_t level_pages = leaf_pages / map->root_count; + + if (root_index <= last_tree_root) + level_pages++; + + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT - 1; height++) { + boundary.levels[height] = level_pages; + level_pages = DIV_ROUND_UP(level_pages, VDO_BLOCK_MAP_ENTRIES_PER_PAGE); + } + + /* The root node always exists, even if the root is otherwise unused. */ + boundary.levels[VDO_BLOCK_MAP_TREE_HEIGHT - 1] = 1; + + return boundary; +} + +/** + * vdo_traverse_forest() - Walk the entire forest of a block map. + * @callback: A function to call with the pbn of each allocated node in the forest. + * @parent: The completion to notify on each traversed PBN, and when the traversal is complete. + */ +void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, + struct vdo_completion *parent) +{ + root_count_t root; + struct cursors *cursors; + int result; + + result = uds_allocate_extended(struct cursors, map->root_count, + struct cursor, __func__, &cursors); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + cursors->zone = &map->zones[0]; + cursors->pool = cursors->zone->vio_pool; + cursors->entry_callback = callback; + cursors->parent = parent; + cursors->active_roots = map->root_count; + for (root = 0; root < map->root_count; root++) { + struct cursor *cursor = &cursors->cursors[root]; + + *cursor = (struct cursor) { + .tree = &map->forest->trees[root], + .height = VDO_BLOCK_MAP_TREE_HEIGHT - 1, + .parent = cursors, + .boundary = compute_boundary(map, root), + }; + + cursor->waiter.callback = launch_cursor; + acquire_vio_from_pool(cursors->pool, &cursor->waiter); + }; +} + +/** + * initialize_block_map_zone() - Initialize the per-zone portions of the block map. + * @maximum_age: The number of journal blocks before a dirtied page is considered old and must be + * written out. + */ +static int __must_check initialize_block_map_zone(struct block_map *map, + zone_count_t zone_number, + struct vdo *vdo, + page_count_t cache_size, + block_count_t maximum_age) +{ + int result; + block_count_t i; + struct block_map_zone *zone = &map->zones[zone_number]; + + BUILD_BUG_ON(sizeof(struct page_descriptor) != sizeof(u64)); + + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.logical_threads[zone_number]; + zone->block_map = map; + + result = uds_allocate_extended(struct dirty_lists, maximum_age, + dirty_era_t, __func__, + &zone->dirty_lists); + if (result != VDO_SUCCESS) + return result; + + zone->dirty_lists->maximum_age = maximum_age; + INIT_LIST_HEAD(&zone->dirty_lists->expired[VDO_TREE_PAGE]); + INIT_LIST_HEAD(&zone->dirty_lists->expired[VDO_CACHE_PAGE]); + + for (i = 0; i < maximum_age; i++) { + INIT_LIST_HEAD(&zone->dirty_lists->eras[i][VDO_TREE_PAGE]); + INIT_LIST_HEAD(&zone->dirty_lists->eras[i][VDO_CACHE_PAGE]); + } + + result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->loading_pages); + if (result != VDO_SUCCESS) + return result; + + result = make_vio_pool(vdo, BLOCK_MAP_VIO_POOL_SIZE, + zone->thread_id, VIO_TYPE_BLOCK_MAP_INTERIOR, + VIO_PRIORITY_METADATA, zone, &zone->vio_pool); + if (result != VDO_SUCCESS) + return result; + + vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + + zone->page_cache.zone = zone; + zone->page_cache.vdo = vdo; + zone->page_cache.page_count = cache_size / map->zone_count; + zone->page_cache.stats.free_pages = zone->page_cache.page_count; + + result = allocate_cache_components(&zone->page_cache); + if (result != VDO_SUCCESS) + return result; + + /* initialize empty circular queues */ + INIT_LIST_HEAD(&zone->page_cache.lru_list); + INIT_LIST_HEAD(&zone->page_cache.outgoing_list); + + return VDO_SUCCESS; +} + +/* Implements vdo_zone_thread_getter_fn */ +static thread_id_t get_block_map_zone_thread_id(void *context, zone_count_t zone_number) +{ + struct block_map *map = context; + + return map->zones[zone_number].thread_id; +} + +/* Implements vdo_action_preamble_fn */ +static void prepare_for_era_advance(void *context, struct vdo_completion *parent) +{ + struct block_map *map = context; + + map->current_era_point = map->pending_era_point; + vdo_finish_completion(parent); +} + +/* Implements vdo_zone_action_fn */ +static void advance_block_map_zone_era(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct block_map *map = context; + struct block_map_zone *zone = &map->zones[zone_number]; + + update_period(zone->dirty_lists, map->current_era_point); + write_expired_elements(zone); + vdo_finish_completion(parent); +} + +/* + * Schedule an era advance if necessary. This method should not be called directly. Rather, call + * vdo_schedule_default_action() on the block map's action manager. + * + * Implements vdo_action_scheduler_fn. + */ +static bool schedule_era_advance(void *context) +{ + struct block_map *map = context; + + if (map->current_era_point == map->pending_era_point) + return false; + + return vdo_schedule_action(map->action_manager, prepare_for_era_advance, + advance_block_map_zone_era, NULL, NULL); +} + +static void uninitialize_block_map_zone(struct block_map_zone *zone) +{ + struct vdo_page_cache *cache = &zone->page_cache; + + uds_free(uds_forget(zone->dirty_lists)); + free_vio_pool(uds_forget(zone->vio_pool)); + vdo_free_int_map(uds_forget(zone->loading_pages)); + if (cache->infos != NULL) { + struct page_info *info; + + for (info = cache->infos; info < cache->infos + cache->page_count; info++) + free_vio(uds_forget(info->vio)); + } + + vdo_free_int_map(uds_forget(cache->page_map)); + uds_free(uds_forget(cache->infos)); + uds_free(uds_forget(cache->pages)); +} + +void vdo_free_block_map(struct block_map *map) +{ + zone_count_t zone; + + if (map == NULL) + return; + + for (zone = 0; zone < map->zone_count; zone++) + uninitialize_block_map_zone(&map->zones[zone]); + + vdo_abandon_block_map_growth(map); + if (map->forest != NULL) + deforest(uds_forget(map->forest), 0); + uds_free(uds_forget(map->action_manager)); + uds_free(map); +} + +/* @journal may be NULL. */ +int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical_blocks, + struct vdo *vdo, struct recovery_journal *journal, + nonce_t nonce, page_count_t cache_size, block_count_t maximum_age, + struct block_map **map_ptr) +{ + struct block_map *map; + int result; + zone_count_t zone = 0; + + BUILD_BUG_ON(VDO_BLOCK_MAP_ENTRIES_PER_PAGE != + ((VDO_BLOCK_SIZE - sizeof(struct block_map_page)) / + sizeof(struct block_map_entry))); + result = ASSERT(cache_size > 0, "block map cache size is specified"); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate_extended(struct block_map, + vdo->thread_config.logical_zone_count, + struct block_map_zone, __func__, &map); + if (result != UDS_SUCCESS) + return result; + + map->vdo = vdo; + map->root_origin = state.root_origin; + map->root_count = state.root_count; + map->entry_count = logical_blocks; + map->journal = journal; + map->nonce = nonce; + + result = make_forest(map, map->entry_count); + if (result != VDO_SUCCESS) { + vdo_free_block_map(map); + return result; + } + + replace_forest(map); + + map->zone_count = vdo->thread_config.logical_zone_count; + for (zone = 0; zone < map->zone_count; zone++) { + result = initialize_block_map_zone(map, zone, vdo, cache_size, + maximum_age); + if (result != VDO_SUCCESS) { + vdo_free_block_map(map); + return result; + } + } + + result = vdo_make_action_manager(map->zone_count, get_block_map_zone_thread_id, + vdo_get_recovery_journal_thread_id(journal), + map, schedule_era_advance, vdo, + &map->action_manager); + if (result != VDO_SUCCESS) { + vdo_free_block_map(map); + return result; + } + + *map_ptr = map; + return VDO_SUCCESS; +} + +struct block_map_state_2_0 vdo_record_block_map(const struct block_map *map) +{ + return (struct block_map_state_2_0) { + .flat_page_origin = VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + /* This is the flat page count, which has turned out to always be 0. */ + .flat_page_count = 0, + .root_origin = map->root_origin, + .root_count = map->root_count, + }; +} + +/* The block map needs to know the journals' sequence number to initialize the eras. */ +void vdo_initialize_block_map_from_journal(struct block_map *map, + struct recovery_journal *journal) +{ + zone_count_t z = 0; + + map->current_era_point = vdo_get_recovery_journal_current_sequence_number(journal); + map->pending_era_point = map->current_era_point; + + for (z = 0; z < map->zone_count; z++) { + struct dirty_lists *dirty_lists = map->zones[z].dirty_lists; + + ASSERT_LOG_ONLY(dirty_lists->next_period == 0, "current period not set"); + dirty_lists->oldest_period = map->current_era_point; + dirty_lists->next_period = map->current_era_point + 1; + dirty_lists->offset = map->current_era_point % dirty_lists->maximum_age; + } +} + +/* Compute the logical zone for the LBN of a data vio. */ +zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio) +{ + struct block_map *map = vdo_from_data_vio(data_vio)->block_map; + struct tree_lock *tree_lock = &data_vio->tree_lock; + page_number_t page_number = data_vio->logical.lbn / VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + tree_lock->tree_slots[0].page_index = page_number; + tree_lock->root_index = page_number % map->root_count; + return (tree_lock->root_index % map->zone_count); +} + +void vdo_advance_block_map_era(struct block_map *map, + sequence_number_t recovery_block_number) +{ + if (map == NULL) + return; + + map->pending_era_point = recovery_block_number; + vdo_schedule_default_action(map->action_manager); +} + +/* Implements vdo_admin_initiator_fn */ +static void initiate_drain(struct admin_state *state) +{ + struct block_map_zone *zone = container_of(state, struct block_map_zone, state); + + ASSERT_LOG_ONLY((zone->active_lookups == 0), + "%s() called with no active lookups", __func__); + + if (!vdo_is_state_suspending(state)) { + while (zone->dirty_lists->oldest_period < zone->dirty_lists->next_period) + expire_oldest_list(zone->dirty_lists); + write_expired_elements(zone); + } + + check_for_drain_complete(zone); +} + +/* Implements vdo_zone_action_fn. */ +static void drain_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct block_map *map = context; + struct block_map_zone *zone = &map->zones[zone_number]; + + vdo_start_draining(&zone->state, + vdo_get_current_manager_operation(map->action_manager), + parent, initiate_drain); +} + +void vdo_drain_block_map(struct block_map *map, const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + vdo_schedule_operation(map->action_manager, operation, NULL, drain_zone, NULL, + parent); +} + +/* Implements vdo_zone_action_fn. */ +static void resume_block_map_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct block_map *map = context; + struct block_map_zone *zone = &map->zones[zone_number]; + + vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); +} + +void vdo_resume_block_map(struct block_map *map, struct vdo_completion *parent) +{ + vdo_schedule_operation(map->action_manager, VDO_ADMIN_STATE_RESUMING, + NULL, resume_block_map_zone, NULL, parent); +} + +/* Allocate an expanded collection of trees, for a future growth. */ +int vdo_prepare_to_grow_block_map(struct block_map *map, + block_count_t new_logical_blocks) +{ + if (map->next_entry_count == new_logical_blocks) + return VDO_SUCCESS; + + if (map->next_entry_count > 0) + vdo_abandon_block_map_growth(map); + + if (new_logical_blocks < map->entry_count) { + map->next_entry_count = map->entry_count; + return VDO_SUCCESS; + } + + return make_forest(map, new_logical_blocks); +} + +/* Implements vdo_action_preamble_fn */ +static void grow_forest(void *context, struct vdo_completion *completion) +{ + replace_forest(context); + vdo_finish_completion(completion); +} + +/* Requires vdo_prepare_to_grow_block_map() to have been previously called. */ +void vdo_grow_block_map(struct block_map *map, struct vdo_completion *parent) +{ + vdo_schedule_operation(map->action_manager, + VDO_ADMIN_STATE_SUSPENDED_OPERATION, + grow_forest, NULL, NULL, parent); +} + +void vdo_abandon_block_map_growth(struct block_map *map) +{ + struct forest *forest = uds_forget(map->next_forest); + + if (forest != NULL) + deforest(forest, forest->segments - 1); + + map->next_entry_count = 0; +} + +/* Release the page completion and then continue the requester. */ +static inline void finish_processing_page(struct vdo_completion *completion, int result) +{ + struct vdo_completion *parent = completion->parent; + + vdo_release_page_completion(completion); + vdo_continue_completion(parent, result); +} + +static void handle_page_error(struct vdo_completion *completion) +{ + finish_processing_page(completion, completion->result); +} + +/* Fetch the mapping page for a block map update, and call the provided handler when fetched. */ +static void fetch_mapping_page(struct data_vio *data_vio, bool modifiable, + vdo_action_fn action) +{ + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + + if (vdo_is_state_draining(&zone->state)) { + continue_data_vio_with_error(data_vio, VDO_SHUTTING_DOWN); + return; + } + + vdo_get_page(&data_vio->page_completion, zone, + data_vio->tree_lock.tree_slots[0].block_map_slot.pbn, + modifiable, &data_vio->vio.completion, + action, handle_page_error, false); +} + +/** + * clear_mapped_location() - Clear a data_vio's mapped block location, setting it to be unmapped. + * + * This indicates the block map entry for the logical block is either unmapped or corrupted. + */ +static void clear_mapped_location(struct data_vio *data_vio) +{ + data_vio->mapped = (struct zoned_pbn) { + .state = VDO_MAPPING_STATE_UNMAPPED, + }; +} + +/** + * set_mapped_location() - Decode and validate a block map entry, and set the mapped location of a + * data_vio. + * + * Return: VDO_SUCCESS or VDO_BAD_MAPPING if the map entry is invalid or an error code for any + * other failure + */ +static int __must_check set_mapped_location(struct data_vio *data_vio, + const struct block_map_entry *entry) +{ + /* Unpack the PBN for logging purposes even if the entry is invalid. */ + struct data_location mapped = vdo_unpack_block_map_entry(entry); + + if (vdo_is_valid_location(&mapped)) { + int result; + + result = vdo_get_physical_zone(vdo_from_data_vio(data_vio), + mapped.pbn, &data_vio->mapped.zone); + if (result == VDO_SUCCESS) { + data_vio->mapped.pbn = mapped.pbn; + data_vio->mapped.state = mapped.state; + return VDO_SUCCESS; + } + + /* + * Return all errors not specifically known to be errors from validating the + * location. + */ + if ((result != VDO_OUT_OF_RANGE) && (result != VDO_BAD_MAPPING)) + return result; + } + + /* + * Log the corruption even if we wind up ignoring it for write VIOs, converting all cases + * to VDO_BAD_MAPPING. + */ + uds_log_error_strerror(VDO_BAD_MAPPING, + "PBN %llu with state %u read from the block map was invalid", + (unsigned long long) mapped.pbn, mapped.state); + + /* + * A read VIO has no option but to report the bad mapping--reading zeros would be hiding + * known data loss. + */ + if (!data_vio->write) + return VDO_BAD_MAPPING; + + /* + * A write VIO only reads this mapping to decref the old block. Treat this as an unmapped + * entry rather than fail the write. + */ + clear_mapped_location(data_vio); + return VDO_SUCCESS; +} + +/* This callback is registered in vdo_get_mapped_block(). */ +static void get_mapping_from_fetched_page(struct vdo_completion *completion) +{ + int result; + struct vdo_page_completion *vpc = as_vdo_page_completion(completion); + const struct block_map_page *page; + const struct block_map_entry *entry; + struct data_vio *data_vio = as_data_vio(completion->parent); + struct block_map_tree_slot *tree_slot; + + if (completion->result != VDO_SUCCESS) { + finish_processing_page(completion, completion->result); + return; + } + + result = validate_completed_page(vpc, false); + if (result != VDO_SUCCESS) { + finish_processing_page(completion, result); + return; + } + + page = (const struct block_map_page *) get_page_buffer(vpc->info); + tree_slot = &data_vio->tree_lock.tree_slots[0]; + entry = &page->entries[tree_slot->block_map_slot.slot]; + + result = set_mapped_location(data_vio, entry); + finish_processing_page(completion, result); +} + +void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio, + physical_block_number_t pbn, + enum block_mapping_state mapping_state, + sequence_number_t *recovery_lock) +{ + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + struct block_map *block_map = zone->block_map; + struct recovery_journal *journal = block_map->journal; + sequence_number_t old_locked, new_locked; + struct tree_lock *tree_lock = &data_vio->tree_lock; + + /* Encode the new mapping. */ + page->entries[tree_lock->tree_slots[tree_lock->height].block_map_slot.slot] = + vdo_pack_block_map_entry(pbn, mapping_state); + + /* Adjust references on the recovery journal blocks. */ + old_locked = *recovery_lock; + new_locked = data_vio->recovery_sequence_number; + + if ((old_locked == 0) || (old_locked > new_locked)) { + vdo_acquire_recovery_journal_block_reference(journal, new_locked, + VDO_ZONE_TYPE_LOGICAL, + zone->zone_number); + + if (old_locked > 0) { + vdo_release_recovery_journal_block_reference(journal, old_locked, + VDO_ZONE_TYPE_LOGICAL, + zone->zone_number); + } + + *recovery_lock = new_locked; + } + + /* + * FIXME: explain this more + * Release the transferred lock from the data_vio. + */ + vdo_release_journal_entry_lock(journal, new_locked); + data_vio->recovery_sequence_number = 0; +} + +static void put_mapping_in_fetched_page(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion->parent); + sequence_number_t old_lock; + struct vdo_page_completion *vpc; + struct page_info *info; + int result; + + if (completion->result != VDO_SUCCESS) { + finish_processing_page(completion, completion->result); + return; + } + + vpc = as_vdo_page_completion(completion); + result = validate_completed_page(vpc, true); + if (result != VDO_SUCCESS) { + finish_processing_page(completion, result); + return; + } + + info = vpc->info; + old_lock = info->recovery_lock; + vdo_update_block_map_page((struct block_map_page *) get_page_buffer(info), + data_vio, data_vio->new_mapped.pbn, + data_vio->new_mapped.state, &info->recovery_lock); + set_info_state(info, PS_DIRTY); + add_to_dirty_lists(info->cache->zone, &info->state_entry, + VDO_CACHE_PAGE, old_lock, info->recovery_lock); + finish_processing_page(completion, VDO_SUCCESS); +} + +/* Read a stored block mapping into a data_vio. */ +void vdo_get_mapped_block(struct data_vio *data_vio) +{ + if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) { + /* + * We know that the block map page for this LBN has not been allocated, so the + * block must be unmapped. + */ + clear_mapped_location(data_vio); + continue_data_vio(data_vio); + return; + } + + fetch_mapping_page(data_vio, false, get_mapping_from_fetched_page); +} + +/* Update a stored block mapping to reflect a data_vio's new mapping. */ +void vdo_put_mapped_block(struct data_vio *data_vio) +{ + fetch_mapping_page(data_vio, true, put_mapping_in_fetched_page); +} + +struct block_map_statistics vdo_get_block_map_statistics(struct block_map *map) +{ + zone_count_t zone = 0; + struct block_map_statistics totals; + + memset(&totals, 0, sizeof(struct block_map_statistics)); + for (zone = 0; zone < map->zone_count; zone++) { + const struct block_map_statistics *stats = + &(map->zones[zone].page_cache.stats); + + totals.dirty_pages += READ_ONCE(stats->dirty_pages); + totals.clean_pages += READ_ONCE(stats->clean_pages); + totals.free_pages += READ_ONCE(stats->free_pages); + totals.failed_pages += READ_ONCE(stats->failed_pages); + totals.incoming_pages += READ_ONCE(stats->incoming_pages); + totals.outgoing_pages += READ_ONCE(stats->outgoing_pages); + totals.cache_pressure += READ_ONCE(stats->cache_pressure); + totals.read_count += READ_ONCE(stats->read_count); + totals.write_count += READ_ONCE(stats->write_count); + totals.failed_reads += READ_ONCE(stats->failed_reads); + totals.failed_writes += READ_ONCE(stats->failed_writes); + totals.reclaimed += READ_ONCE(stats->reclaimed); + totals.read_outgoing += READ_ONCE(stats->read_outgoing); + totals.found_in_cache += READ_ONCE(stats->found_in_cache); + totals.discard_required += READ_ONCE(stats->discard_required); + totals.wait_for_page += READ_ONCE(stats->wait_for_page); + totals.fetch_required += READ_ONCE(stats->fetch_required); + totals.pages_loaded += READ_ONCE(stats->pages_loaded); + totals.pages_saved += READ_ONCE(stats->pages_saved); + totals.flush_count += READ_ONCE(stats->flush_count); + } + + return totals; +} diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h new file mode 100644 index 000000000000..eaf27d41af8b --- /dev/null +++ b/drivers/md/dm-vdo/block-map.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_BLOCK_MAP_H +#define VDO_BLOCK_MAP_H + +#include + +#include "numeric.h" + +#include "admin-state.h" +#include "completion.h" +#include "encodings.h" +#include "int-map.h" +#include "statistics.h" +#include "types.h" +#include "vio.h" +#include "wait-queue.h" + +enum { + BLOCK_MAP_VIO_POOL_SIZE = 64, +}; + +/* + * Generation counter for page references. + */ +typedef u32 vdo_page_generation; + +extern const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY; + +struct forest; + +struct tree_page { + struct waiter waiter; + + /* Dirty list entry */ + struct list_head entry; + + /* If dirty, the tree zone flush generation in which it was last dirtied. */ + u8 generation; + + /* Whether this page is an interior tree page being written out. */ + bool writing; + + /* If writing, the tree zone flush generation of the copy being written. */ + u8 writing_generation; + + /* + * Sequence number of the earliest recovery journal block containing uncommitted updates to + * this page + */ + sequence_number_t recovery_lock; + + /* The value of recovery_lock when the this page last started writing */ + sequence_number_t writing_recovery_lock; + + char page_buffer[VDO_BLOCK_SIZE]; +}; + +enum block_map_page_type { + VDO_TREE_PAGE, + VDO_CACHE_PAGE, +}; + +typedef struct list_head dirty_era_t[2]; + +struct dirty_lists { + /** The number of periods after which an element will be expired */ + block_count_t maximum_age; + /** The oldest period which has unexpired elements */ + sequence_number_t oldest_period; + /** One more than the current period */ + sequence_number_t next_period; + /** The offset in the array of lists of the oldest period */ + block_count_t offset; + /** Expired pages */ + dirty_era_t expired; + /** The lists of dirty pages */ + dirty_era_t eras[]; +}; + +struct block_map_zone { + zone_count_t zone_number; + thread_id_t thread_id; + struct admin_state state; + struct block_map *block_map; + /* Dirty pages, by era*/ + struct dirty_lists *dirty_lists; + struct vdo_page_cache page_cache; + data_vio_count_t active_lookups; + struct int_map *loading_pages; + struct vio_pool *vio_pool; + /* The tree page which has issued or will be issuing a flush */ + struct tree_page *flusher; + struct wait_queue flush_waiters; + /* The generation after the most recent flush */ + u8 generation; + u8 oldest_generation; + /* The counts of dirty pages in each generation */ + u32 dirty_page_counts[256]; +}; + +struct block_map { + struct vdo *vdo; + struct action_manager *action_manager; + /* The absolute PBN of the first root of the tree part of the block map */ + physical_block_number_t root_origin; + block_count_t root_count; + + /* The era point we are currently distributing to the zones */ + sequence_number_t current_era_point; + /* The next era point */ + sequence_number_t pending_era_point; + + /* The number of entries in block map */ + block_count_t entry_count; + nonce_t nonce; + struct recovery_journal *journal; + + /* The trees for finding block map pages */ + struct forest *forest; + /* The expanded trees awaiting growth */ + struct forest *next_forest; + /* The number of entries after growth */ + block_count_t next_entry_count; + + zone_count_t zone_count; + struct block_map_zone zones[]; +}; + +/** + * typedef vdo_entry_callback_fn - A function to be called for each allocated PBN when traversing + * the forest. + * @pbn: A PBN of a tree node. + * @completion: The parent completion of the traversal. + * + * Return: VDO_SUCCESS or an error. + */ +typedef int (*vdo_entry_callback_fn)(physical_block_number_t pbn, + struct vdo_completion *completion); + +static inline struct block_map_page * __must_check +vdo_as_block_map_page(struct tree_page *tree_page) +{ + return (struct block_map_page *) tree_page->page_buffer; +} + +bool vdo_copy_valid_page(char *buffer, nonce_t nonce, + physical_block_number_t pbn, + struct block_map_page *page); + +void vdo_find_block_map_slot(struct data_vio *data_vio); + +physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, + page_number_t page_number); + +void vdo_write_tree_page(struct tree_page *page, struct block_map_zone *zone); + +void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, + struct vdo_completion *parent); + +int __must_check vdo_decode_block_map(struct block_map_state_2_0 state, + block_count_t logical_blocks, struct vdo *vdo, + struct recovery_journal *journal, nonce_t nonce, + page_count_t cache_size, block_count_t maximum_age, + struct block_map **map_ptr); + +void vdo_drain_block_map(struct block_map *map, const struct admin_state_code *operation, + struct vdo_completion *parent); + +void vdo_resume_block_map(struct block_map *map, struct vdo_completion *parent); + +int __must_check vdo_prepare_to_grow_block_map(struct block_map *map, + block_count_t new_logical_blocks); + +void vdo_grow_block_map(struct block_map *map, struct vdo_completion *parent); + +void vdo_abandon_block_map_growth(struct block_map *map); + +void vdo_free_block_map(struct block_map *map); + +struct block_map_state_2_0 __must_check vdo_record_block_map(const struct block_map *map); + +void vdo_initialize_block_map_from_journal(struct block_map *map, + struct recovery_journal *journal); + +zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio); + +void vdo_advance_block_map_era(struct block_map *map, + sequence_number_t recovery_block_number); + +void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio, + physical_block_number_t pbn, + enum block_mapping_state mapping_state, + sequence_number_t *recovery_lock); + +void vdo_get_mapped_block(struct data_vio *data_vio); + +void vdo_put_mapped_block(struct data_vio *data_vio); + +struct block_map_statistics __must_check vdo_get_block_map_statistics(struct block_map *map); + +/** + * vdo_convert_maximum_age() - Convert the maximum age to reflect the new recovery journal format + * @age: The configured maximum age + * + * Return: The converted age + * + * In the old recovery journal format, each journal block held 311 entries, and every write bio + * made two entries. The old maximum age was half the usable journal length. In the new format, + * each block holds only 217 entries, but each bio only makes one entry. We convert the configured + * age so that the number of writes in a block map era is the same in the old and new formats. This + * keeps the bound on the amount of work required to recover the block map from the recovery + * journal the same across the format change. It also keeps the amortization of block map page + * writes to write bios the same. + */ +static inline block_count_t vdo_convert_maximum_age(block_count_t age) +{ + return DIV_ROUND_UP(age * RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK, + 2 * RECOVERY_JOURNAL_ENTRIES_PER_BLOCK); +} + +#endif /* VDO_BLOCK_MAP_H */ From 14d531d7b7a124151c54f05e929d6e6fa896b68d Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:06:33 -0500 Subject: [PATCH 0578/1038] dm vdo: implement the block map page cache The set of leaf pages of the block map tree is too large to fit in memory, so each block map zone maintains a cache of leaf pages. This patch adds the implementation of that cache. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 1235 +++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/block-map.h | 154 ++++ 2 files changed, 1389 insertions(+) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index bc50564f7316..e703cb9eafc6 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -103,6 +103,8 @@ struct cursors { struct cursor cursors[]; }; +static const physical_block_number_t NO_PAGE = 0xFFFFFFFFFFFFFFFF; + /* Used to indicate that the page holding the location of a tree root has been "loaded". */ static const physical_block_number_t VDO_INVALID_PBN = 0xFFFFFFFFFFFFFFFF; @@ -112,6 +114,1239 @@ const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY = { .pbn_low_word = __cpu_to_le32(VDO_ZERO_BLOCK & UINT_MAX), }; +enum { + LOG_INTERVAL = 4000, + DISPLAY_INTERVAL = 100000, +}; + +/* + * For adjusting VDO page cache statistic fields which are only mutated on the logical zone thread. + * Prevents any compiler shenanigans from affecting other threads reading those stats. + */ +#define ADD_ONCE(value, delta) WRITE_ONCE(value, (value) + (delta)) + +static inline bool is_dirty(const struct page_info *info) +{ + return info->state == PS_DIRTY; +} + +static inline bool is_present(const struct page_info *info) +{ + return (info->state == PS_RESIDENT) || (info->state == PS_DIRTY); +} + +static inline bool is_in_flight(const struct page_info *info) +{ + return (info->state == PS_INCOMING) || (info->state == PS_OUTGOING); +} + +static inline bool is_incoming(const struct page_info *info) +{ + return info->state == PS_INCOMING; +} + +static inline bool is_outgoing(const struct page_info *info) +{ + return info->state == PS_OUTGOING; +} + +static inline bool is_valid(const struct page_info *info) +{ + return is_present(info) || is_outgoing(info); +} + +static char *get_page_buffer(struct page_info *info) +{ + struct vdo_page_cache *cache = info->cache; + + return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE]; +} + +static inline struct vdo_page_completion *page_completion_from_waiter(struct waiter *waiter) +{ + struct vdo_page_completion *completion; + + if (waiter == NULL) + return NULL; + + completion = container_of(waiter, struct vdo_page_completion, waiter); + vdo_assert_completion_type(&completion->completion, VDO_PAGE_COMPLETION); + return completion; +} + +/** + * initialize_info() - Initialize all page info structures and put them on the free list. + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_info(struct vdo_page_cache *cache) +{ + struct page_info *info; + + INIT_LIST_HEAD(&cache->free_list); + for (info = cache->infos; info < cache->infos + cache->page_count; info++) { + int result; + + info->cache = cache; + info->state = PS_FREE; + info->pbn = NO_PAGE; + + result = create_metadata_vio(cache->vdo, VIO_TYPE_BLOCK_MAP, + VIO_PRIORITY_METADATA, info, + get_page_buffer(info), &info->vio); + if (result != VDO_SUCCESS) + return result; + + /* The thread ID should never change. */ + info->vio->completion.callback_thread_id = cache->zone->thread_id; + + INIT_LIST_HEAD(&info->state_entry); + list_add_tail(&info->state_entry, &cache->free_list); + INIT_LIST_HEAD(&info->lru_entry); + } + + return VDO_SUCCESS; +} + +/** + * allocate_cache_components() - Allocate components of the cache which require their own + * allocation. + * @maximum_age: The number of journal blocks before a dirtied page is considered old and must be + * written out. + * + * The caller is responsible for all clean up on errors. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check allocate_cache_components(struct vdo_page_cache *cache) +{ + u64 size = cache->page_count * (u64) VDO_BLOCK_SIZE; + int result; + + result = uds_allocate(cache->page_count, struct page_info, "page infos", + &cache->infos); + if (result != UDS_SUCCESS) + return result; + + result = uds_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages); + if (result != UDS_SUCCESS) + return result; + + result = vdo_make_int_map(cache->page_count, 0, &cache->page_map); + if (result != UDS_SUCCESS) + return result; + + return initialize_info(cache); +} + +/** + * assert_on_cache_thread() - Assert that a function has been called on the VDO page cache's + * thread. + */ +static inline void assert_on_cache_thread(struct vdo_page_cache *cache, + const char *function_name) +{ + thread_id_t thread_id = vdo_get_callback_thread_id(); + + ASSERT_LOG_ONLY((thread_id == cache->zone->thread_id), + "%s() must only be called on cache thread %d, not thread %d", + function_name, cache->zone->thread_id, thread_id); +} + +/** assert_io_allowed() - Assert that a page cache may issue I/O. */ +static inline void assert_io_allowed(struct vdo_page_cache *cache) +{ + ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&cache->zone->state), + "VDO page cache may issue I/O"); +} + +/** report_cache_pressure() - Log and, if enabled, report cache pressure. */ +static void report_cache_pressure(struct vdo_page_cache *cache) +{ + ADD_ONCE(cache->stats.cache_pressure, 1); + if (cache->waiter_count > cache->page_count) { + if ((cache->pressure_report % LOG_INTERVAL) == 0) + uds_log_info("page cache pressure %u", cache->stats.cache_pressure); + + if (++cache->pressure_report >= DISPLAY_INTERVAL) + cache->pressure_report = 0; + } +} + +/** + * get_page_state_name() - Return the name of a page state. + * + * If the page state is invalid a static string is returned and the invalid state is logged. + * + * Return: A pointer to a static page state name. + */ +static const char * __must_check get_page_state_name(enum vdo_page_buffer_state state) +{ + int result; + static const char * const state_names[] = { + "UDS_FREE", "INCOMING", "FAILED", "RESIDENT", "DIRTY", "OUTGOING" + }; + + BUILD_BUG_ON(ARRAY_SIZE(state_names) != PAGE_STATE_COUNT); + + result = ASSERT(state < ARRAY_SIZE(state_names), + "Unknown page_state value %d", state); + if (result != UDS_SUCCESS) + return "[UNKNOWN PAGE STATE]"; + + return state_names[state]; +} + +/** + * update_counter() - Update the counter associated with a given state. + * @info: The page info to count. + * @delta: The delta to apply to the counter. + */ +static void update_counter(struct page_info *info, s32 delta) +{ + struct block_map_statistics *stats = &info->cache->stats; + + switch (info->state) { + case PS_FREE: + ADD_ONCE(stats->free_pages, delta); + return; + + case PS_INCOMING: + ADD_ONCE(stats->incoming_pages, delta); + return; + + case PS_OUTGOING: + ADD_ONCE(stats->outgoing_pages, delta); + return; + + case PS_FAILED: + ADD_ONCE(stats->failed_pages, delta); + return; + + case PS_RESIDENT: + ADD_ONCE(stats->clean_pages, delta); + return; + + case PS_DIRTY: + ADD_ONCE(stats->dirty_pages, delta); + return; + + default: + return; + } +} + +/** update_lru() - Update the lru information for an active page. */ +static void update_lru(struct page_info *info) +{ + if (info->cache->lru_list.prev != &info->lru_entry) + list_move_tail(&info->lru_entry, &info->cache->lru_list); +} + +/** + * set_info_state() - Set the state of a page_info and put it on the right list, adjusting + * counters. + */ +static void set_info_state(struct page_info *info, enum vdo_page_buffer_state new_state) +{ + if (new_state == info->state) + return; + + update_counter(info, -1); + info->state = new_state; + update_counter(info, 1); + + switch (info->state) { + case PS_FREE: + case PS_FAILED: + list_move_tail(&info->state_entry, &info->cache->free_list); + return; + + case PS_OUTGOING: + list_move_tail(&info->state_entry, &info->cache->outgoing_list); + return; + + case PS_DIRTY: + return; + + default: + list_del_init(&info->state_entry); + } +} + +/** set_info_pbn() - Set the pbn for an info, updating the map as needed. */ +static int __must_check set_info_pbn(struct page_info *info, physical_block_number_t pbn) +{ + struct vdo_page_cache *cache = info->cache; + + /* Either the new or the old page number must be NO_PAGE. */ + int result = ASSERT((pbn == NO_PAGE) || (info->pbn == NO_PAGE), + "Must free a page before reusing it."); + if (result != VDO_SUCCESS) + return result; + + if (info->pbn != NO_PAGE) + vdo_int_map_remove(cache->page_map, info->pbn); + + info->pbn = pbn; + + if (pbn != NO_PAGE) { + result = vdo_int_map_put(cache->page_map, pbn, info, true, NULL); + if (result != UDS_SUCCESS) + return result; + } + return VDO_SUCCESS; +} + +/** reset_page_info() - Reset page info to represent an unallocated page. */ +static int reset_page_info(struct page_info *info) +{ + int result; + + result = ASSERT(info->busy == 0, "VDO Page must not be busy"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(!vdo_has_waiters(&info->waiting), + "VDO Page must not have waiters"); + if (result != UDS_SUCCESS) + return result; + + result = set_info_pbn(info, NO_PAGE); + set_info_state(info, PS_FREE); + list_del_init(&info->lru_entry); + return result; +} + +/** + * find_free_page() - Find a free page. + * + * Return: A pointer to the page info structure (if found), NULL otherwise. + */ +static struct page_info * __must_check find_free_page(struct vdo_page_cache *cache) +{ + struct page_info *info; + + info = list_first_entry_or_null(&cache->free_list, struct page_info, + state_entry); + if (info != NULL) + list_del_init(&info->state_entry); + + return info; +} + +/** + * find_page() - Find the page info (if any) associated with a given pbn. + * @pbn: The absolute physical block number of the page. + * + * Return: The page info for the page if available, or NULL if not. + */ +static struct page_info * __must_check find_page(struct vdo_page_cache *cache, + physical_block_number_t pbn) +{ + if ((cache->last_found != NULL) && (cache->last_found->pbn == pbn)) + return cache->last_found; + + cache->last_found = vdo_int_map_get(cache->page_map, pbn); + return cache->last_found; +} + +/** + * select_lru_page() - Determine which page is least recently used. + * + * Picks the least recently used from among the non-busy entries at the front of each of the lru + * ring. Since whenever we mark a page busy we also put it to the end of the ring it is unlikely + * that the entries at the front are busy unless the queue is very short, but not impossible. + * + * Return: A pointer to the info structure for a relevant page, or NULL if no such page can be + * found. The page can be dirty or resident. + */ +static struct page_info * __must_check select_lru_page(struct vdo_page_cache *cache) +{ + struct page_info *info; + + list_for_each_entry(info, &cache->lru_list, lru_entry) + if ((info->busy == 0) && !is_in_flight(info)) + return info; + + return NULL; +} + +/* ASYNCHRONOUS INTERFACE BEYOND THIS POINT */ + +/** + * complete_with_page() - Helper to complete the VDO Page Completion request successfully. + * @info: The page info representing the result page. + * @vdo_page_comp: The VDO page completion to complete. + */ +static void complete_with_page(struct page_info *info, + struct vdo_page_completion *vdo_page_comp) +{ + bool available = vdo_page_comp->writable ? is_present(info) : is_valid(info); + + if (!available) { + uds_log_error_strerror(VDO_BAD_PAGE, + "Requested cache page %llu in state %s is not %s", + (unsigned long long) info->pbn, + get_page_state_name(info->state), + vdo_page_comp->writable ? "present" : "valid"); + vdo_fail_completion(&vdo_page_comp->completion, VDO_BAD_PAGE); + return; + } + + vdo_page_comp->info = info; + vdo_page_comp->ready = true; + vdo_finish_completion(&vdo_page_comp->completion); +} + +/** + * complete_waiter_with_error() - Complete a page completion with an error code. + * @waiter: The page completion, as a waiter. + * @result_ptr: A pointer to the error code. + * + * Implements waiter_callback_fn. + */ +static void complete_waiter_with_error(struct waiter *waiter, void *result_ptr) +{ + int *result = result_ptr; + + vdo_fail_completion(&page_completion_from_waiter(waiter)->completion, *result); +} + +/** + * complete_waiter_with_page() - Complete a page completion with a page. + * @waiter: The page completion, as a waiter. + * @page_info: The page info to complete with. + * + * Implements waiter_callback_fn. + */ +static void complete_waiter_with_page(struct waiter *waiter, void *page_info) +{ + complete_with_page(page_info, page_completion_from_waiter(waiter)); +} + +/** + * distribute_page_over_queue() - Complete a queue of VDO page completions with a page result. + * + * Upon completion the queue will be empty. + * + * Return: The number of pages distributed. + */ +static unsigned int distribute_page_over_queue(struct page_info *info, + struct wait_queue *queue) +{ + size_t pages; + + update_lru(info); + pages = vdo_count_waiters(queue); + + /* + * Increment the busy count once for each pending completion so that this page does not + * stop being busy until all completions have been processed (VDO-83). + */ + info->busy += pages; + + vdo_notify_all_waiters(queue, complete_waiter_with_page, info); + return pages; +} + +/** + * set_persistent_error() - Set a persistent error which all requests will receive in the future. + * @context: A string describing what triggered the error. + * + * Once triggered, all enqueued completions will get this error. Any future requests will result in + * this error as well. + */ +static void set_persistent_error(struct vdo_page_cache *cache, const char *context, + int result) +{ + struct page_info *info; + /* If we're already read-only, there's no need to log. */ + struct vdo *vdo = cache->zone->block_map->vdo; + + if ((result != VDO_READ_ONLY) && !vdo_is_read_only(vdo)) { + uds_log_error_strerror(result, "VDO Page Cache persistent error: %s", + context); + vdo_enter_read_only_mode(vdo, result); + } + + assert_on_cache_thread(cache, __func__); + + vdo_notify_all_waiters(&cache->free_waiters, complete_waiter_with_error, + &result); + cache->waiter_count = 0; + + for (info = cache->infos; info < cache->infos + cache->page_count; info++) + vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, + &result); +} + +/** + * validate_completed_page() - Check that a page completion which is being freed to the cache + * referred to a valid page and is in a valid state. + * @writable: Whether a writable page is required. + * + * Return: VDO_SUCCESS if the page was valid, otherwise as error + */ +static int __must_check validate_completed_page(struct vdo_page_completion *completion, + bool writable) +{ + int result; + + result = ASSERT(completion->ready, "VDO Page completion not ready"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(completion->info != NULL, + "VDO Page Completion must be complete"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(completion->info->pbn == completion->pbn, + "VDO Page Completion pbn must be consistent"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(is_valid(completion->info), + "VDO Page Completion page must be valid"); + if (result != UDS_SUCCESS) + return result; + + if (writable) { + result = ASSERT(completion->writable, "VDO Page Completion is writable"); + if (result != UDS_SUCCESS) + return result; + } + + return VDO_SUCCESS; +} + +static void check_for_drain_complete(struct block_map_zone *zone) +{ + if (vdo_is_state_draining(&zone->state) && + (zone->active_lookups == 0) && + !vdo_has_waiters(&zone->flush_waiters) && + !is_vio_pool_busy(zone->vio_pool) && + (zone->page_cache.outstanding_reads == 0) && + (zone->page_cache.outstanding_writes == 0)) { + vdo_finish_draining_with_result(&zone->state, + (vdo_is_read_only(zone->block_map->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS)); + } +} + +static void enter_zone_read_only_mode(struct block_map_zone *zone, int result) +{ + vdo_enter_read_only_mode(zone->block_map->vdo, result); + + /* + * We are in read-only mode, so we won't ever write any page out. Just take all waiters off + * the queue so the zone can drain. + */ + while (vdo_has_waiters(&zone->flush_waiters)) + vdo_dequeue_next_waiter(&zone->flush_waiters); + + check_for_drain_complete(zone); +} + +static bool __must_check +validate_completed_page_or_enter_read_only_mode(struct vdo_page_completion *completion, + bool writable) +{ + int result = validate_completed_page(completion, writable); + + if (result == VDO_SUCCESS) + return true; + + enter_zone_read_only_mode(completion->info->cache->zone, result); + return false; +} + +/** + * handle_load_error() - Handle page load errors. + * @completion: The page read vio. + */ +static void handle_load_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + + assert_on_cache_thread(cache, __func__); + vio_record_metadata_io_error(as_vio(completion)); + vdo_enter_read_only_mode(cache->zone->block_map->vdo, result); + ADD_ONCE(cache->stats.failed_reads, 1); + set_info_state(info, PS_FAILED); + vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); + reset_page_info(info); + + /* + * Don't decrement until right before calling check_for_drain_complete() to + * ensure that the above work can't cause the page cache to be freed out from under us. + */ + cache->outstanding_reads--; + check_for_drain_complete(cache->zone); +} + +/** + * page_is_loaded() - Callback used when a page has been loaded. + * @completion: The vio which has loaded the page. Its parent is the page_info. + */ +static void page_is_loaded(struct vdo_completion *completion) +{ + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + nonce_t nonce = info->cache->zone->block_map->nonce; + struct block_map_page *page; + enum block_map_page_validity validity; + + assert_on_cache_thread(cache, __func__); + + page = (struct block_map_page *) get_page_buffer(info); + validity = vdo_validate_block_map_page(page, nonce, info->pbn); + if (validity == VDO_BLOCK_MAP_PAGE_BAD) { + physical_block_number_t pbn = vdo_get_block_map_page_pbn(page); + int result = uds_log_error_strerror(VDO_BAD_PAGE, + "Expected page %llu but got page %llu instead", + (unsigned long long) info->pbn, + (unsigned long long) pbn); + + vdo_continue_completion(completion, result); + return; + } + + if (validity == VDO_BLOCK_MAP_PAGE_INVALID) + vdo_format_block_map_page(page, nonce, info->pbn, false); + + info->recovery_lock = 0; + set_info_state(info, PS_RESIDENT); + distribute_page_over_queue(info, &info->waiting); + + /* + * Don't decrement until right before calling check_for_drain_complete() to + * ensure that the above work can't cause the page cache to be freed out from under us. + */ + cache->outstanding_reads--; + check_for_drain_complete(cache->zone); +} + +/** + * handle_rebuild_read_error() - Handle a read error during a read-only rebuild. + * @completion: The page load completion. + */ +static void handle_rebuild_read_error(struct vdo_completion *completion) +{ + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + + assert_on_cache_thread(cache, __func__); + + /* + * We are doing a read-only rebuild, so treat this as a successful read of an uninitialized + * page. + */ + vio_record_metadata_io_error(as_vio(completion)); + ADD_ONCE(cache->stats.failed_reads, 1); + memset(get_page_buffer(info), 0, VDO_BLOCK_SIZE); + vdo_reset_completion(completion); + page_is_loaded(completion); +} + +static void load_cache_page_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct page_info *info = vio->completion.parent; + + continue_vio_after_io(vio, page_is_loaded, info->cache->zone->thread_id); +} + +/** + * launch_page_load() - Begin the process of loading a page. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check launch_page_load(struct page_info *info, + physical_block_number_t pbn) +{ + int result; + vdo_action_fn callback; + struct vdo_page_cache *cache = info->cache; + + assert_io_allowed(cache); + + result = set_info_pbn(info, pbn); + if (result != VDO_SUCCESS) + return result; + + result = ASSERT((info->busy == 0), "Page is not busy before loading."); + if (result != VDO_SUCCESS) + return result; + + set_info_state(info, PS_INCOMING); + cache->outstanding_reads++; + ADD_ONCE(cache->stats.pages_loaded, 1); + callback = (cache->rebuilding ? handle_rebuild_read_error : handle_load_error); + submit_metadata_vio(info->vio, pbn, load_cache_page_endio, + callback, REQ_OP_READ | REQ_PRIO); + return VDO_SUCCESS; +} + +static void write_pages(struct vdo_completion *completion); + +/** handle_flush_error() - Handle errors flushing the layer. */ +static void handle_flush_error(struct vdo_completion *completion) +{ + struct page_info *info = completion->parent; + + vio_record_metadata_io_error(as_vio(completion)); + set_persistent_error(info->cache, "flush failed", completion->result); + write_pages(completion); +} + +static void flush_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct page_info *info = vio->completion.parent; + + continue_vio_after_io(vio, write_pages, info->cache->zone->thread_id); +} + +/** save_pages() - Attempt to save the outgoing pages by first flushing the layer. */ +static void save_pages(struct vdo_page_cache *cache) +{ + struct page_info *info; + struct vio *vio; + + if ((cache->pages_in_flush > 0) || (cache->pages_to_flush == 0)) + return; + + assert_io_allowed(cache); + + info = list_first_entry(&cache->outgoing_list, struct page_info, state_entry); + + cache->pages_in_flush = cache->pages_to_flush; + cache->pages_to_flush = 0; + ADD_ONCE(cache->stats.flush_count, 1); + + vio = info->vio; + + /* + * We must make sure that the recovery journal entries that changed these pages were + * successfully persisted, and thus must issue a flush before each batch of pages is + * written to ensure this. + */ + submit_flush_vio(vio, flush_endio, handle_flush_error); +} + +/** + * schedule_page_save() - Add a page to the outgoing list of pages waiting to be saved. + * + * Once in the list, a page may not be used until it has been written out. + */ +static void schedule_page_save(struct page_info *info) +{ + if (info->busy > 0) { + info->write_status = WRITE_STATUS_DEFERRED; + return; + } + + info->cache->pages_to_flush++; + info->cache->outstanding_writes++; + set_info_state(info, PS_OUTGOING); +} + +/** + * launch_page_save() - Add a page to outgoing pages waiting to be saved, and then start saving + * pages if another save is not in progress. + */ +static void launch_page_save(struct page_info *info) +{ + schedule_page_save(info); + save_pages(info->cache); +} + +/** + * completion_needs_page() - Determine whether a given vdo_page_completion (as a waiter) is + * requesting a given page number. + * @context: A pointer to the pbn of the desired page. + * + * Implements waiter_match_fn. + * + * Return: true if the page completion is for the desired page number. + */ +static bool completion_needs_page(struct waiter *waiter, void *context) +{ + physical_block_number_t *pbn = context; + + return (page_completion_from_waiter(waiter)->pbn == *pbn); +} + +/** + * allocate_free_page() - Allocate a free page to the first completion in the waiting queue, and + * any other completions that match it in page number. + */ +static void allocate_free_page(struct page_info *info) +{ + int result; + struct waiter *oldest_waiter; + physical_block_number_t pbn; + struct vdo_page_cache *cache = info->cache; + + assert_on_cache_thread(cache, __func__); + + if (!vdo_has_waiters(&cache->free_waiters)) { + if (cache->stats.cache_pressure > 0) { + uds_log_info("page cache pressure relieved"); + WRITE_ONCE(cache->stats.cache_pressure, 0); + } + + return; + } + + result = reset_page_info(info); + if (result != VDO_SUCCESS) { + set_persistent_error(cache, "cannot reset page info", result); + return; + } + + oldest_waiter = vdo_get_first_waiter(&cache->free_waiters); + pbn = page_completion_from_waiter(oldest_waiter)->pbn; + + /* + * Remove all entries which match the page number in question and push them onto the page + * info's wait queue. + */ + vdo_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page, + &pbn, &info->waiting); + cache->waiter_count -= vdo_count_waiters(&info->waiting); + + result = launch_page_load(info, pbn); + if (result != VDO_SUCCESS) + vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); +} + +/** + * discard_a_page() - Begin the process of discarding a page. + * + * If no page is discardable, increments a count of deferred frees so that the next release of a + * page which is no longer busy will kick off another discard cycle. This is an indication that the + * cache is not big enough. + * + * If the selected page is not dirty, immediately allocates the page to the oldest completion + * waiting for a free page. + */ +static void discard_a_page(struct vdo_page_cache *cache) +{ + struct page_info *info = select_lru_page(cache); + + if (info == NULL) { + report_cache_pressure(cache); + return; + } + + if (!is_dirty(info)) { + allocate_free_page(info); + return; + } + + ASSERT_LOG_ONLY(!is_in_flight(info), + "page selected for discard is not in flight"); + + cache->discard_count++; + info->write_status = WRITE_STATUS_DISCARD; + launch_page_save(info); +} + +/** + * discard_page_for_completion() - Helper used to trigger a discard so that the completion can get + * a different page. + */ +static void discard_page_for_completion(struct vdo_page_completion *vdo_page_comp) +{ + struct vdo_page_cache *cache = vdo_page_comp->cache; + + cache->waiter_count++; + vdo_enqueue_waiter(&cache->free_waiters, &vdo_page_comp->waiter); + discard_a_page(cache); +} + +/** + * discard_page_if_needed() - Helper used to trigger a discard if the cache needs another free + * page. + * @cache: The page cache. + */ +static void discard_page_if_needed(struct vdo_page_cache *cache) +{ + if (cache->waiter_count > cache->discard_count) + discard_a_page(cache); +} + +/** + * write_has_finished() - Inform the cache that a write has finished (possibly with an error). + * @info: The info structure for the page whose write just completed. + * + * Return: true if the page write was a discard. + */ +static bool write_has_finished(struct page_info *info) +{ + bool was_discard = (info->write_status == WRITE_STATUS_DISCARD); + + assert_on_cache_thread(info->cache, __func__); + info->cache->outstanding_writes--; + + info->write_status = WRITE_STATUS_NORMAL; + return was_discard; +} + +/** + * handle_page_write_error() - Handler for page write errors. + * @completion: The page write vio. + */ +static void handle_page_write_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + + vio_record_metadata_io_error(as_vio(completion)); + + /* If we're already read-only, write failures are to be expected. */ + if (result != VDO_READ_ONLY) { + static DEFINE_RATELIMIT_STATE(error_limiter, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + if (__ratelimit(&error_limiter)) { + uds_log_error("failed to write block map page %llu", + (unsigned long long) info->pbn); + } + } + + set_info_state(info, PS_DIRTY); + ADD_ONCE(cache->stats.failed_writes, 1); + set_persistent_error(cache, "cannot write page", result); + + if (!write_has_finished(info)) + discard_page_if_needed(cache); + + check_for_drain_complete(cache->zone); +} + +static void page_is_written_out(struct vdo_completion *completion); + +static void write_cache_page_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct page_info *info = vio->completion.parent; + + continue_vio_after_io(vio, page_is_written_out, info->cache->zone->thread_id); +} + +/** + * page_is_written_out() - Callback used when a page has been written out. + * @completion: The vio which wrote the page. Its parent is a page_info. + */ +static void page_is_written_out(struct vdo_completion *completion) +{ + bool was_discard, reclaimed; + u32 reclamations; + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + struct block_map_page *page = (struct block_map_page *) get_page_buffer(info); + + if (!page->header.initialized) { + page->header.initialized = true; + submit_metadata_vio(info->vio, info->pbn, + write_cache_page_endio, + handle_page_write_error, + (REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH)); + return; + } + + /* Handle journal updates and torn write protection. */ + vdo_release_recovery_journal_block_reference(cache->zone->block_map->journal, + info->recovery_lock, + VDO_ZONE_TYPE_LOGICAL, + cache->zone->zone_number); + info->recovery_lock = 0; + was_discard = write_has_finished(info); + reclaimed = (!was_discard || (info->busy > 0) || vdo_has_waiters(&info->waiting)); + + set_info_state(info, PS_RESIDENT); + + reclamations = distribute_page_over_queue(info, &info->waiting); + ADD_ONCE(cache->stats.reclaimed, reclamations); + + if (was_discard) + cache->discard_count--; + + if (reclaimed) + discard_page_if_needed(cache); + else + allocate_free_page(info); + + check_for_drain_complete(cache->zone); +} + +/** + * write_pages() - Write the batch of pages which were covered by the layer flush which just + * completed. + * @flush_completion: The flush vio. + * + * This callback is registered in save_pages(). + */ +static void write_pages(struct vdo_completion *flush_completion) +{ + struct vdo_page_cache *cache = ((struct page_info *) flush_completion->parent)->cache; + + /* + * We need to cache these two values on the stack since in the error case below, it is + * possible for the last page info to cause the page cache to get freed. Hence once we + * launch the last page, it may be unsafe to dereference the cache [VDO-4724]. + */ + bool has_unflushed_pages = (cache->pages_to_flush > 0); + page_count_t pages_in_flush = cache->pages_in_flush; + + cache->pages_in_flush = 0; + while (pages_in_flush-- > 0) { + struct page_info *info = + list_first_entry(&cache->outgoing_list, struct page_info, + state_entry); + + list_del_init(&info->state_entry); + if (vdo_is_read_only(info->cache->zone->block_map->vdo)) { + struct vdo_completion *completion = &info->vio->completion; + + vdo_reset_completion(completion); + completion->callback = page_is_written_out; + completion->error_handler = handle_page_write_error; + vdo_fail_completion(completion, VDO_READ_ONLY); + continue; + } + ADD_ONCE(info->cache->stats.pages_saved, 1); + submit_metadata_vio(info->vio, info->pbn, write_cache_page_endio, + handle_page_write_error, REQ_OP_WRITE | REQ_PRIO); + } + + if (has_unflushed_pages) { + /* + * If there are unflushed pages, the cache can't have been freed, so this call is + * safe. + */ + save_pages(cache); + } +} + +/** + * vdo_release_page_completion() - Release a VDO Page Completion. + * + * The page referenced by this completion (if any) will no longer be held busy by this completion. + * If a page becomes discardable and there are completions awaiting free pages then a new round of + * page discarding is started. + */ +void vdo_release_page_completion(struct vdo_completion *completion) +{ + struct page_info *discard_info = NULL; + struct vdo_page_completion *page_completion = as_vdo_page_completion(completion); + struct vdo_page_cache *cache; + + if (completion->result == VDO_SUCCESS) { + if (!validate_completed_page_or_enter_read_only_mode(page_completion, false)) + return; + + if (--page_completion->info->busy == 0) + discard_info = page_completion->info; + } + + ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), + "Page being released after leaving all queues"); + + page_completion->info = NULL; + cache = page_completion->cache; + assert_on_cache_thread(cache, __func__); + + if (discard_info != NULL) { + if (discard_info->write_status == WRITE_STATUS_DEFERRED) { + discard_info->write_status = WRITE_STATUS_NORMAL; + launch_page_save(discard_info); + } + + /* + * if there are excess requests for pages (that have not already started discards) + * we need to discard some page (which may be this one) + */ + discard_page_if_needed(cache); + } +} + +/** + * load_page_for_completion() - Helper function to load a page as described by a VDO Page + * Completion. + */ +static void load_page_for_completion(struct page_info *info, + struct vdo_page_completion *vdo_page_comp) +{ + int result; + + vdo_enqueue_waiter(&info->waiting, &vdo_page_comp->waiter); + result = launch_page_load(info, vdo_page_comp->pbn); + if (result != VDO_SUCCESS) + vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); +} + +/** + * vdo_get_page() - Initialize a page completion and get a block map page. + * @page_completion: The vdo_page_completion to initialize. + * @zone: The block map zone of the desired page. + * @pbn: The absolute physical block of the desired page. + * @writable: Whether the page can be modified. + * @parent: The object to notify when the fetch is complete. + * @callback: The notification callback. + * @error_handler: The handler for fetch errors. + * @requeue: Whether we must requeue when notifying the parent. + * + * May cause another page to be discarded (potentially writing a dirty page) and the one nominated + * by the completion to be loaded from disk. When the callback is invoked, the page will be + * resident in the cache and marked busy. All callers must call vdo_release_page_completion() + * when they are done with the page to clear the busy mark. + */ +void vdo_get_page(struct vdo_page_completion *page_completion, + struct block_map_zone *zone, physical_block_number_t pbn, + bool writable, void *parent, vdo_action_fn callback, + vdo_action_fn error_handler, bool requeue) +{ + struct vdo_page_cache *cache = &zone->page_cache; + struct vdo_completion *completion = &page_completion->completion; + struct page_info *info; + + assert_on_cache_thread(cache, __func__); + ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), + "New page completion was not already on a wait queue"); + + *page_completion = (struct vdo_page_completion) { + .pbn = pbn, + .writable = writable, + .cache = cache, + }; + + vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION); + vdo_prepare_completion(completion, callback, error_handler, + cache->zone->thread_id, parent); + completion->requeue = requeue; + + if (page_completion->writable && vdo_is_read_only(cache->zone->block_map->vdo)) { + vdo_fail_completion(completion, VDO_READ_ONLY); + return; + } + + if (page_completion->writable) + ADD_ONCE(cache->stats.write_count, 1); + else + ADD_ONCE(cache->stats.read_count, 1); + + info = find_page(cache, page_completion->pbn); + if (info != NULL) { + /* The page is in the cache already. */ + if ((info->write_status == WRITE_STATUS_DEFERRED) || + is_incoming(info) || + (is_outgoing(info) && page_completion->writable)) { + /* The page is unusable until it has finished I/O. */ + ADD_ONCE(cache->stats.wait_for_page, 1); + vdo_enqueue_waiter(&info->waiting, &page_completion->waiter); + return; + } + + if (is_valid(info)) { + /* The page is usable. */ + ADD_ONCE(cache->stats.found_in_cache, 1); + if (!is_present(info)) + ADD_ONCE(cache->stats.read_outgoing, 1); + update_lru(info); + info->busy++; + complete_with_page(info, page_completion); + return; + } + + /* Something horrible has gone wrong. */ + ASSERT_LOG_ONLY(false, "Info found in a usable state."); + } + + /* The page must be fetched. */ + info = find_free_page(cache); + if (info != NULL) { + ADD_ONCE(cache->stats.fetch_required, 1); + load_page_for_completion(info, page_completion); + return; + } + + /* The page must wait for a page to be discarded. */ + ADD_ONCE(cache->stats.discard_required, 1); + discard_page_for_completion(page_completion); +} + +/** + * vdo_request_page_write() - Request that a VDO page be written out as soon as it is not busy. + * @completion: The vdo_page_completion containing the page. + */ +void vdo_request_page_write(struct vdo_completion *completion) +{ + struct page_info *info; + struct vdo_page_completion *vdo_page_comp = as_vdo_page_completion(completion); + + if (!validate_completed_page_or_enter_read_only_mode(vdo_page_comp, true)) + return; + + info = vdo_page_comp->info; + set_info_state(info, PS_DIRTY); + launch_page_save(info); +} + +/** + * vdo_get_cached_page() - Get the block map page from a page completion. + * @completion: A vdo page completion whose callback has been called. + * @page_ptr: A pointer to hold the page + * + * Return: VDO_SUCCESS or an error + */ +int vdo_get_cached_page(struct vdo_completion *completion, + struct block_map_page **page_ptr) +{ + int result; + struct vdo_page_completion *vpc; + + vpc = as_vdo_page_completion(completion); + result = validate_completed_page(vpc, true); + if (result == VDO_SUCCESS) + *page_ptr = (struct block_map_page *) get_page_buffer(vpc->info); + + return result; +} + +/** + * vdo_invalidate_page_cache() - Invalidate all entries in the VDO page cache. + * + * There must not be any dirty pages in the cache. + * + * Return: A success or error code. + */ +int vdo_invalidate_page_cache(struct vdo_page_cache *cache) +{ + struct page_info *info; + + assert_on_cache_thread(cache, __func__); + + /* Make sure we don't throw away any dirty pages. */ + for (info = cache->infos; info < cache->infos + cache->page_count; info++) { + int result = ASSERT(!is_dirty(info), "cache must have no dirty pages"); + + if (result != VDO_SUCCESS) + return result; + } + + /* Reset the page map by re-allocating it. */ + vdo_free_int_map(uds_forget(cache->page_map)); + return vdo_make_int_map(cache->page_count, 0, &cache->page_map); +} + /** * get_tree_page_by_index() - Get the tree page for a given height and page index. * diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h index eaf27d41af8b..dc807111b0e6 100644 --- a/drivers/md/dm-vdo/block-map.h +++ b/drivers/md/dm-vdo/block-map.h @@ -30,6 +30,140 @@ typedef u32 vdo_page_generation; extern const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY; +/* The VDO Page Cache abstraction. */ +struct vdo_page_cache { + /* the VDO which owns this cache */ + struct vdo *vdo; + /* number of pages in cache */ + page_count_t page_count; + /* number of pages to write in the current batch */ + page_count_t pages_in_batch; + /* Whether the VDO is doing a read-only rebuild */ + bool rebuilding; + + /* array of page information entries */ + struct page_info *infos; + /* raw memory for pages */ + char *pages; + /* cache last found page info */ + struct page_info *last_found; + /* map of page number to info */ + struct int_map *page_map; + /* main LRU list (all infos) */ + struct list_head lru_list; + /* free page list (oldest first) */ + struct list_head free_list; + /* outgoing page list */ + struct list_head outgoing_list; + /* number of read I/O operations pending */ + page_count_t outstanding_reads; + /* number of write I/O operations pending */ + page_count_t outstanding_writes; + /* number of pages covered by the current flush */ + page_count_t pages_in_flush; + /* number of pages waiting to be included in the next flush */ + page_count_t pages_to_flush; + /* number of discards in progress */ + unsigned int discard_count; + /* how many VPCs waiting for free page */ + unsigned int waiter_count; + /* queue of waiters who want a free page */ + struct wait_queue free_waiters; + /* + * Statistics are only updated on the logical zone thread, but are accessed from other + * threads. + */ + struct block_map_statistics stats; + /* counter for pressure reports */ + u32 pressure_report; + /* the block map zone to which this cache belongs */ + struct block_map_zone *zone; +}; + +/* + * The state of a page buffer. If the page buffer is free no particular page is bound to it, + * otherwise the page buffer is bound to particular page whose absolute pbn is in the pbn field. If + * the page is resident or dirty the page data is stable and may be accessed. Otherwise the page is + * in flight (incoming or outgoing) and its data should not be accessed. + * + * @note Update the static data in get_page_state_name() if you change this enumeration. + */ +enum vdo_page_buffer_state { + /* this page buffer is not being used */ + PS_FREE, + /* this page is being read from store */ + PS_INCOMING, + /* attempt to load this page failed */ + PS_FAILED, + /* this page is valid and un-modified */ + PS_RESIDENT, + /* this page is valid and modified */ + PS_DIRTY, + /* this page is being written and should not be used */ + PS_OUTGOING, + /* not a state */ + PAGE_STATE_COUNT, +} __packed; + +/* + * The write status of page + */ +enum vdo_page_write_status { + WRITE_STATUS_NORMAL, + WRITE_STATUS_DISCARD, + WRITE_STATUS_DEFERRED, +} __packed; + +/* Per-page-slot information. */ +struct page_info { + /* Preallocated page struct vio */ + struct vio *vio; + /* back-link for references */ + struct vdo_page_cache *cache; + /* the pbn of the page */ + physical_block_number_t pbn; + /* page is busy (temporarily locked) */ + u16 busy; + /* the write status the page */ + enum vdo_page_write_status write_status; + /* page state */ + enum vdo_page_buffer_state state; + /* queue of completions awaiting this item */ + struct wait_queue waiting; + /* state linked list entry */ + struct list_head state_entry; + /* LRU entry */ + struct list_head lru_entry; + /* + * The earliest recovery journal block containing uncommitted updates to the block map page + * associated with this page_info. A reference (lock) is held on that block to prevent it + * from being reaped. When this value changes, the reference on the old value must be + * released and a reference on the new value must be acquired. + */ + sequence_number_t recovery_lock; +}; + +/* + * A completion awaiting a specific page. Also a live reference into the page once completed, until + * freed. + */ +struct vdo_page_completion { + /* The generic completion */ + struct vdo_completion completion; + /* The cache involved */ + struct vdo_page_cache *cache; + /* The waiter for the pending list */ + struct waiter waiter; + /* The absolute physical block number of the page on disk */ + physical_block_number_t pbn; + /* Whether the page may be modified */ + bool writable; + /* Whether the page is available */ + bool ready; + /* The info structure for the page, only valid when ready */ + struct page_info *info; +}; + struct forest; struct tree_page { @@ -141,6 +275,26 @@ struct block_map { typedef int (*vdo_entry_callback_fn)(physical_block_number_t pbn, struct vdo_completion *completion); +static inline struct vdo_page_completion *as_vdo_page_completion(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_PAGE_COMPLETION); + return container_of(completion, struct vdo_page_completion, completion); +} + +void vdo_release_page_completion(struct vdo_completion *completion); + +void vdo_get_page(struct vdo_page_completion *page_completion, + struct block_map_zone *zone, physical_block_number_t pbn, + bool writable, void *parent, vdo_action_fn callback, + vdo_action_fn error_handler, bool requeue); + +void vdo_request_page_write(struct vdo_completion *completion); + +int __must_check vdo_get_cached_page(struct vdo_completion *completion, + struct block_map_page **page_ptr); + +int __must_check vdo_invalidate_page_cache(struct vdo_page_cache *cache); + static inline struct block_map_page * __must_check vdo_as_block_map_page(struct tree_page *tree_page) { From 95a72357688803736bf60db973cbf94c0ff3a6da Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:07:26 -0500 Subject: [PATCH 0579/1038] dm vdo: add the recovery journal The recovery journal is used to amortize updates across the block map and slab depot. Each write request causes an entry to be made in the journal. Entries are either "data remappings" or "block map remappings." For a data remapping, the journal records the logical address affected and its old and new physical mappings. For a block map remapping, the journal records the block map page number and the physical block allocated for it (block map pages are never reclaimed, so the old mapping is always 0). Each journal entry and the data write it represents must be stable on disk before the other metadata structures may be updated to reflect the operation. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/recovery-journal.c | 1761 ++++++++++++++++++++++++++ drivers/md/dm-vdo/recovery-journal.h | 312 +++++ 2 files changed, 2073 insertions(+) create mode 100644 drivers/md/dm-vdo/recovery-journal.c create mode 100644 drivers/md/dm-vdo/recovery-journal.h diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c new file mode 100644 index 000000000000..463f35ae1430 --- /dev/null +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -0,0 +1,1761 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "recovery-journal.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "encodings.h" +#include "io-submitter.h" +#include "slab-depot.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +static const u64 RECOVERY_COUNT_MASK = 0xff; + +enum { + /* + * The number of reserved blocks must be large enough to prevent a new recovery journal + * block write from overwriting a block which appears to still be a valid head block of the + * journal. Currently, that means reserving enough space for all 2048 data_vios. + */ + RECOVERY_JOURNAL_RESERVED_BLOCKS = + (MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2, + WRITE_FLAGS = REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH | REQ_SYNC | REQ_FUA, +}; + +/** + * DOC: Lock Counters. + * + * A lock_counter is intended to keep all of the locks for the blocks in the recovery journal. The + * per-zone counters are all kept in a single array which is arranged by zone (i.e. zone 0's lock 0 + * is at index 0, zone 0's lock 1 is at index 1, and zone 1's lock 0 is at index 'locks'. This + * arrangement is intended to minimize cache-line contention for counters from different zones. + * + * The locks are implemented as a single object instead of as a lock counter per lock both to + * afford this opportunity to reduce cache line contention and also to eliminate the need to have a + * completion per lock. + * + * Lock sets are laid out with the set for recovery journal first, followed by the logical zones, + * and then the physical zones. + */ + +enum lock_counter_state { + LOCK_COUNTER_STATE_NOT_NOTIFYING, + LOCK_COUNTER_STATE_NOTIFYING, + LOCK_COUNTER_STATE_SUSPENDED, +}; + +/** + * get_zone_count_ptr() - Get a pointer to the zone count for a given lock on a given zone. + * @journal: The recovery journal. + * @lock_number: The lock to get. + * @zone_type: The zone type whose count is desired. + * + * Return: A pointer to the zone count for the given lock and zone. + */ +static inline atomic_t *get_zone_count_ptr(struct recovery_journal *journal, + block_count_t lock_number, + enum vdo_zone_type zone_type) +{ + return ((zone_type == VDO_ZONE_TYPE_LOGICAL) + ? &journal->lock_counter.logical_zone_counts[lock_number] + : &journal->lock_counter.physical_zone_counts[lock_number]); +} + +/** + * get_counter() - Get the zone counter for a given lock on a given zone. + * @journal: The recovery journal. + * @lock_number: The lock to get. + * @zone_type: The zone type whose count is desired. + * @zone_id: The zone index whose count is desired. + * + * Return: The counter for the given lock and zone. + */ +static inline u16 *get_counter(struct recovery_journal *journal, + block_count_t lock_number, enum vdo_zone_type zone_type, + zone_count_t zone_id) +{ + struct lock_counter *counter = &journal->lock_counter; + block_count_t zone_counter = (counter->locks * zone_id) + lock_number; + + if (zone_type == VDO_ZONE_TYPE_JOURNAL) + return &counter->journal_counters[zone_counter]; + + if (zone_type == VDO_ZONE_TYPE_LOGICAL) + return &counter->logical_counters[zone_counter]; + + return &counter->physical_counters[zone_counter]; +} + +static atomic_t *get_decrement_counter(struct recovery_journal *journal, + block_count_t lock_number) +{ + return &journal->lock_counter.journal_decrement_counts[lock_number]; +} + +/** + * is_journal_zone_locked() - Check whether the journal zone is locked for a given lock. + * @journal: The recovery journal. + * @lock_number: The lock to check. + * + * Return: true if the journal zone is locked. + */ +static bool is_journal_zone_locked(struct recovery_journal *journal, + block_count_t lock_number) +{ + u16 journal_value = *get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0); + u32 decrements = atomic_read(get_decrement_counter(journal, lock_number)); + + /* Pairs with barrier in vdo_release_journal_entry_lock() */ + smp_rmb(); + ASSERT_LOG_ONLY((decrements <= journal_value), + "journal zone lock counter must not underflow"); + return (journal_value != decrements); +} + +/** + * vdo_release_recovery_journal_block_reference() - Release a reference to a recovery journal + * block. + * @journal: The recovery journal. + * @sequence_number: The journal sequence number of the referenced block. + * @zone_type: The type of the zone making the adjustment. + * @zone_id: The ID of the zone making the adjustment. + * + * If this is the last reference for a given zone type, an attempt will be made to reap the + * journal. + */ +void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id) +{ + u16 *current_value; + block_count_t lock_number; + int prior_state; + + if (sequence_number == 0) + return; + + lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); + current_value = get_counter(journal, lock_number, zone_type, zone_id); + + ASSERT_LOG_ONLY((*current_value >= 1), + "decrement of lock counter must not underflow"); + *current_value -= 1; + + if (zone_type == VDO_ZONE_TYPE_JOURNAL) { + if (is_journal_zone_locked(journal, lock_number)) + return; + } else { + atomic_t *zone_count; + + if (*current_value != 0) + return; + + zone_count = get_zone_count_ptr(journal, lock_number, zone_type); + + if (atomic_add_return(-1, zone_count) > 0) + return; + } + + /* + * Extra barriers because this was original developed using a CAS operation that implicitly + * had them. + */ + smp_mb__before_atomic(); + prior_state = atomic_cmpxchg(&journal->lock_counter.state, + LOCK_COUNTER_STATE_NOT_NOTIFYING, + LOCK_COUNTER_STATE_NOTIFYING); + /* same as before_atomic */ + smp_mb__after_atomic(); + + if (prior_state != LOCK_COUNTER_STATE_NOT_NOTIFYING) + return; + + vdo_launch_completion(&journal->lock_counter.completion); +} + +static inline struct recovery_journal_block * __must_check get_journal_block(struct list_head *list) +{ + return list_first_entry_or_null(list, struct recovery_journal_block, list_node); +} + +/** + * pop_free_list() - Get a block from the end of the free list. + * @journal: The journal. + * + * Return: The block or NULL if the list is empty. + */ +static struct recovery_journal_block * __must_check pop_free_list(struct recovery_journal *journal) +{ + struct recovery_journal_block *block; + + if (list_empty(&journal->free_tail_blocks)) + return NULL; + + block = list_last_entry(&journal->free_tail_blocks, + struct recovery_journal_block, list_node); + list_del_init(&block->list_node); + return block; +} + +/** + * is_block_dirty() - Check whether a recovery block is dirty. + * @block: The block to check. + * + * Indicates it has any uncommitted entries, which includes both entries not written and entries + * written but not yet acknowledged. + * + * Return: true if the block has any uncommitted entries. + */ +static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block) +{ + return (block->uncommitted_entry_count > 0); +} + +/** + * is_block_empty() - Check whether a journal block is empty. + * @block: The block to check. + * + * Return: true if the block has no entries. + */ +static inline bool __must_check is_block_empty(const struct recovery_journal_block *block) +{ + return (block->entry_count == 0); +} + +/** + * is_block_full() - Check whether a journal block is full. + * @block: The block to check. + * + * Return: true if the block is full. + */ +static inline bool __must_check is_block_full(const struct recovery_journal_block *block) +{ + return ((block == NULL) || (block->journal->entries_per_block == block->entry_count)); +} + +/** + * assert_on_journal_thread() - Assert that we are running on the journal thread. + * @journal: The journal. + * @function_name: The function doing the check (for logging). + */ +static void assert_on_journal_thread(struct recovery_journal *journal, + const char *function_name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id), + "%s() called on journal thread", function_name); +} + +/** + * continue_waiter() - Release a data_vio from the journal. + * + * Invoked whenever a data_vio is to be released from the journal, either because its entry was + * committed to disk, or because there was an error. Implements waiter_callback_fn. + */ +static void continue_waiter(struct waiter *waiter, void *context) +{ + continue_data_vio_with_error(waiter_as_data_vio(waiter), *((int *) context)); +} + +/** + * has_block_waiters() - Check whether the journal has any waiters on any blocks. + * @journal: The journal in question. + * + * Return: true if any block has a waiter. + */ +static inline bool has_block_waiters(struct recovery_journal *journal) +{ + struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks); + + /* + * Either the first active tail block (if it exists) has waiters, or no active tail block + * has waiters. + */ + return ((block != NULL) && + (vdo_has_waiters(&block->entry_waiters) || + vdo_has_waiters(&block->commit_waiters))); +} + +static void recycle_journal_blocks(struct recovery_journal *journal); +static void recycle_journal_block(struct recovery_journal_block *block); +static void notify_commit_waiters(struct recovery_journal *journal); + +/** + * suspend_lock_counter() - Prevent the lock counter from notifying. + * @counter: The counter. + * + * Return: true if the lock counter was not notifying and hence the suspend was efficacious. + */ +static bool suspend_lock_counter(struct lock_counter *counter) +{ + int prior_state; + + /* + * Extra barriers because this was originally developed using a CAS operation that + * implicitly had them. + */ + smp_mb__before_atomic(); + prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_NOT_NOTIFYING, + LOCK_COUNTER_STATE_SUSPENDED); + /* same as before_atomic */ + smp_mb__after_atomic(); + + return ((prior_state == LOCK_COUNTER_STATE_SUSPENDED) || + (prior_state == LOCK_COUNTER_STATE_NOT_NOTIFYING)); +} + +static inline bool is_read_only(struct recovery_journal *journal) +{ + return vdo_is_read_only(journal->flush_vio->completion.vdo); +} + +/** + * check_for_drain_complete() - Check whether the journal has drained. + * @journal: The journal which may have just drained. + */ +static void check_for_drain_complete(struct recovery_journal *journal) +{ + int result = VDO_SUCCESS; + + if (is_read_only(journal)) { + result = VDO_READ_ONLY; + /* + * Clean up any full active blocks which were not written due to read-only mode. + * + * FIXME: This would probably be better as a short-circuit in write_block(). + */ + notify_commit_waiters(journal); + recycle_journal_blocks(journal); + + /* Release any data_vios waiting to be assigned entries. */ + vdo_notify_all_waiters(&journal->entry_waiters, continue_waiter, + &result); + } + + if (!vdo_is_state_draining(&journal->state) || + journal->reaping || + has_block_waiters(journal) || + vdo_has_waiters(&journal->entry_waiters) || + !suspend_lock_counter(&journal->lock_counter)) + return; + + if (vdo_is_state_saving(&journal->state)) { + if (journal->active_block != NULL) { + ASSERT_LOG_ONLY(((result == VDO_READ_ONLY) || + !is_block_dirty(journal->active_block)), + "journal being saved has clean active block"); + recycle_journal_block(journal->active_block); + } + + ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), + "all blocks in a journal being saved must be inactive"); + } + + vdo_finish_draining_with_result(&journal->state, result); +} + +/** + * notify_recovery_journal_of_read_only_mode() - Notify a recovery journal that the VDO has gone + * read-only. + * @listener: The journal. + * @parent: The completion to notify in order to acknowledge the notification. + * + * Implements vdo_read_only_notification_fn. + */ +static void notify_recovery_journal_of_read_only_mode(void *listener, + struct vdo_completion *parent) +{ + check_for_drain_complete(listener); + vdo_finish_completion(parent); +} + +/** + * enter_journal_read_only_mode() - Put the journal in read-only mode. + * @journal: The journal which has failed. + * @error_code: The error result triggering this call. + * + * All attempts to add entries after this function is called will fail. All VIOs waiting for + * commits will be awakened with an error. + */ +static void enter_journal_read_only_mode(struct recovery_journal *journal, + int error_code) +{ + vdo_enter_read_only_mode(journal->flush_vio->completion.vdo, error_code); + check_for_drain_complete(journal); +} + +/** + * vdo_get_recovery_journal_current_sequence_number() - Obtain the recovery journal's current + * sequence number. + * @journal: The journal in question. + * + * Exposed only so the block map can be initialized therefrom. + * + * Return: The sequence number of the tail block. + */ +sequence_number_t vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal) +{ + return journal->tail; +} + +/** + * get_recovery_journal_head() - Get the head of the recovery journal. + * @journal: The journal. + * + * The head is the lowest sequence number of the block map head and the slab journal head. + * + * Return: the head of the journal. + */ +static inline sequence_number_t get_recovery_journal_head(const struct recovery_journal *journal) +{ + return min(journal->block_map_head, journal->slab_journal_head); +} + +/** + * compute_recovery_count_byte() - Compute the recovery count byte for a given recovery count. + * @recovery_count: The recovery count. + * + * Return: The byte corresponding to the recovery count. + */ +static inline u8 __must_check compute_recovery_count_byte(u64 recovery_count) +{ + return (u8)(recovery_count & RECOVERY_COUNT_MASK); +} + +/** + * check_slab_journal_commit_threshold() - Check whether the journal is over the threshold, and if + * so, force the oldest slab journal tail block to commit. + * @journal: The journal. + */ +static void check_slab_journal_commit_threshold(struct recovery_journal *journal) +{ + block_count_t current_length = journal->tail - journal->slab_journal_head; + + if (current_length > journal->slab_journal_commit_threshold) { + journal->events.slab_journal_commits_requested++; + vdo_commit_oldest_slab_journal_tail_blocks(journal->depot, + journal->slab_journal_head); + } +} + +static void reap_recovery_journal(struct recovery_journal *journal); +static void assign_entries(struct recovery_journal *journal); + +/** + * finish_reaping() - Finish reaping the journal. + * @journal: The journal being reaped. + */ +static void finish_reaping(struct recovery_journal *journal) +{ + block_count_t blocks_reaped; + sequence_number_t old_head = get_recovery_journal_head(journal); + + journal->block_map_head = journal->block_map_reap_head; + journal->slab_journal_head = journal->slab_journal_reap_head; + blocks_reaped = get_recovery_journal_head(journal) - old_head; + journal->available_space += blocks_reaped * journal->entries_per_block; + journal->reaping = false; + check_slab_journal_commit_threshold(journal); + assign_entries(journal); + check_for_drain_complete(journal); +} + +/** + * complete_reaping() - Finish reaping the journal after flushing the lower layer. + * @completion: The journal's flush VIO. + * + * This is the callback registered in reap_recovery_journal(). + */ +static void complete_reaping(struct vdo_completion *completion) +{ + struct recovery_journal *journal = completion->parent; + + finish_reaping(journal); + + /* Try reaping again in case more locks were released while flush was out. */ + reap_recovery_journal(journal); +} + +/** + * handle_flush_error() - Handle an error when flushing the lower layer due to reaping. + * @completion: The journal's flush VIO. + */ +static void handle_flush_error(struct vdo_completion *completion) +{ + struct recovery_journal *journal = completion->parent; + + vio_record_metadata_io_error(as_vio(completion)); + journal->reaping = false; + enter_journal_read_only_mode(journal, completion->result); +} + +static void flush_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct recovery_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, complete_reaping, journal->thread_id); +} + +/** + * initialize_journal_state() - Set all journal fields appropriately to start journaling from the + * current active block. + * @journal: The journal to be reset based on its active block. + */ +static void initialize_journal_state(struct recovery_journal *journal) +{ + journal->append_point.sequence_number = journal->tail; + journal->last_write_acknowledged = journal->tail; + journal->block_map_head = journal->tail; + journal->slab_journal_head = journal->tail; + journal->block_map_reap_head = journal->tail; + journal->slab_journal_reap_head = journal->tail; + journal->block_map_head_block_number = + vdo_get_recovery_journal_block_number(journal, journal->block_map_head); + journal->slab_journal_head_block_number = + vdo_get_recovery_journal_block_number(journal, + journal->slab_journal_head); + journal->available_space = + (journal->entries_per_block * vdo_get_recovery_journal_length(journal->size)); +} + +/** + * vdo_get_recovery_journal_length() - Get the number of usable recovery journal blocks. + * @journal_size: The size of the recovery journal in blocks. + * + * Return: the number of recovery journal blocks usable for entries. + */ +block_count_t vdo_get_recovery_journal_length(block_count_t journal_size) +{ + block_count_t reserved_blocks = journal_size / 4; + + if (reserved_blocks > RECOVERY_JOURNAL_RESERVED_BLOCKS) + reserved_blocks = RECOVERY_JOURNAL_RESERVED_BLOCKS; + return (journal_size - reserved_blocks); +} + +/** + * reap_recovery_journal_callback() - Attempt to reap the journal. + * @completion: The lock counter completion. + * + * Attempts to reap the journal now that all the locks on some journal block have been released. + * This is the callback registered with the lock counter. + */ +static void reap_recovery_journal_callback(struct vdo_completion *completion) +{ + struct recovery_journal *journal = (struct recovery_journal *) completion->parent; + /* + * The acknowledgment must be done before reaping so that there is no race between + * acknowledging the notification and unlocks wishing to notify. + */ + smp_wmb(); + atomic_set(&journal->lock_counter.state, LOCK_COUNTER_STATE_NOT_NOTIFYING); + + if (vdo_is_state_quiescing(&journal->state)) { + /* + * Don't start reaping when the journal is trying to quiesce. Do check if this + * notification is the last thing the is waiting on. + */ + check_for_drain_complete(journal); + return; + } + + reap_recovery_journal(journal); + check_slab_journal_commit_threshold(journal); +} + +/** + * initialize_lock_counter() - Initialize a lock counter. + * + * @journal: The recovery journal. + * @vdo: The vdo. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check initialize_lock_counter(struct recovery_journal *journal, + struct vdo *vdo) +{ + int result; + struct thread_config *config = &vdo->thread_config; + struct lock_counter *counter = &journal->lock_counter; + + result = uds_allocate(journal->size, u16, __func__, &counter->journal_counters); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(journal->size, atomic_t, __func__, + &counter->journal_decrement_counts); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(journal->size * config->logical_zone_count, u16, __func__, + &counter->logical_counters); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(journal->size, atomic_t, __func__, + &counter->logical_zone_counts); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(journal->size * config->physical_zone_count, u16, __func__, + &counter->physical_counters); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(journal->size, atomic_t, __func__, + &counter->physical_zone_counts); + if (result != VDO_SUCCESS) + return result; + + vdo_initialize_completion(&counter->completion, vdo, + VDO_LOCK_COUNTER_COMPLETION); + vdo_prepare_completion(&counter->completion, reap_recovery_journal_callback, + reap_recovery_journal_callback, config->journal_thread, + journal); + counter->logical_zones = config->logical_zone_count; + counter->physical_zones = config->physical_zone_count; + counter->locks = journal->size; + return VDO_SUCCESS; +} + +/** + * set_journal_tail() - Set the journal's tail sequence number. + * @journal: The journal whose tail is to be set. + * @tail: The new tail value. + */ +static void set_journal_tail(struct recovery_journal *journal, sequence_number_t tail) +{ + /* VDO does not support sequence numbers above 1 << 48 in the slab journal. */ + if (tail >= (1ULL << 48)) + enter_journal_read_only_mode(journal, VDO_JOURNAL_OVERFLOW); + + journal->tail = tail; +} + +/** + * initialize_recovery_block() - Initialize a journal block. + * @vdo: The vdo from which to construct vios. + * @journal: The journal to which the block will belong. + * @block: The block to initialize. + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *journal, + struct recovery_journal_block *block) +{ + char *data; + int result; + + /* + * Ensure that a block is large enough to store RECOVERY_JOURNAL_ENTRIES_PER_BLOCK entries. + */ + BUILD_BUG_ON(RECOVERY_JOURNAL_ENTRIES_PER_BLOCK > + ((VDO_BLOCK_SIZE - sizeof(struct packed_journal_header)) / + sizeof(struct packed_recovery_journal_entry))); + + /* + * Allocate a full block for the journal block even though not all of the space is used + * since the VIO needs to write a full disk block. + */ + result = uds_allocate(VDO_BLOCK_SIZE, char, __func__, &data); + if (result != VDO_SUCCESS) + return result; + + result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL, + VIO_PRIORITY_HIGH, block, 1, data, &block->vio); + if (result != VDO_SUCCESS) { + uds_free(data); + return result; + } + + list_add_tail(&block->list_node, &journal->free_tail_blocks); + block->journal = journal; + return VDO_SUCCESS; +} + +/** + * vdo_decode_recovery_journal() - Make a recovery journal and initialize it with the state that + * was decoded from the super block. + * + * @state: The decoded state of the journal. + * @nonce: The nonce of the VDO. + * @vdo: The VDO. + * @partition: The partition for the journal. + * @recovery_count: The VDO's number of completed recoveries. + * @journal_size: The number of blocks in the journal on disk. + * @journal_ptr: The pointer to hold the new recovery journal. + * + * Return: A success or error code. + */ +int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t nonce, + struct vdo *vdo, struct partition *partition, + u64 recovery_count, block_count_t journal_size, + struct recovery_journal **journal_ptr) +{ + block_count_t i; + struct recovery_journal *journal; + int result; + + result = uds_allocate_extended(struct recovery_journal, + RECOVERY_JOURNAL_RESERVED_BLOCKS, + struct recovery_journal_block, __func__, + &journal); + if (result != VDO_SUCCESS) + return result; + + INIT_LIST_HEAD(&journal->free_tail_blocks); + INIT_LIST_HEAD(&journal->active_tail_blocks); + vdo_initialize_wait_queue(&journal->pending_writes); + + journal->thread_id = vdo->thread_config.journal_thread; + journal->origin = partition->offset; + journal->nonce = nonce; + journal->recovery_count = compute_recovery_count_byte(recovery_count); + journal->size = journal_size; + journal->slab_journal_commit_threshold = (journal_size * 2) / 3; + journal->logical_blocks_used = state.logical_blocks_used; + journal->block_map_data_blocks = state.block_map_data_blocks; + journal->entries_per_block = RECOVERY_JOURNAL_ENTRIES_PER_BLOCK; + set_journal_tail(journal, state.journal_start); + initialize_journal_state(journal); + /* TODO: this will have to change if we make initial resume of a VDO a real resume */ + vdo_set_admin_state_code(&journal->state, VDO_ADMIN_STATE_SUSPENDED); + + for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) { + struct recovery_journal_block *block = &journal->blocks[i]; + + result = initialize_recovery_block(vdo, journal, block); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + } + + result = initialize_lock_counter(journal, vdo); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + result = create_metadata_vio(vdo, VIO_TYPE_RECOVERY_JOURNAL, VIO_PRIORITY_HIGH, + journal, NULL, &journal->flush_vio); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + result = vdo_register_read_only_listener(vdo, journal, + notify_recovery_journal_of_read_only_mode, + journal->thread_id); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + result = vdo_make_default_thread(vdo, journal->thread_id); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + journal->flush_vio->completion.callback_thread_id = journal->thread_id; + *journal_ptr = journal; + return VDO_SUCCESS; +} + +/** + * vdo_free_recovery_journal() - Free a recovery journal. + * @journal: The recovery journal to free. + */ +void vdo_free_recovery_journal(struct recovery_journal *journal) +{ + block_count_t i; + + if (journal == NULL) + return; + + uds_free(uds_forget(journal->lock_counter.logical_zone_counts)); + uds_free(uds_forget(journal->lock_counter.physical_zone_counts)); + uds_free(uds_forget(journal->lock_counter.journal_counters)); + uds_free(uds_forget(journal->lock_counter.journal_decrement_counts)); + uds_free(uds_forget(journal->lock_counter.logical_counters)); + uds_free(uds_forget(journal->lock_counter.physical_counters)); + free_vio(uds_forget(journal->flush_vio)); + + /* + * FIXME: eventually, the journal should be constructed in a quiescent state which + * requires opening before use. + */ + if (!vdo_is_state_quiescent(&journal->state)) { + ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), + "journal being freed has no active tail blocks"); + } else if (!vdo_is_state_saved(&journal->state) && + !list_empty(&journal->active_tail_blocks)) { + uds_log_warning("journal being freed has uncommitted entries"); + } + + for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) { + struct recovery_journal_block *block = &journal->blocks[i]; + + uds_free(uds_forget(block->vio.data)); + free_vio_components(&block->vio); + } + + uds_free(journal); +} + +/** + * vdo_initialize_recovery_journal_post_repair() - Initialize the journal after a repair. + * @journal: The journal in question. + * @recovery_count: The number of completed recoveries. + * @tail: The new tail block sequence number. + * @logical_blocks_used: The new number of logical blocks used. + * @block_map_data_blocks: The new number of block map data blocks. + */ +void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal, + u64 recovery_count, + sequence_number_t tail, + block_count_t logical_blocks_used, + block_count_t block_map_data_blocks) +{ + set_journal_tail(journal, tail + 1); + journal->recovery_count = compute_recovery_count_byte(recovery_count); + initialize_journal_state(journal); + journal->logical_blocks_used = logical_blocks_used; + journal->block_map_data_blocks = block_map_data_blocks; +} + +/** + * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from + * data blocks, currently in use. + * @journal: The journal in question. + * + * Return: The number of block map pages allocated from slabs. + */ +block_count_t vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal) +{ + return journal->block_map_data_blocks; +} + +/** + * vdo_get_recovery_journal_thread_id() - Get the ID of a recovery journal's thread. + * @journal: The journal to query. + * + * Return: The ID of the journal's thread. + */ +thread_id_t vdo_get_recovery_journal_thread_id(struct recovery_journal *journal) +{ + return journal->thread_id; +} + +/** + * vdo_open_recovery_journal() - Prepare the journal for new entries. + * @journal: The journal in question. + * @depot: The slab depot for this VDO. + * @block_map: The block map for this VDO. + */ +void vdo_open_recovery_journal(struct recovery_journal *journal, + struct slab_depot *depot, struct block_map *block_map) +{ + journal->depot = depot; + journal->block_map = block_map; + WRITE_ONCE(journal->state.current_state, VDO_ADMIN_STATE_NORMAL_OPERATION); +} + +/** + * vdo_record_recovery_journal() - Record the state of a recovery journal for encoding in the super + * block. + * @journal: the recovery journal. + * + * Return: the state of the journal. + */ +struct recovery_journal_state_7_0 +vdo_record_recovery_journal(const struct recovery_journal *journal) +{ + struct recovery_journal_state_7_0 state = { + .logical_blocks_used = journal->logical_blocks_used, + .block_map_data_blocks = journal->block_map_data_blocks, + }; + + if (vdo_is_state_saved(&journal->state)) { + /* + * If the journal is saved, we should start one past the active block (since the + * active block is not guaranteed to be empty). + */ + state.journal_start = journal->tail; + } else { + /* + * When we're merely suspended or have gone read-only, we must record the first + * block that might have entries that need to be applied. + */ + state.journal_start = get_recovery_journal_head(journal); + } + + return state; +} + +/** + * get_block_header() - Get a pointer to the packed journal block header in the block buffer. + * @block: The recovery block. + * + * Return: The block's header. + */ +static inline struct packed_journal_header * +get_block_header(const struct recovery_journal_block *block) +{ + return (struct packed_journal_header *) block->vio.data; +} + +/** + * set_active_sector() - Set the current sector of the current block and initialize it. + * @block: The block to update. + * @sector: A pointer to the first byte of the new sector. + */ +static void set_active_sector(struct recovery_journal_block *block, void *sector) +{ + block->sector = sector; + block->sector->check_byte = get_block_header(block)->check_byte; + block->sector->recovery_count = block->journal->recovery_count; + block->sector->entry_count = 0; +} + +/** + * advance_tail() - Advance the tail of the journal. + * @journal: The journal whose tail should be advanced. + * + * Return: true if the tail was advanced. + */ +static bool advance_tail(struct recovery_journal *journal) +{ + struct recovery_block_header unpacked; + struct packed_journal_header *header; + struct recovery_journal_block *block; + + block = journal->active_block = pop_free_list(journal); + if (block == NULL) + return false; + + list_move_tail(&block->list_node, &journal->active_tail_blocks); + + unpacked = (struct recovery_block_header) { + .metadata_type = VDO_METADATA_RECOVERY_JOURNAL_2, + .block_map_data_blocks = journal->block_map_data_blocks, + .logical_blocks_used = journal->logical_blocks_used, + .nonce = journal->nonce, + .recovery_count = journal->recovery_count, + .sequence_number = journal->tail, + .check_byte = vdo_compute_recovery_journal_check_byte(journal, + journal->tail), + }; + + header = get_block_header(block); + memset(block->vio.data, 0x0, VDO_BLOCK_SIZE); + block->sequence_number = journal->tail; + block->entry_count = 0; + block->uncommitted_entry_count = 0; + block->block_number = vdo_get_recovery_journal_block_number(journal, + journal->tail); + + vdo_pack_recovery_block_header(&unpacked, header); + set_active_sector(block, vdo_get_journal_block_sector(header, 1)); + set_journal_tail(journal, journal->tail + 1); + vdo_advance_block_map_era(journal->block_map, journal->tail); + return true; +} + +/** + * initialize_lock_count() - Initialize the value of the journal zone's counter for a given lock. + * @journal: The recovery journal. + * + * Context: This must be called from the journal zone. + */ +static void initialize_lock_count(struct recovery_journal *journal) +{ + u16 *journal_value; + block_count_t lock_number = journal->active_block->block_number; + atomic_t *decrement_counter = get_decrement_counter(journal, lock_number); + + journal_value = get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0); + ASSERT_LOG_ONLY((*journal_value == atomic_read(decrement_counter)), + "count to be initialized not in use"); + *journal_value = journal->entries_per_block + 1; + atomic_set(decrement_counter, 0); +} + +/** + * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check + * whether an entry of the given type may be assigned at this time. + * @journal: The journal receiving an entry. + * + * Return: true if there is space in the journal to store an entry of the specified type. + */ +static bool prepare_to_assign_entry(struct recovery_journal *journal) +{ + if (journal->available_space == 0) + return false; + + if (is_block_full(journal->active_block) && !advance_tail(journal)) + return false; + + if (!is_block_empty(journal->active_block)) + return true; + + if ((journal->tail - get_recovery_journal_head(journal)) > journal->size) { + /* Cannot use this block since the journal is full. */ + journal->events.disk_full++; + return false; + } + + /* + * Don't allow the new block to be reaped until all of its entries have been committed to + * the block map and until the journal block has been fully committed as well. Because the + * block map update is done only after any slab journal entries have been made, the + * per-entry lock for the block map entry serves to protect those as well. + */ + initialize_lock_count(journal); + return true; +} + +static void write_blocks(struct recovery_journal *journal); + +/** + * schedule_block_write() - Queue a block for writing. + * @journal: The journal in question. + * @block: The block which is now ready to write. + * + * The block is expected to be full. If the block is currently writing, this is a noop as the block + * will be queued for writing when the write finishes. The block must not currently be queued for + * writing. + */ +static void schedule_block_write(struct recovery_journal *journal, + struct recovery_journal_block *block) +{ + if (!block->committing) + vdo_enqueue_waiter(&journal->pending_writes, &block->write_waiter); + /* + * At the end of adding entries, or discovering this partial block is now full and ready to + * rewrite, we will call write_blocks() and write a whole batch. + */ +} + +/** + * release_journal_block_reference() - Release a reference to a journal block. + * @block: The journal block from which to release a reference. + */ +static void release_journal_block_reference(struct recovery_journal_block *block) +{ + vdo_release_recovery_journal_block_reference(block->journal, + block->sequence_number, + VDO_ZONE_TYPE_JOURNAL, 0); +} + +static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio) +{ + if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + journal->block_map_data_blocks++; + return; + } + + if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED) + journal->logical_blocks_used++; + + if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED) + journal->logical_blocks_used--; +} + +/** + * assign_entry() - Assign an entry waiter to the active block. + * + * Implements waiter_callback_fn. + */ +static void assign_entry(struct waiter *waiter, void *context) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct recovery_journal_block *block = context; + struct recovery_journal *journal = block->journal; + + /* Record the point at which we will make the journal entry. */ + data_vio->recovery_journal_point = (struct journal_point) { + .sequence_number = block->sequence_number, + .entry_count = block->entry_count, + }; + + update_usages(journal, data_vio); + journal->available_space--; + + if (!vdo_has_waiters(&block->entry_waiters)) + journal->events.blocks.started++; + + vdo_enqueue_waiter(&block->entry_waiters, &data_vio->waiter); + block->entry_count++; + block->uncommitted_entry_count++; + journal->events.entries.started++; + + if (is_block_full(block)) { + /* + * The block is full, so we can write it anytime henceforth. If it is already + * committing, we'll queue it for writing when it comes back. + */ + schedule_block_write(journal, block); + } + + /* Force out slab journal tail blocks when threshold is reached. */ + check_slab_journal_commit_threshold(journal); +} + +static void assign_entries(struct recovery_journal *journal) +{ + if (journal->adding_entries) { + /* Protect against re-entrancy. */ + return; + } + + journal->adding_entries = true; + while (vdo_has_waiters(&journal->entry_waiters) && prepare_to_assign_entry(journal)) { + vdo_notify_next_waiter(&journal->entry_waiters, assign_entry, + journal->active_block); + } + + /* Now that we've finished with entries, see if we have a batch of blocks to write. */ + write_blocks(journal); + journal->adding_entries = false; +} + +/** + * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been + * fully committed. + * @block: The block to be recycled. + */ +static void recycle_journal_block(struct recovery_journal_block *block) +{ + struct recovery_journal *journal = block->journal; + block_count_t i; + + list_move_tail(&block->list_node, &journal->free_tail_blocks); + + /* Release any unused entry locks. */ + for (i = block->entry_count; i < journal->entries_per_block; i++) + release_journal_block_reference(block); + + /* + * Release our own lock against reaping now that the block is completely committed, or + * we're giving up because we're in read-only mode. + */ + if (block->entry_count > 0) + release_journal_block_reference(block); + + if (block == journal->active_block) + journal->active_block = NULL; +} + +/** + * continue_committed_waiter() - invoked whenever a VIO is to be released from the journal because + * its entry was committed to disk. + * + * Implements waiter_callback_fn. + */ +static void continue_committed_waiter(struct waiter *waiter, void *context) +{ + struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct recovery_journal *journal = context; + int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS); + bool has_decrement; + + ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point, + &data_vio->recovery_journal_point), + "DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commit waiter point is (%llu, %u)", + (unsigned long long) journal->commit_point.sequence_number, + journal->commit_point.entry_count, + (unsigned long long) data_vio->recovery_journal_point.sequence_number, + data_vio->recovery_journal_point.entry_count); + + journal->commit_point = data_vio->recovery_journal_point; + data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS; + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + /* + * The increment must be launched first since it must come before the + * decrement if they are in the same slab. + */ + has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK); + if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement) + continue_data_vio(data_vio); + + if (has_decrement) + vdo_launch_completion(&data_vio->decrement_completion); +} + +/** + * notify_commit_waiters() - Notify any VIOs whose entries have now committed. + * @journal: The recovery journal to update. + */ +static void notify_commit_waiters(struct recovery_journal *journal) +{ + struct recovery_journal_block *block; + + list_for_each_entry(block, &journal->active_tail_blocks, list_node) { + if (block->committing) + return; + + vdo_notify_all_waiters(&block->commit_waiters, continue_committed_waiter, + journal); + if (is_read_only(journal)) { + vdo_notify_all_waiters(&block->entry_waiters, + continue_committed_waiter, journal); + } else if (is_block_dirty(block) || !is_block_full(block)) { + /* Stop at partially-committed or partially-filled blocks. */ + return; + } + } +} + +/** + * recycle_journal_blocks() - Recycle any journal blocks which have been fully committed. + * @journal: The recovery journal to update. + */ +static void recycle_journal_blocks(struct recovery_journal *journal) +{ + struct recovery_journal_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) { + if (block->committing) { + /* Don't recycle committing blocks. */ + return; + } + + if (!is_read_only(journal) && + (is_block_dirty(block) || !is_block_full(block))) { + /* + * Don't recycle partially written or partially full blocks, except in + * read-only mode. + */ + return; + } + + recycle_journal_block(block); + } +} + +/** + * complete_write() - Handle post-commit processing. + * @completion: The completion of the VIO writing this block. + * + * This is the callback registered by write_block(). If more entries accumulated in the block being + * committed while the commit was in progress, another commit will be initiated. + */ +static void complete_write(struct vdo_completion *completion) +{ + struct recovery_journal_block *block = completion->parent; + struct recovery_journal *journal = block->journal; + struct recovery_journal_block *last_active_block; + + assert_on_journal_thread(journal, __func__); + + journal->pending_write_count -= 1; + journal->events.blocks.committed += 1; + journal->events.entries.committed += block->entries_in_commit; + block->uncommitted_entry_count -= block->entries_in_commit; + block->entries_in_commit = 0; + block->committing = false; + + /* If this block is the latest block to be acknowledged, record that fact. */ + if (block->sequence_number > journal->last_write_acknowledged) + journal->last_write_acknowledged = block->sequence_number; + + last_active_block = get_journal_block(&journal->active_tail_blocks); + ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number), + "completed journal write is still active"); + + notify_commit_waiters(journal); + + /* + * Is this block now full? Reaping, and adding entries, might have already sent it off for + * rewriting; else, queue it for rewrite. + */ + if (is_block_dirty(block) && is_block_full(block)) + schedule_block_write(journal, block); + + recycle_journal_blocks(journal); + write_blocks(journal); + + check_for_drain_complete(journal); +} + +static void handle_write_error(struct vdo_completion *completion) +{ + struct recovery_journal_block *block = completion->parent; + struct recovery_journal *journal = block->journal; + + vio_record_metadata_io_error(as_vio(completion)); + uds_log_error_strerror(completion->result, + "cannot write recovery journal block %llu", + (unsigned long long) block->sequence_number); + enter_journal_read_only_mode(journal, completion->result); + complete_write(completion); +} + +static void complete_write_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct recovery_journal_block *block = vio->completion.parent; + struct recovery_journal *journal = block->journal; + + continue_vio_after_io(vio, complete_write, journal->thread_id); +} + +/** + * add_queued_recovery_entries() - Actually add entries from the queue to the given block. + * @block: The journal block. + */ +static void add_queued_recovery_entries(struct recovery_journal_block *block) +{ + while (vdo_has_waiters(&block->entry_waiters)) { + struct data_vio *data_vio = + waiter_as_data_vio(vdo_dequeue_next_waiter(&block->entry_waiters)); + struct tree_lock *lock = &data_vio->tree_lock; + struct packed_recovery_journal_entry *packed_entry; + struct recovery_journal_entry new_entry; + + if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) + set_active_sector(block, + (char *) block->sector + VDO_SECTOR_SIZE); + + /* Compose and encode the entry. */ + packed_entry = &block->sector->entries[block->sector->entry_count++]; + new_entry = (struct recovery_journal_entry) { + .mapping = { + .pbn = data_vio->increment_updater.zpbn.pbn, + .state = data_vio->increment_updater.zpbn.state, + }, + .unmapping = { + .pbn = data_vio->decrement_updater.zpbn.pbn, + .state = data_vio->decrement_updater.zpbn.state, + }, + .operation = data_vio->increment_updater.operation, + .slot = lock->tree_slots[lock->height].block_map_slot, + }; + *packed_entry = vdo_pack_recovery_journal_entry(&new_entry); + data_vio->recovery_sequence_number = block->sequence_number; + + /* Enqueue the data_vio to wait for its entry to commit. */ + vdo_enqueue_waiter(&block->commit_waiters, &data_vio->waiter); + } +} + +/** + * write_block() - Issue a block for writing. + * + * Implements waiter_callback_fn. + */ +static void write_block(struct waiter *waiter, void *context __always_unused) +{ + struct recovery_journal_block *block = + container_of(waiter, struct recovery_journal_block, write_waiter); + struct recovery_journal *journal = block->journal; + struct packed_journal_header *header = get_block_header(block); + + if (block->committing || !vdo_has_waiters(&block->entry_waiters) || is_read_only(journal)) + return; + + block->entries_in_commit = vdo_count_waiters(&block->entry_waiters); + add_queued_recovery_entries(block); + + journal->pending_write_count += 1; + journal->events.blocks.written += 1; + journal->events.entries.written += block->entries_in_commit; + + header->block_map_head = __cpu_to_le64(journal->block_map_head); + header->slab_journal_head = __cpu_to_le64(journal->slab_journal_head); + header->entry_count = __cpu_to_le16(block->entry_count); + + block->committing = true; + + /* + * We must issue a flush and a FUA for every commit. The flush is necessary to ensure that + * the data being referenced is stable. The FUA is necessary to ensure that the journal + * block itself is stable before allowing overwrites of the lbn's previous data. + */ + submit_metadata_vio(&block->vio, journal->origin + block->block_number, + complete_write_endio, handle_write_error, WRITE_FLAGS); +} + + +/** + * write_blocks() - Attempt to commit blocks, according to write policy. + * @journal: The recovery journal. + */ +static void write_blocks(struct recovery_journal *journal) +{ + assert_on_journal_thread(journal, __func__); + /* + * We call this function after adding entries to the journal and after finishing a block + * write. Thus, when this function terminates we must either have no VIOs waiting in the + * journal or have some outstanding IO to provide a future wakeup. + * + * We want to only issue full blocks if there are no pending writes. However, if there are + * no outstanding writes and some unwritten entries, we must issue a block, even if it's + * the active block and it isn't full. + */ + if (journal->pending_write_count > 0) + return; + + /* Write all the full blocks. */ + vdo_notify_all_waiters(&journal->pending_writes, write_block, NULL); + + /* + * Do we need to write the active block? Only if we have no outstanding writes, even after + * issuing all of the full writes. + */ + if ((journal->pending_write_count == 0) && (journal->active_block != NULL)) + write_block(&journal->active_block->write_waiter, NULL); +} + +/** + * vdo_add_recovery_journal_entry() - Add an entry to a recovery journal. + * @journal: The journal in which to make an entry. + * @data_vio: The data_vio for which to add the entry. The entry will be taken + * from the logical and new_mapped fields of the data_vio. The + * data_vio's recovery_sequence_number field will be set to the + * sequence number of the journal block in which the entry was + * made. + * + * This method is asynchronous. The data_vio will not be called back until the entry is committed + * to the on-disk journal. + */ +void vdo_add_recovery_journal_entry(struct recovery_journal *journal, + struct data_vio *data_vio) +{ + assert_on_journal_thread(journal, __func__); + if (!vdo_is_state_normal(&journal->state)) { + continue_data_vio_with_error(data_vio, VDO_INVALID_ADMIN_STATE); + return; + } + + if (is_read_only(journal)) { + continue_data_vio_with_error(data_vio, VDO_READ_ONLY); + return; + } + + ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0, + "journal lock not held for new entry"); + + vdo_advance_journal_point(&journal->append_point, journal->entries_per_block); + vdo_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter); + assign_entries(journal); +} + +/** + * is_lock_locked() - Check whether a lock is locked for a zone type. + * @journal: The recovery journal. + * @lock_number: The lock to check. + * @zone_type: The type of the zone. + * + * If the recovery journal has a lock on the lock number, both logical and physical zones are + * considered locked. + * + * Return: true if the specified lock has references (is locked). + */ +static bool is_lock_locked(struct recovery_journal *journal, block_count_t lock_number, + enum vdo_zone_type zone_type) +{ + atomic_t *zone_count; + bool locked; + + if (is_journal_zone_locked(journal, lock_number)) + return true; + + zone_count = get_zone_count_ptr(journal, lock_number, zone_type); + locked = (atomic_read(zone_count) != 0); + /* Pairs with implicit barrier in vdo_release_recovery_journal_block_reference() */ + smp_rmb(); + return locked; +} + +/** + * reap_recovery_journal() - Conduct a sweep on a recovery journal to reclaim unreferenced blocks. + * @journal: The recovery journal. + */ +static void reap_recovery_journal(struct recovery_journal *journal) +{ + if (journal->reaping) { + /* + * We already have an outstanding reap in progress. We need to wait for it to + * finish. + */ + return; + } + + if (vdo_is_state_quiescent(&journal->state)) { + /* We are supposed to not do IO. Don't botch it by reaping. */ + return; + } + + /* + * Start reclaiming blocks only when the journal head has no references. Then stop when a + * block is referenced. + */ + while ((journal->block_map_reap_head < journal->last_write_acknowledged) && + !is_lock_locked(journal, journal->block_map_head_block_number, + VDO_ZONE_TYPE_LOGICAL)) { + journal->block_map_reap_head++; + if (++journal->block_map_head_block_number == journal->size) + journal->block_map_head_block_number = 0; + } + + while ((journal->slab_journal_reap_head < journal->last_write_acknowledged) && + !is_lock_locked(journal, journal->slab_journal_head_block_number, + VDO_ZONE_TYPE_PHYSICAL)) { + journal->slab_journal_reap_head++; + if (++journal->slab_journal_head_block_number == journal->size) + journal->slab_journal_head_block_number = 0; + } + + if ((journal->block_map_reap_head == journal->block_map_head) && + (journal->slab_journal_reap_head == journal->slab_journal_head)) { + /* Nothing happened. */ + return; + } + + /* + * If the block map head will advance, we must flush any block map page modified by the + * entries we are reaping. If the slab journal head will advance, we must flush the slab + * summary update covering the slab journal that just released some lock. + */ + journal->reaping = true; + submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error); +} + +/** + * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block + * from somewhere other than the journal itself. + * @journal: The recovery journal. + * @sequence_number: The journal sequence number of the referenced block. + * @zone_type: The type of the zone making the adjustment. + * @zone_id: The ID of the zone making the adjustment. + */ +void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id) +{ + block_count_t lock_number; + u16 *current_value; + + if (sequence_number == 0) + return; + + ASSERT_LOG_ONLY((zone_type != VDO_ZONE_TYPE_JOURNAL), + "invalid lock count increment from journal zone"); + + lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); + current_value = get_counter(journal, lock_number, zone_type, zone_id); + ASSERT_LOG_ONLY(*current_value < U16_MAX, + "increment of lock counter must not overflow"); + + if (*current_value == 0) { + /* + * This zone is acquiring this lock for the first time. Extra barriers because this + * was original developed using an atomic add operation that implicitly had them. + */ + smp_mb__before_atomic(); + atomic_inc(get_zone_count_ptr(journal, lock_number, zone_type)); + /* same as before_atomic */ + smp_mb__after_atomic(); + } + + *current_value += 1; +} + +/** + * vdo_release_journal_entry_lock() - Release a single per-entry reference count for a recovery + * journal block. + * @journal: The recovery journal. + * @sequence_number: The journal sequence number of the referenced block. + */ +void vdo_release_journal_entry_lock(struct recovery_journal *journal, + sequence_number_t sequence_number) +{ + block_count_t lock_number; + + if (sequence_number == 0) + return; + + lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); + /* + * Extra barriers because this was originally developed using an atomic add operation that + * implicitly had them. + */ + smp_mb__before_atomic(); + atomic_inc(get_decrement_counter(journal, lock_number)); + /* same as before_atomic */ + smp_mb__after_atomic(); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct recovery_journal, state)); +} + +/** + * vdo_drain_recovery_journal() - Drain recovery journal I/O. + * @journal: The journal to drain. + * @operation: The drain operation (suspend or save). + * @parent: The completion to notify once the journal is drained. + * + * All uncommitted entries will be written out. + */ +void vdo_drain_recovery_journal(struct recovery_journal *journal, + const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + assert_on_journal_thread(journal, __func__); + vdo_start_draining(&journal->state, operation, parent, initiate_drain); +} + +/** + * resume_lock_counter() - Re-allow notifications from a suspended lock counter. + * @counter: The counter. + * + * Return: true if the lock counter was suspended. + */ +static bool resume_lock_counter(struct lock_counter *counter) +{ + int prior_state; + + /* + * Extra barriers because this was original developed using a CAS operation that implicitly + * had them. + */ + smp_mb__before_atomic(); + prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_SUSPENDED, + LOCK_COUNTER_STATE_NOT_NOTIFYING); + /* same as before_atomic */ + smp_mb__after_atomic(); + + return (prior_state == LOCK_COUNTER_STATE_SUSPENDED); +} + +/** + * vdo_resume_recovery_journal() - Resume a recovery journal which has been drained. + * @journal: The journal to resume. + * @parent: The completion to finish once the journal is resumed. + */ +void vdo_resume_recovery_journal(struct recovery_journal *journal, + struct vdo_completion *parent) +{ + bool saved; + + assert_on_journal_thread(journal, __func__); + saved = vdo_is_state_saved(&journal->state); + vdo_set_completion_result(parent, vdo_resume_if_quiescent(&journal->state)); + if (is_read_only(journal)) { + vdo_continue_completion(parent, VDO_READ_ONLY); + return; + } + + if (saved) + initialize_journal_state(journal); + + if (resume_lock_counter(&journal->lock_counter)) { + /* We might have missed a notification. */ + reap_recovery_journal(journal); + } + + vdo_launch_completion(parent); +} + +/** + * vdo_get_recovery_journal_logical_blocks_used() - Get the number of logical blocks in use by the + * VDO. + * @journal: The journal. + * + * Return: The number of logical blocks in use by the VDO. + */ +block_count_t vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal) +{ + return journal->logical_blocks_used; +} + +/** + * vdo_get_recovery_journal_statistics() - Get the current statistics from the recovery journal. + * @journal: The recovery journal to query. + * + * Return: A copy of the current statistics for the journal. + */ +struct recovery_journal_statistics +vdo_get_recovery_journal_statistics(const struct recovery_journal *journal) +{ + return journal->events; +} + +/** + * dump_recovery_block() - Dump the contents of the recovery block to the log. + * @block: The block to dump. + */ +static void dump_recovery_block(const struct recovery_journal_block *block) +{ + uds_log_info(" sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters", + (unsigned long long) block->sequence_number, block->entry_count, + (block->committing ? "committing" : "waiting"), + vdo_count_waiters(&block->entry_waiters), + vdo_count_waiters(&block->commit_waiters)); +} + +/** + * vdo_dump_recovery_journal_statistics() - Dump some current statistics and other debug info from + * the recovery journal. + * @journal: The recovery journal to dump. + */ +void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal) +{ + const struct recovery_journal_block *block; + struct recovery_journal_statistics stats = vdo_get_recovery_journal_statistics(journal); + + uds_log_info("Recovery Journal"); + uds_log_info(" block_map_head=%llu slab_journal_head=%llu last_write_acknowledged=%llu tail=%llu block_map_reap_head=%llu slab_journal_reap_head=%llu disk_full=%llu slab_journal_commits_requested=%llu entry_waiters=%zu", + (unsigned long long) journal->block_map_head, + (unsigned long long) journal->slab_journal_head, + (unsigned long long) journal->last_write_acknowledged, + (unsigned long long) journal->tail, + (unsigned long long) journal->block_map_reap_head, + (unsigned long long) journal->slab_journal_reap_head, + (unsigned long long) stats.disk_full, + (unsigned long long) stats.slab_journal_commits_requested, + vdo_count_waiters(&journal->entry_waiters)); + uds_log_info(" entries: started=%llu written=%llu committed=%llu", + (unsigned long long) stats.entries.started, + (unsigned long long) stats.entries.written, + (unsigned long long) stats.entries.committed); + uds_log_info(" blocks: started=%llu written=%llu committed=%llu", + (unsigned long long) stats.blocks.started, + (unsigned long long) stats.blocks.written, + (unsigned long long) stats.blocks.committed); + + uds_log_info(" active blocks:"); + list_for_each_entry(block, &journal->active_tail_blocks, list_node) + dump_recovery_block(block); +} diff --git a/drivers/md/dm-vdo/recovery-journal.h b/drivers/md/dm-vdo/recovery-journal.h new file mode 100644 index 000000000000..c6d83019f918 --- /dev/null +++ b/drivers/md/dm-vdo/recovery-journal.h @@ -0,0 +1,312 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_RECOVERY_JOURNAL_H +#define VDO_RECOVERY_JOURNAL_H + +#include + +#include "numeric.h" + +#include "admin-state.h" +#include "constants.h" +#include "encodings.h" +#include "flush.h" +#include "statistics.h" +#include "types.h" +#include "wait-queue.h" + +/** + * DOC: recovery journal. + * + * The recovery_journal provides a log of all block mapping and reference count changes which have + * not yet been stably written to the block map or slab journals. This log helps to reduce the + * write amplification of writes by providing amortization of slab journal and block map page + * updates. + * + * The journal consists of a set of on-disk blocks arranged as a circular log with monotonically + * increasing sequence numbers. Three sequence numbers serve to define the active extent of the + * journal. The 'head' is the oldest active block in the journal. The 'tail' is the end of the + * half-open interval containing the active blocks. 'active' is the number of the block actively + * receiving entries. In an empty journal, head == active == tail. Once any entries are added, tail + * = active + 1, and head may be any value in the interval [tail - size, active]. + * + * The journal also contains a set of in-memory blocks which are used to buffer up entries until + * they can be committed. In general the number of in-memory blocks ('tail_buffer_count') will be + * less than the on-disk size. Each in-memory block is also a vdo_completion. Each in-memory block + * has a vio which is used to commit that block to disk. The vio's data is the on-disk + * representation of the journal block. In addition each in-memory block has a buffer which is used + * to accumulate entries while a partial commit of the block is in progress. In-memory blocks are + * kept on two rings. Free blocks live on the 'free_tail_blocks' ring. When a block becomes active + * (see below) it is moved to the 'active_tail_blocks' ring. When a block is fully committed, it is + * moved back to the 'free_tail_blocks' ring. + * + * When entries are added to the journal, they are added to the active in-memory block, as + * indicated by the 'active_block' field. If the caller wishes to wait for the entry to be + * committed, the requesting VIO will be attached to the in-memory block to which the caller's + * entry was added. If the caller does wish to wait, or if the entry filled the active block, an + * attempt will be made to commit that block to disk. If there is already another commit in + * progress, the attempt will be ignored and then automatically retried when the in-progress commit + * completes. If there is no commit in progress, any data_vios waiting on the block are transferred + * to the block's vio which is then written, automatically waking all of the waiters when it + * completes. When the write completes, any entries which accumulated in the block are copied to + * the vio's data buffer. + * + * Finally, the journal maintains a set of counters, one for each on disk journal block. These + * counters are used as locks to prevent premature reaping of journal blocks. Each time a new + * sequence number is used, the counter for the corresponding block is incremented. The counter is + * subsequently decremented when that block is filled and then committed for the last time. This + * prevents blocks from being reaped while they are still being updated. The counter is also + * incremented once for each entry added to a block, and decremented once each time the block map + * is updated in memory for that request. This prevents blocks from being reaped while their VIOs + * are still active. Finally, each in-memory block map page tracks the oldest journal block that + * contains entries corresponding to uncommitted updates to that block map page. Each time an + * in-memory block map page is updated, it checks if the journal block for the VIO is earlier than + * the one it references, in which case it increments the count on the earlier journal block and + * decrements the count on the later journal block, maintaining a lock on the oldest journal block + * containing entries for that page. When a block map page has been flushed from the cache, the + * counter for the journal block it references is decremented. Whenever the counter for the head + * block goes to 0, the head is advanced until it comes to a block whose counter is not 0 or until + * it reaches the active block. This is the mechanism for reclaiming journal space on disk. + * + * If there is no in-memory space when a VIO attempts to add an entry, the VIO will be attached to + * the 'commit_completion' and will be woken the next time a full block has committed. If there is + * no on-disk space when a VIO attempts to add an entry, the VIO will be attached to the + * 'reap_completion', and will be woken the next time a journal block is reaped. + */ + +enum vdo_zone_type { + VDO_ZONE_TYPE_ADMIN, + VDO_ZONE_TYPE_JOURNAL, + VDO_ZONE_TYPE_LOGICAL, + VDO_ZONE_TYPE_PHYSICAL, +}; + +struct lock_counter { + /** The completion for notifying the owner of a lock release */ + struct vdo_completion completion; + /** The number of logical zones which may hold locks */ + zone_count_t logical_zones; + /** The number of physical zones which may hold locks */ + zone_count_t physical_zones; + /** The number of locks */ + block_count_t locks; + /** Whether the lock release notification is in flight */ + atomic_t state; + /** The number of logical zones which hold each lock */ + atomic_t *logical_zone_counts; + /** The number of physical zones which hold each lock */ + atomic_t *physical_zone_counts; + /** The per-lock counts for the journal zone */ + u16 *journal_counters; + /** The per-lock decrement counts for the journal zone */ + atomic_t *journal_decrement_counts; + /** The per-zone, per-lock reference counts for logical zones */ + u16 *logical_counters; + /** The per-zone, per-lock reference counts for physical zones */ + u16 *physical_counters; +}; + +struct recovery_journal_block { + /* The doubly linked pointers for the free or active lists */ + struct list_head list_node; + /* The waiter for the pending full block list */ + struct waiter write_waiter; + /* The journal to which this block belongs */ + struct recovery_journal *journal; + /* A pointer to the current sector in the packed block buffer */ + struct packed_journal_sector *sector; + /* The vio for writing this block */ + struct vio vio; + /* The sequence number for this block */ + sequence_number_t sequence_number; + /* The location of this block in the on-disk journal */ + physical_block_number_t block_number; + /* Whether this block is being committed */ + bool committing; + /* The total number of entries in this block */ + journal_entry_count_t entry_count; + /* The total number of uncommitted entries (queued or committing) */ + journal_entry_count_t uncommitted_entry_count; + /* The number of new entries in the current commit */ + journal_entry_count_t entries_in_commit; + /* The queue of vios which will make entries for the next commit */ + struct wait_queue entry_waiters; + /* The queue of vios waiting for the current commit */ + struct wait_queue commit_waiters; +}; + +struct recovery_journal { + /* The thread ID of the journal zone */ + thread_id_t thread_id; + /* The slab depot which can hold locks on this journal */ + struct slab_depot *depot; + /* The block map which can hold locks on this journal */ + struct block_map *block_map; + /* The queue of vios waiting to make entries */ + struct wait_queue entry_waiters; + /* The number of free entries in the journal */ + u64 available_space; + /* The number of decrement entries which need to be made */ + data_vio_count_t pending_decrement_count; + /* Whether the journal is adding entries from the increment or decrement waiters queues */ + bool adding_entries; + /* The administrative state of the journal */ + struct admin_state state; + /* Whether a reap is in progress */ + bool reaping; + /* The location of the first journal block */ + physical_block_number_t origin; + /* The oldest active block in the journal on disk for block map rebuild */ + sequence_number_t block_map_head; + /* The oldest active block in the journal on disk for slab journal replay */ + sequence_number_t slab_journal_head; + /* The newest block in the journal on disk to which a write has finished */ + sequence_number_t last_write_acknowledged; + /* The end of the half-open interval of the active journal */ + sequence_number_t tail; + /* The point at which the last entry will have been added */ + struct journal_point append_point; + /* The journal point of the vio most recently released from the journal */ + struct journal_point commit_point; + /* The nonce of the VDO */ + nonce_t nonce; + /* The number of recoveries completed by the VDO */ + u8 recovery_count; + /* The number of entries which fit in a single block */ + journal_entry_count_t entries_per_block; + /* Unused in-memory journal blocks */ + struct list_head free_tail_blocks; + /* In-memory journal blocks with records */ + struct list_head active_tail_blocks; + /* A pointer to the active block (the one we are adding entries to now) */ + struct recovery_journal_block *active_block; + /* Journal blocks that need writing */ + struct wait_queue pending_writes; + /* The new block map reap head after reaping */ + sequence_number_t block_map_reap_head; + /* The head block number for the block map rebuild range */ + block_count_t block_map_head_block_number; + /* The new slab journal reap head after reaping */ + sequence_number_t slab_journal_reap_head; + /* The head block number for the slab journal replay range */ + block_count_t slab_journal_head_block_number; + /* The data-less vio, usable only for flushing */ + struct vio *flush_vio; + /* The number of blocks in the on-disk journal */ + block_count_t size; + /* The number of logical blocks that are in-use */ + block_count_t logical_blocks_used; + /* The number of block map pages that are allocated */ + block_count_t block_map_data_blocks; + /* The number of journal blocks written but not yet acknowledged */ + block_count_t pending_write_count; + /* The threshold at which slab journal tail blocks will be written out */ + block_count_t slab_journal_commit_threshold; + /* Counters for events in the journal that are reported as statistics */ + struct recovery_journal_statistics events; + /* The locks for each on-disk block */ + struct lock_counter lock_counter; + /* The tail blocks */ + struct recovery_journal_block blocks[]; +}; + +/** + * vdo_get_recovery_journal_block_number() - Get the physical block number for a given sequence + * number. + * @journal: The journal. + * @sequence: The sequence number of the desired block. + * + * Return: The block number corresponding to the sequence number. + */ +static inline physical_block_number_t __must_check +vdo_get_recovery_journal_block_number(const struct recovery_journal *journal, + sequence_number_t sequence) +{ + /* + * Since journal size is a power of two, the block number modulus can just be extracted + * from the low-order bits of the sequence. + */ + return vdo_compute_recovery_journal_block_number(journal->size, sequence); +} + +/** + * vdo_compute_recovery_journal_check_byte() - Compute the check byte for a given sequence number. + * @journal: The journal. + * @sequence: The sequence number. + * + * Return: The check byte corresponding to the sequence number. + */ +static inline u8 __must_check +vdo_compute_recovery_journal_check_byte(const struct recovery_journal *journal, + sequence_number_t sequence) +{ + /* The check byte must change with each trip around the journal. */ + return (((sequence / journal->size) & 0x7F) | 0x80); +} + +int __must_check vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, + nonce_t nonce, struct vdo *vdo, + struct partition *partition, + u64 recovery_count, + block_count_t journal_size, + struct recovery_journal **journal_ptr); + +void vdo_free_recovery_journal(struct recovery_journal *journal); + +void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal, + u64 recovery_count, + sequence_number_t tail, + block_count_t logical_blocks_used, + block_count_t block_map_data_blocks); + +block_count_t __must_check +vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal); + +thread_id_t __must_check vdo_get_recovery_journal_thread_id(struct recovery_journal *journal); + +void vdo_open_recovery_journal(struct recovery_journal *journal, + struct slab_depot *depot, struct block_map *block_map); + +sequence_number_t +vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal); + +block_count_t __must_check vdo_get_recovery_journal_length(block_count_t journal_size); + +struct recovery_journal_state_7_0 __must_check +vdo_record_recovery_journal(const struct recovery_journal *journal); + +void vdo_add_recovery_journal_entry(struct recovery_journal *journal, + struct data_vio *data_vio); + +void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id); + +void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id); + +void vdo_release_journal_entry_lock(struct recovery_journal *journal, + sequence_number_t sequence_number); + +void vdo_drain_recovery_journal(struct recovery_journal *journal, + const struct admin_state_code *operation, + struct vdo_completion *parent); + +void vdo_resume_recovery_journal(struct recovery_journal *journal, + struct vdo_completion *parent); + +block_count_t __must_check +vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal); + +struct recovery_journal_statistics __must_check +vdo_get_recovery_journal_statistics(const struct recovery_journal *journal); + +void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal); + +#endif /* VDO_RECOVERY_JOURNAL_H */ From 4fa98386be2ff32f18b939d3bf5f6841d638fccd Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:08:05 -0500 Subject: [PATCH 0580/1038] dm vdo: add repair of damaged vdo volumes When a vdo is restarted after a crash, it will automatically attempt to recover from its journals. If a vdo encounters an unrecoverable error, it will enter read-only mode. This mode indicates that some previously acknowledged data may have been lost. The vdo may be instructed to rebuild as best it can in order to return to a writable state. Although some data may be lost, this process will ensure that the vdo's own metadata is self-consistent. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/repair.c | 1757 ++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/repair.h | 14 + 2 files changed, 1771 insertions(+) create mode 100644 drivers/md/dm-vdo/repair.c create mode 100644 drivers/md/dm-vdo/repair.h diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c new file mode 100644 index 000000000000..2cf99a7ce958 --- /dev/null +++ b/drivers/md/dm-vdo/repair.c @@ -0,0 +1,1757 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "repair.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "encodings.h" +#include "int-map.h" +#include "io-submitter.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "types.h" +#include "vdo.h" +#include "wait-queue.h" + +/* + * An explicitly numbered block mapping. Numbering the mappings allows them to be sorted by logical + * block number during repair while still preserving the relative order of journal entries with + * the same logical block number. + */ +struct numbered_block_mapping { + struct block_map_slot block_map_slot; + struct block_map_entry block_map_entry; + /* A serial number to use during replay */ + u32 number; +} __packed; + +/* + * The absolute position of an entry in the recovery journal, including the sector number and the + * entry number within the sector. + */ +struct recovery_point { + /* Block sequence number */ + sequence_number_t sequence_number; + /* Sector number */ + u8 sector_count; + /* Entry number */ + journal_entry_count_t entry_count; + /* Whether or not the increment portion of the current entry has been applied */ + bool increment_applied; +}; + +struct repair_completion { + /* The completion header */ + struct vdo_completion completion; + + /* A buffer to hold the data read off disk */ + char *journal_data; + + /* For loading the journal */ + data_vio_count_t vio_count; + data_vio_count_t vios_complete; + struct vio *vios; + + /* The number of entries to be applied to the block map */ + size_t block_map_entry_count; + /* The sequence number of the first valid block for block map recovery */ + sequence_number_t block_map_head; + /* The sequence number of the first valid block for slab journal replay */ + sequence_number_t slab_journal_head; + /* The sequence number of the last valid block of the journal (if known) */ + sequence_number_t tail; + /* + * The highest sequence number of the journal. During recovery (vs read-only rebuild), not + * the same as the tail, since the tail ignores blocks after the first hole. + */ + sequence_number_t highest_tail; + + /* The number of logical blocks currently known to be in use */ + block_count_t logical_blocks_used; + /* The number of block map data blocks known to be allocated */ + block_count_t block_map_data_blocks; + + /* These fields are for playing the journal into the block map */ + /* The entry data for the block map recovery */ + struct numbered_block_mapping *entries; + /* The number of entries in the entry array */ + size_t entry_count; + /* number of pending (non-ready) requests*/ + page_count_t outstanding; + /* number of page completions */ + page_count_t page_count; + bool launching; + /* + * a heap wrapping journal_entries. It re-orders and sorts journal entries in ascending LBN + * order, then original journal order. This permits efficient iteration over the journal + * entries in order. + */ + struct min_heap replay_heap; + /* Fields tracking progress through the journal entries. */ + struct numbered_block_mapping *current_entry; + struct numbered_block_mapping *current_unfetched_entry; + /* Current requested page's PBN */ + physical_block_number_t pbn; + + /* These fields are only used during recovery. */ + /* A location just beyond the last valid entry of the journal */ + struct recovery_point tail_recovery_point; + /* The location of the next recovery journal entry to apply */ + struct recovery_point next_recovery_point; + /* The journal point to give to the next synthesized decref */ + struct journal_point next_journal_point; + /* The number of entries played into slab journals */ + size_t entries_added_to_slab_journals; + + /* These fields are only used during read-only rebuild */ + page_count_t page_to_fetch; + /* the number of leaf pages in the block map */ + page_count_t leaf_pages; + /* the last slot of the block map */ + struct block_map_slot last_slot; + + /* + * The page completions used for playing the journal into the block map, and, during + * read-only rebuild, for rebuilding the reference counts from the block map. + */ + struct vdo_page_completion page_completions[]; +}; + +/* + * This is a min_heap callback function that orders numbered_block_mappings using the + * 'block_map_slot' field as the primary key and the mapping 'number' field as the secondary key. + * Using the mapping number preserves the journal order of entries for the same slot, allowing us + * to sort by slot while still ensuring we replay all entries with the same slot in the exact order + * as they appeared in the journal. + */ +static bool mapping_is_less_than(const void *item1, const void *item2) +{ + const struct numbered_block_mapping *mapping1 = + (const struct numbered_block_mapping *) item1; + const struct numbered_block_mapping *mapping2 = + (const struct numbered_block_mapping *) item2; + + if (mapping1->block_map_slot.pbn != mapping2->block_map_slot.pbn) + return mapping1->block_map_slot.pbn < mapping2->block_map_slot.pbn; + + if (mapping1->block_map_slot.slot != mapping2->block_map_slot.slot) + return mapping1->block_map_slot.slot < mapping2->block_map_slot.slot; + + if (mapping1->number != mapping2->number) + return mapping1->number < mapping2->number; + + return 0; +} + +static void swap_mappings(void *item1, void *item2) +{ + struct numbered_block_mapping *mapping1 = item1; + struct numbered_block_mapping *mapping2 = item2; + + swap(*mapping1, *mapping2); +} + +static const struct min_heap_callbacks repair_min_heap = { + .elem_size = sizeof(struct numbered_block_mapping), + .less = mapping_is_less_than, + .swp = swap_mappings, +}; + +static struct numbered_block_mapping *sort_next_heap_element(struct repair_completion *repair) +{ + struct min_heap *heap = &repair->replay_heap; + struct numbered_block_mapping *last; + + if (heap->nr == 0) + return NULL; + + /* + * Swap the next heap element with the last one on the heap, popping it off the heap, + * restore the heap invariant, and return a pointer to the popped element. + */ + last = &repair->entries[--heap->nr]; + swap_mappings(heap->data, last); + min_heapify(heap, 0, &repair_min_heap); + return last; +} + +/** + * as_repair_completion() - Convert a generic completion to a repair_completion. + * @completion: The completion to convert. + * + * Return: The repair_completion. + */ +static inline struct repair_completion * __must_check +as_repair_completion(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_REPAIR_COMPLETION); + return container_of(completion, struct repair_completion, completion); +} + +static void prepare_repair_completion(struct repair_completion *repair, + vdo_action_fn callback, enum vdo_zone_type zone_type) +{ + struct vdo_completion *completion = &repair->completion; + const struct thread_config *thread_config = &completion->vdo->thread_config; + thread_id_t thread_id; + + /* All blockmap access is done on single thread, so use logical zone 0. */ + thread_id = ((zone_type == VDO_ZONE_TYPE_LOGICAL) ? + thread_config->logical_threads[0] : + thread_config->admin_thread); + vdo_reset_completion(completion); + vdo_set_completion_callback(completion, callback, thread_id); +} + +static void launch_repair_completion(struct repair_completion *repair, + vdo_action_fn callback, enum vdo_zone_type zone_type) +{ + prepare_repair_completion(repair, callback, zone_type); + vdo_launch_completion(&repair->completion); +} + +static void uninitialize_vios(struct repair_completion *repair) +{ + while (repair->vio_count > 0) + free_vio_components(&repair->vios[--repair->vio_count]); + + uds_free(uds_forget(repair->vios)); +} + +static void free_repair_completion(struct repair_completion *repair) +{ + if (repair == NULL) + return; + + /* + * We do this here because this function is the only common bottleneck for all clean up + * paths. + */ + repair->completion.vdo->block_map->zones[0].page_cache.rebuilding = false; + + uninitialize_vios(repair); + uds_free(uds_forget(repair->journal_data)); + uds_free(uds_forget(repair->entries)); + uds_free(repair); +} + +static void finish_repair(struct vdo_completion *completion) +{ + struct vdo_completion *parent = completion->parent; + struct vdo *vdo = completion->vdo; + struct repair_completion *repair = as_repair_completion(completion); + + vdo_assert_on_admin_thread(vdo, __func__); + + if (vdo->load_state != VDO_REBUILD_FOR_UPGRADE) + vdo->states.vdo.complete_recoveries++; + + vdo_initialize_recovery_journal_post_repair(vdo->recovery_journal, + vdo->states.vdo.complete_recoveries, + repair->highest_tail, + repair->logical_blocks_used, + repair->block_map_data_blocks); + free_repair_completion(uds_forget(repair)); + + if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { + uds_log_info("Read-only rebuild complete"); + vdo_launch_completion(parent); + return; + } + + /* FIXME: shouldn't this say either "recovery" or "repair"? */ + uds_log_info("Rebuild complete"); + + /* + * Now that we've freed the repair completion and its vast array of journal entries, we + * can allocate refcounts. + */ + vdo_continue_completion(parent, vdo_allocate_reference_counters(vdo->depot)); +} + +/** + * abort_repair() - Handle a repair error. + * @completion: The repair completion. + */ +static void abort_repair(struct vdo_completion *completion) +{ + struct vdo_completion *parent = completion->parent; + int result = completion->result; + struct repair_completion *repair = as_repair_completion(completion); + + if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) + uds_log_info("Read-only rebuild aborted"); + else + uds_log_warning("Recovery aborted"); + + free_repair_completion(uds_forget(repair)); + vdo_continue_completion(parent, result); +} + +/** + * abort_on_error() - Abort a repair if there is an error. + * @result: The result to check. + * @repair: The repair completion. + * + * Return: true if the result was an error. + */ +static bool __must_check abort_on_error(int result, struct repair_completion *repair) +{ + if (result == VDO_SUCCESS) + return false; + + vdo_fail_completion(&repair->completion, result); + return true; +} + +/** + * drain_slab_depot() - Flush out all dirty refcounts blocks now that they have been rebuilt or + * recovered. + */ +static void drain_slab_depot(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + struct repair_completion *repair = as_repair_completion(completion); + const struct admin_state_code *operation; + + vdo_assert_on_admin_thread(vdo, __func__); + + prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); + if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { + uds_log_info("Saving rebuilt state"); + operation = VDO_ADMIN_STATE_REBUILDING; + } else { + uds_log_info("Replayed %zu journal entries into slab journals", + repair->entries_added_to_slab_journals); + operation = VDO_ADMIN_STATE_RECOVERING; + } + + vdo_drain_slab_depot(vdo->depot, operation, completion); +} + +/** + * flush_block_map_updates() - Flush the block map now that all the reference counts are rebuilt. + * @completion: The repair completion. + * + * This callback is registered in finish_if_done(). + */ +static void flush_block_map_updates(struct vdo_completion *completion) +{ + vdo_assert_on_admin_thread(completion->vdo, __func__); + + uds_log_info("Flushing block map changes"); + prepare_repair_completion(as_repair_completion(completion), drain_slab_depot, + VDO_ZONE_TYPE_ADMIN); + vdo_drain_block_map(completion->vdo->block_map, VDO_ADMIN_STATE_RECOVERING, + completion); +} + +static bool fetch_page(struct repair_completion *repair, + struct vdo_completion *completion); + +/** + * handle_page_load_error() - Handle an error loading a page. + * @completion: The vdo_page_completion. + */ +static void handle_page_load_error(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + repair->outstanding--; + vdo_set_completion_result(&repair->completion, completion->result); + vdo_release_page_completion(completion); + fetch_page(repair, completion); +} + +/** + * unmap_entry() - Unmap an invalid entry and indicate that its page must be written out. + * @page: The page containing the entries + * @completion: The page_completion for writing the page + * @slot: The slot to unmap + */ +static void unmap_entry(struct block_map_page *page, struct vdo_completion *completion, + slot_number_t slot) +{ + page->entries[slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_request_page_write(completion); +} + +/** + * remove_out_of_bounds_entries() - Unmap entries which outside the logical space. + * @page: The page containing the entries + * @completion: The page_completion for writing the page + * @start: The first slot to check + */ +static void remove_out_of_bounds_entries(struct block_map_page *page, + struct vdo_completion *completion, + slot_number_t start) +{ + slot_number_t slot; + + for (slot = start; slot < VDO_BLOCK_MAP_ENTRIES_PER_PAGE; slot++) { + struct data_location mapping = vdo_unpack_block_map_entry(&page->entries[slot]); + + if (vdo_is_mapped_location(&mapping)) + unmap_entry(page, completion, slot); + } +} + +/** + * process_slot() - Update the reference counts for a single entry. + * @page: The page containing the entries + * @completion: The page_completion for writing the page + * @slot: The slot to check + * + * Return: true if the entry was a valid mapping + */ +static bool process_slot(struct block_map_page *page, struct vdo_completion *completion, + slot_number_t slot) +{ + struct slab_depot *depot = completion->vdo->depot; + int result; + struct data_location mapping = vdo_unpack_block_map_entry(&page->entries[slot]); + + if (!vdo_is_valid_location(&mapping)) { + /* This entry is invalid, so remove it from the page. */ + unmap_entry(page, completion, slot); + return false; + } + + if (!vdo_is_mapped_location(&mapping)) + return false; + + + if (mapping.pbn == VDO_ZERO_BLOCK) + return true; + + if (!vdo_is_physical_data_block(depot, mapping.pbn)) { + /* + * This is a nonsense mapping. Remove it from the map so we're at least consistent + * and mark the page dirty. + */ + unmap_entry(page, completion, slot); + return false; + } + + result = vdo_adjust_reference_count_for_rebuild(depot, mapping.pbn, + VDO_JOURNAL_DATA_REMAPPING); + if (result == VDO_SUCCESS) + return true; + + uds_log_error_strerror(result, + "Could not adjust reference count for PBN %llu, slot %u mapped to PBN %llu", + (unsigned long long) vdo_get_block_map_page_pbn(page), + slot, (unsigned long long) mapping.pbn); + unmap_entry(page, completion, slot); + return false; +} + +/** + * rebuild_reference_counts_from_page() - Rebuild reference counts from a block map page. + * @repair: The repair completion. + * @completion: The page completion holding the page. + */ +static void rebuild_reference_counts_from_page(struct repair_completion *repair, + struct vdo_completion *completion) +{ + slot_number_t slot, last_slot; + struct block_map_page *page; + int result; + + result = vdo_get_cached_page(completion, &page); + if (result != VDO_SUCCESS) { + vdo_set_completion_result(&repair->completion, result); + return; + } + + if (!page->header.initialized) + return; + + /* Remove any bogus entries which exist beyond the end of the logical space. */ + if (vdo_get_block_map_page_pbn(page) == repair->last_slot.pbn) { + last_slot = repair->last_slot.slot; + remove_out_of_bounds_entries(page, completion, last_slot); + } else { + last_slot = VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + } + + /* Inform the slab depot of all entries on this page. */ + for (slot = 0; slot < last_slot; slot++) { + if (process_slot(page, completion, slot)) + repair->logical_blocks_used++; + } +} + +/** + * page_loaded() - Process a page which has just been loaded. + * @completion: The vdo_page_completion for the fetched page. + * + * This callback is registered by fetch_page(). + */ +static void page_loaded(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + repair->outstanding--; + rebuild_reference_counts_from_page(repair, completion); + vdo_release_page_completion(completion); + + /* Advance progress to the next page, and fetch the next page we haven't yet requested. */ + fetch_page(repair, completion); +} + +static physical_block_number_t get_pbn_to_fetch(struct repair_completion *repair, + struct block_map *block_map) +{ + physical_block_number_t pbn = VDO_ZERO_BLOCK; + + if (repair->completion.result != VDO_SUCCESS) + return VDO_ZERO_BLOCK; + + while ((pbn == VDO_ZERO_BLOCK) && (repair->page_to_fetch < repair->leaf_pages)) + pbn = vdo_find_block_map_page_pbn(block_map, repair->page_to_fetch++); + + if (vdo_is_physical_data_block(repair->completion.vdo->depot, pbn)) + return pbn; + + vdo_set_completion_result(&repair->completion, VDO_BAD_MAPPING); + return VDO_ZERO_BLOCK; +} + +/** + * fetch_page() - Fetch a page from the block map. + * @repair: The repair_completion. + * @completion: The page completion to use. + * + * Return true if the rebuild is complete + */ +static bool fetch_page(struct repair_completion *repair, + struct vdo_completion *completion) +{ + struct vdo_page_completion *page_completion = (struct vdo_page_completion *) completion; + struct block_map *block_map = repair->completion.vdo->block_map; + physical_block_number_t pbn = get_pbn_to_fetch(repair, block_map); + + if (pbn != VDO_ZERO_BLOCK) { + repair->outstanding++; + /* + * We must set the requeue flag here to ensure that we don't blow the stack if all + * the requested pages are already in the cache or get load errors. + */ + vdo_get_page(page_completion, &block_map->zones[0], pbn, true, repair, + page_loaded, handle_page_load_error, true); + } + + if (repair->outstanding > 0) + return false; + + launch_repair_completion(repair, flush_block_map_updates, VDO_ZONE_TYPE_ADMIN); + return true; +} + +/** + * rebuild_from_leaves() - Rebuild reference counts from the leaf block map pages. + * @completion: The repair completion. + * + * Rebuilds reference counts from the leaf block map pages now that reference counts have been + * rebuilt from the interior tree pages (which have been loaded in the process). This callback is + * registered in rebuild_reference_counts(). + */ +static void rebuild_from_leaves(struct vdo_completion *completion) +{ + page_count_t i; + struct repair_completion *repair = as_repair_completion(completion); + struct block_map *map = completion->vdo->block_map; + + repair->logical_blocks_used = 0; + + /* + * The PBN calculation doesn't work until the tree pages have been loaded, so we can't set + * this value at the start of repair. + */ + repair->leaf_pages = vdo_compute_block_map_page_count(map->entry_count); + repair->last_slot = (struct block_map_slot) { + .slot = map->entry_count % VDO_BLOCK_MAP_ENTRIES_PER_PAGE, + .pbn = vdo_find_block_map_page_pbn(map, repair->leaf_pages - 1), + }; + if (repair->last_slot.slot == 0) + repair->last_slot.slot = VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + for (i = 0; i < repair->page_count; i++) { + if (fetch_page(repair, &repair->page_completions[i].completion)) { + /* + * The rebuild has already moved on, so it isn't safe nor is there a need + * to launch any more fetches. + */ + return; + } + } +} + +/** + * process_entry() - Process a single entry from the block map tree. + * @pbn: A pbn which holds a block map tree page. + * @completion: The parent completion of the traversal. + * + * Implements vdo_entry_callback_fn. + * + * Return: VDO_SUCCESS or an error. + */ +static int process_entry(physical_block_number_t pbn, struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + struct slab_depot *depot = completion->vdo->depot; + int result; + + if ((pbn == VDO_ZERO_BLOCK) || !vdo_is_physical_data_block(depot, pbn)) { + return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + "PBN %llu out of range", + (unsigned long long) pbn); + } + + result = vdo_adjust_reference_count_for_rebuild(depot, pbn, + VDO_JOURNAL_BLOCK_MAP_REMAPPING); + if (result != VDO_SUCCESS) { + return uds_log_error_strerror(result, + "Could not adjust reference count for block map tree PBN %llu", + (unsigned long long) pbn); + } + + repair->block_map_data_blocks++; + return VDO_SUCCESS; +} + +static void rebuild_reference_counts(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + struct vdo *vdo = completion->vdo; + struct vdo_page_cache *cache = &vdo->block_map->zones[0].page_cache; + + /* We must allocate ref_counts before we can rebuild them. */ + if (abort_on_error(vdo_allocate_reference_counters(vdo->depot), repair)) + return; + + /* + * Completion chaining from page cache hits can lead to stack overflow during the rebuild, + * so clear out the cache before this rebuild phase. + */ + if (abort_on_error(vdo_invalidate_page_cache(cache), repair)) + return; + + prepare_repair_completion(repair, rebuild_from_leaves, VDO_ZONE_TYPE_LOGICAL); + vdo_traverse_forest(vdo->block_map, process_entry, completion); +} + +/** + * increment_recovery_point() - Move the given recovery point forward by one entry. + */ +static void increment_recovery_point(struct recovery_point *point) +{ + if (++point->entry_count < RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) + return; + + point->entry_count = 0; + if (point->sector_count < (VDO_SECTORS_PER_BLOCK - 1)) { + point->sector_count++; + return; + } + + point->sequence_number++; + point->sector_count = 1; +} + +/** + * advance_points() - Advance the current recovery and journal points. + * @repair: The repair_completion whose points are to be advanced. + * @entries_per_block: The number of entries in a recovery journal block. + */ +static void advance_points(struct repair_completion *repair, + journal_entry_count_t entries_per_block) +{ + if (!repair->next_recovery_point.increment_applied) { + repair->next_recovery_point.increment_applied = true; + return; + } + + increment_recovery_point(&repair->next_recovery_point); + vdo_advance_journal_point(&repair->next_journal_point, entries_per_block); + repair->next_recovery_point.increment_applied = false; +} + +/** + * before_recovery_point() - Check whether the first point precedes the second point. + * @first: The first recovery point. + * @second: The second recovery point. + * + * Return: true if the first point precedes the second point. + */ +static bool __must_check before_recovery_point(const struct recovery_point *first, + const struct recovery_point *second) +{ + if (first->sequence_number < second->sequence_number) + return true; + + if (first->sequence_number > second->sequence_number) + return false; + + if (first->sector_count < second->sector_count) + return true; + + return ((first->sector_count == second->sector_count) && + (first->entry_count < second->entry_count)); +} + +static struct packed_journal_sector * __must_check get_sector(struct recovery_journal *journal, + char *journal_data, + sequence_number_t sequence, + u8 sector_number) +{ + off_t offset; + + offset = ((vdo_get_recovery_journal_block_number(journal, sequence) * VDO_BLOCK_SIZE) + + (VDO_SECTOR_SIZE * sector_number)); + return (struct packed_journal_sector *) (journal_data + offset); +} + +/** + * get_entry() - Unpack the recovery journal entry associated with the given recovery point. + * @repair: The repair completion. + * @point: The recovery point. + * + * Return: The unpacked contents of the matching recovery journal entry. + */ +static struct recovery_journal_entry get_entry(const struct repair_completion *repair, + const struct recovery_point *point) +{ + struct packed_journal_sector *sector; + + sector = get_sector(repair->completion.vdo->recovery_journal, + repair->journal_data, point->sequence_number, + point->sector_count); + return vdo_unpack_recovery_journal_entry(§or->entries[point->entry_count]); +} + +/** + * validate_recovery_journal_entry() - Validate a recovery journal entry. + * @vdo: The vdo. + * @entry: The entry to validate. + * + * Return: VDO_SUCCESS or an error. + */ +static int validate_recovery_journal_entry(const struct vdo *vdo, + const struct recovery_journal_entry *entry) +{ + if ((entry->slot.pbn >= vdo->states.vdo.config.physical_blocks) || + (entry->slot.slot >= VDO_BLOCK_MAP_ENTRIES_PER_PAGE) || + !vdo_is_valid_location(&entry->mapping) || + !vdo_is_valid_location(&entry->unmapping) || + !vdo_is_physical_data_block(vdo->depot, entry->mapping.pbn) || + !vdo_is_physical_data_block(vdo->depot, entry->unmapping.pbn)) { + return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Invalid entry: %s (%llu, %u) from %llu to %llu is not within bounds", + vdo_get_journal_operation_name(entry->operation), + (unsigned long long) entry->slot.pbn, + entry->slot.slot, + (unsigned long long) entry->unmapping.pbn, + (unsigned long long) entry->mapping.pbn); + } + + if ((entry->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) && + (vdo_is_state_compressed(entry->mapping.state) || + (entry->mapping.pbn == VDO_ZERO_BLOCK) || + (entry->unmapping.state != VDO_MAPPING_STATE_UNMAPPED) || + (entry->unmapping.pbn != VDO_ZERO_BLOCK))) { + return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Invalid entry: %s (%llu, %u) from %llu to %llu is not a valid tree mapping", + vdo_get_journal_operation_name(entry->operation), + (unsigned long long) entry->slot.pbn, + entry->slot.slot, + (unsigned long long) entry->unmapping.pbn, + (unsigned long long) entry->mapping.pbn); + } + + return VDO_SUCCESS; +} + +/** + * add_slab_journal_entries() - Replay recovery journal entries into the slab journals of the + * allocator currently being recovered. + * @completion: The allocator completion. + * + * Waits for slab journal tailblock space when necessary. This method is its own callback. + */ +static void add_slab_journal_entries(struct vdo_completion *completion) +{ + struct recovery_point *recovery_point; + struct repair_completion *repair = completion->parent; + struct vdo *vdo = completion->vdo; + struct recovery_journal *journal = vdo->recovery_journal; + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + /* Get ready in case we need to enqueue again. */ + vdo_prepare_completion(completion, add_slab_journal_entries, + vdo_notify_slab_journals_are_recovered, + completion->callback_thread_id, repair); + for (recovery_point = &repair->next_recovery_point; + before_recovery_point(recovery_point, &repair->tail_recovery_point); + advance_points(repair, journal->entries_per_block)) { + int result; + physical_block_number_t pbn; + struct vdo_slab *slab; + struct recovery_journal_entry entry = get_entry(repair, recovery_point); + bool increment = !repair->next_recovery_point.increment_applied; + + if (increment) { + result = validate_recovery_journal_entry(vdo, &entry); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(vdo, result); + vdo_fail_completion(completion, result); + return; + } + + pbn = entry.mapping.pbn; + } else { + pbn = entry.unmapping.pbn; + } + + if (pbn == VDO_ZERO_BLOCK) + continue; + + slab = vdo_get_slab(vdo->depot, pbn); + if (slab->allocator != allocator) + continue; + + if (!vdo_attempt_replay_into_slab(slab, pbn, entry.operation, increment, + &repair->next_journal_point, + completion)) + return; + + repair->entries_added_to_slab_journals++; + } + + vdo_notify_slab_journals_are_recovered(completion); +} + +/** + * vdo_replay_into_slab_journals() - Replay recovery journal entries in the slab journals of slabs + * owned by a given block_allocator. + * @allocator: The allocator whose slab journals are to be recovered. + * @context: The slab depot load context supplied by a recovery when it loads the depot. + */ +void vdo_replay_into_slab_journals(struct block_allocator *allocator, void *context) +{ + struct vdo_completion *completion = &allocator->completion; + struct repair_completion *repair = context; + struct vdo *vdo = completion->vdo; + + vdo_assert_on_physical_zone_thread(vdo, allocator->zone_number, __func__); + if (repair->entry_count == 0) { + /* there's nothing to replay */ + repair->logical_blocks_used = vdo->recovery_journal->logical_blocks_used; + repair->block_map_data_blocks = vdo->recovery_journal->block_map_data_blocks; + vdo_notify_slab_journals_are_recovered(completion); + return; + } + + repair->next_recovery_point = (struct recovery_point) { + .sequence_number = repair->slab_journal_head, + .sector_count = 1, + .entry_count = 0, + }; + + repair->next_journal_point = (struct journal_point) { + .sequence_number = repair->slab_journal_head, + .entry_count = 0, + }; + + uds_log_info("Replaying entries into slab journals for zone %u", + allocator->zone_number); + completion->parent = repair; + add_slab_journal_entries(completion); +} + +static void load_slab_depot(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + const struct admin_state_code *operation; + + vdo_assert_on_admin_thread(completion->vdo, __func__); + + if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) { + prepare_repair_completion(repair, rebuild_reference_counts, + VDO_ZONE_TYPE_LOGICAL); + operation = VDO_ADMIN_STATE_LOADING_FOR_REBUILD; + } else { + prepare_repair_completion(repair, drain_slab_depot, VDO_ZONE_TYPE_ADMIN); + operation = VDO_ADMIN_STATE_LOADING_FOR_RECOVERY; + } + + vdo_load_slab_depot(completion->vdo->depot, operation, completion, repair); +} + +static void flush_block_map(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + const struct admin_state_code *operation; + + vdo_assert_on_admin_thread(completion->vdo, __func__); + + uds_log_info("Flushing block map changes"); + prepare_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); + operation = (vdo_state_requires_read_only_rebuild(completion->vdo->load_state) ? + VDO_ADMIN_STATE_REBUILDING : + VDO_ADMIN_STATE_RECOVERING); + vdo_drain_block_map(completion->vdo->block_map, operation, completion); +} + +static bool finish_if_done(struct repair_completion *repair) +{ + /* Pages are still being launched or there is still work to do */ + if (repair->launching || (repair->outstanding > 0)) + return false; + + if (repair->completion.result != VDO_SUCCESS) { + page_count_t i; + + for (i = 0; i < repair->page_count; i++) { + struct vdo_page_completion *page_completion = + &repair->page_completions[i]; + + if (page_completion->ready) + vdo_release_page_completion(&page_completion->completion); + } + + vdo_launch_completion(&repair->completion); + return true; + } + + if (repair->current_entry >= repair->entries) + return false; + + launch_repair_completion(repair, flush_block_map, VDO_ZONE_TYPE_ADMIN); + return true; +} + +static void abort_block_map_recovery(struct repair_completion *repair, int result) +{ + vdo_set_completion_result(&repair->completion, result); + finish_if_done(repair); +} + +/** + * find_entry_starting_next_page() - Find the first journal entry after a given entry which is not + * on the same block map page. + * @current_entry: The entry to search from. + * @needs_sort: Whether sorting is needed to proceed. + * + * Return: Pointer to the first later journal entry on a different block map page, or a pointer to + * just before the journal entries if no subsequent entry is on a different block map page. + */ +static struct numbered_block_mapping * +find_entry_starting_next_page(struct repair_completion *repair, + struct numbered_block_mapping *current_entry, bool needs_sort) +{ + size_t current_page; + + /* If current_entry is invalid, return immediately. */ + if (current_entry < repair->entries) + return current_entry; + + current_page = current_entry->block_map_slot.pbn; + + /* Decrement current_entry until it's out of bounds or on a different page. */ + while ((current_entry >= repair->entries) && + (current_entry->block_map_slot.pbn == current_page)) { + if (needs_sort) { + struct numbered_block_mapping *just_sorted_entry = + sort_next_heap_element(repair); + ASSERT_LOG_ONLY(just_sorted_entry < current_entry, + "heap is returning elements in an unexpected order"); + } + + current_entry--; + } + + return current_entry; +} + +/* + * Apply a range of journal entries [starting_entry, ending_entry) journal + * entries to a block map page. + */ +static void apply_journal_entries_to_page(struct block_map_page *page, + struct numbered_block_mapping *starting_entry, + struct numbered_block_mapping *ending_entry) +{ + struct numbered_block_mapping *current_entry = starting_entry; + + while (current_entry != ending_entry) { + page->entries[current_entry->block_map_slot.slot] = current_entry->block_map_entry; + current_entry--; + } +} + +static void recover_ready_pages(struct repair_completion *repair, + struct vdo_completion *completion); + +static void block_map_page_loaded(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion->parent); + + repair->outstanding--; + if (!repair->launching) + recover_ready_pages(repair, completion); +} + +static void handle_block_map_page_load_error(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion->parent); + + repair->outstanding--; + abort_block_map_recovery(repair, completion->result); +} + +static void fetch_block_map_page(struct repair_completion *repair, + struct vdo_completion *completion) +{ + physical_block_number_t pbn; + + if (repair->current_unfetched_entry < repair->entries) + /* Nothing left to fetch. */ + return; + + /* Fetch the next page we haven't yet requested. */ + pbn = repair->current_unfetched_entry->block_map_slot.pbn; + repair->current_unfetched_entry = + find_entry_starting_next_page(repair, repair->current_unfetched_entry, + true); + repair->outstanding++; + vdo_get_page(((struct vdo_page_completion *) completion), + &repair->completion.vdo->block_map->zones[0], pbn, true, + &repair->completion, block_map_page_loaded, + handle_block_map_page_load_error, false); +} + +static struct vdo_page_completion *get_next_page_completion(struct repair_completion *repair, + struct vdo_page_completion *completion) +{ + completion++; + if (completion == (&repair->page_completions[repair->page_count])) + completion = &repair->page_completions[0]; + return completion; +} + +static void recover_ready_pages(struct repair_completion *repair, + struct vdo_completion *completion) +{ + struct vdo_page_completion *page_completion = (struct vdo_page_completion *) completion; + + if (finish_if_done(repair)) + return; + + if (repair->pbn != page_completion->pbn) + return; + + while (page_completion->ready) { + struct numbered_block_mapping *start_of_next_page; + struct block_map_page *page; + int result; + + result = vdo_get_cached_page(completion, &page); + if (result != VDO_SUCCESS) { + abort_block_map_recovery(repair, result); + return; + } + + start_of_next_page = + find_entry_starting_next_page(repair, repair->current_entry, + false); + apply_journal_entries_to_page(page, repair->current_entry, + start_of_next_page); + repair->current_entry = start_of_next_page; + vdo_request_page_write(completion); + vdo_release_page_completion(completion); + + if (finish_if_done(repair)) + return; + + repair->pbn = repair->current_entry->block_map_slot.pbn; + fetch_block_map_page(repair, completion); + page_completion = get_next_page_completion(repair, page_completion); + completion = &page_completion->completion; + } +} + +static void recover_block_map(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + struct vdo *vdo = completion->vdo; + struct numbered_block_mapping *first_sorted_entry; + page_count_t i; + + vdo_assert_on_logical_zone_thread(vdo, 0, __func__); + + /* Suppress block map errors. */ + vdo->block_map->zones[0].page_cache.rebuilding = + vdo_state_requires_read_only_rebuild(vdo->load_state); + + if (repair->block_map_entry_count == 0) { + uds_log_info("Replaying 0 recovery entries into block map"); + uds_free(uds_forget(repair->journal_data)); + launch_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); + return; + } + + /* + * Organize the journal entries into a binary heap so we can iterate over them in sorted + * order incrementally, avoiding an expensive sort call. + */ + repair->replay_heap = (struct min_heap) { + .data = repair->entries, + .nr = repair->block_map_entry_count, + .size = repair->block_map_entry_count, + }; + min_heapify_all(&repair->replay_heap, &repair_min_heap); + + uds_log_info("Replaying %zu recovery entries into block map", + repair->block_map_entry_count); + + repair->current_entry = &repair->entries[repair->block_map_entry_count - 1]; + first_sorted_entry = sort_next_heap_element(repair); + ASSERT_LOG_ONLY(first_sorted_entry == repair->current_entry, + "heap is returning elements in an unexpected order"); + + /* Prevent any page from being processed until all pages have been launched. */ + repair->launching = true; + repair->pbn = repair->current_entry->block_map_slot.pbn; + repair->current_unfetched_entry = repair->current_entry; + for (i = 0; i < repair->page_count; i++) { + if (repair->current_unfetched_entry < repair->entries) + break; + + fetch_block_map_page(repair, &repair->page_completions[i].completion); + } + repair->launching = false; + + /* Process any ready pages. */ + recover_ready_pages(repair, &repair->page_completions[0].completion); +} + +/** + * get_recovery_journal_block_header() - Get the block header for a block at a position in the + * journal data and unpack it. + * @journal: The recovery journal. + * @data: The recovery journal data. + * @sequence: The sequence number. + * + * Return: The unpacked header. + */ +static struct recovery_block_header __must_check +get_recovery_journal_block_header(struct recovery_journal *journal, char *data, + sequence_number_t sequence) +{ + physical_block_number_t pbn = + vdo_get_recovery_journal_block_number(journal, sequence); + char *header = &data[pbn * VDO_BLOCK_SIZE]; + + return vdo_unpack_recovery_block_header((struct packed_journal_header *) header); +} + +/** + * is_valid_recovery_journal_block() - Determine whether the given header describes a valid block + * for the given journal. + * @journal: The journal to use. + * @header: The unpacked block header to check. + * @old_ok: Whether an old format header is valid. + * + * A block is not valid if it is unformatted, or if it is older than the last successful recovery + * or reformat. + * + * Return: True if the header is valid. + */ +static bool __must_check is_valid_recovery_journal_block(const struct recovery_journal *journal, + const struct recovery_block_header *header, + bool old_ok) +{ + if ((header->nonce != journal->nonce) || + (header->recovery_count != journal->recovery_count)) + return false; + + if (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL_2) + return (header->entry_count <= journal->entries_per_block); + + return (old_ok && + (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL) && + (header->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK)); +} + +/** + * is_exact_recovery_journal_block() - Determine whether the given header describes the exact block + * indicated. + * @journal: The journal to use. + * @header: The unpacked block header to check. + * @sequence: The expected sequence number. + * @type: The expected metadata type. + * + * Return: True if the block matches. + */ +static bool __must_check is_exact_recovery_journal_block(const struct recovery_journal *journal, + const struct recovery_block_header *header, + sequence_number_t sequence, + enum vdo_metadata_type type) +{ + return ((header->metadata_type == type) && + (header->sequence_number == sequence) && + (is_valid_recovery_journal_block(journal, header, true))); +} + +/** + * find_recovery_journal_head_and_tail() - Find the tail and head of the journal. + * + * Return: True if there were valid journal blocks. + */ +static bool find_recovery_journal_head_and_tail(struct repair_completion *repair) +{ + struct recovery_journal *journal = repair->completion.vdo->recovery_journal; + bool found_entries = false; + physical_block_number_t i; + + /* + * Ensure that we don't replay old entries since we know the tail recorded in the super + * block must be a lower bound. Not doing so can result in extra data loss by setting the + * tail too early. + */ + repair->highest_tail = journal->tail; + for (i = 0; i < journal->size; i++) { + struct recovery_block_header header = + get_recovery_journal_block_header(journal, repair->journal_data, i); + + if (!is_valid_recovery_journal_block(journal, &header, true)) { + /* This block is old or incorrectly formatted */ + continue; + } + + if (vdo_get_recovery_journal_block_number(journal, header.sequence_number) != i) { + /* This block is in the wrong location */ + continue; + } + + if (header.sequence_number >= repair->highest_tail) { + found_entries = true; + repair->highest_tail = header.sequence_number; + } + + if (!found_entries) + continue; + + if (header.block_map_head > repair->block_map_head) + repair->block_map_head = header.block_map_head; + + if (header.slab_journal_head > repair->slab_journal_head) + repair->slab_journal_head = header.slab_journal_head; + } + + return found_entries; +} + +/** + * unpack_entry() - Unpack a recovery journal entry in either format. + * @vdo: The vdo. + * @packed: The entry to unpack. + * @format: The expected format of the entry. + * @entry: The unpacked entry. + * + * Return: true if the entry should be applied.3 + */ +static bool unpack_entry(struct vdo *vdo, char *packed, enum vdo_metadata_type format, + struct recovery_journal_entry *entry) +{ + if (format == VDO_METADATA_RECOVERY_JOURNAL_2) { + struct packed_recovery_journal_entry *packed_entry = + (struct packed_recovery_journal_entry *) packed; + + *entry = vdo_unpack_recovery_journal_entry(packed_entry); + } else { + physical_block_number_t low32, high4; + + struct packed_recovery_journal_entry_1 *packed_entry = + (struct packed_recovery_journal_entry_1 *) packed; + + if (packed_entry->operation == VDO_JOURNAL_DATA_INCREMENT) + entry->operation = VDO_JOURNAL_DATA_REMAPPING; + else if (packed_entry->operation == VDO_JOURNAL_BLOCK_MAP_INCREMENT) + entry->operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING; + else + return false; + + low32 = __le32_to_cpu(packed_entry->pbn_low_word); + high4 = packed_entry->pbn_high_nibble; + entry->slot = (struct block_map_slot) { + .pbn = ((high4 << 32) | low32), + .slot = (packed_entry->slot_low | (packed_entry->slot_high << 6)), + }; + entry->mapping = vdo_unpack_block_map_entry(&packed_entry->block_map_entry); + entry->unmapping = (struct data_location) { + .pbn = VDO_ZERO_BLOCK, + .state = VDO_MAPPING_STATE_UNMAPPED, + }; + } + + return (validate_recovery_journal_entry(vdo, entry) == VDO_SUCCESS); +} + +/** + * append_sector_entries() - Append an array of recovery journal entries from a journal block + * sector to the array of numbered mappings in the repair completion, + * numbering each entry in the order they are appended. + * @repair: The repair completion. + * @entries: The entries in the sector. + * @format: The format of the sector. + * @entry_count: The number of entries to append. + */ +static void append_sector_entries(struct repair_completion *repair, char *entries, + enum vdo_metadata_type format, + journal_entry_count_t entry_count) +{ + journal_entry_count_t i; + struct vdo *vdo = repair->completion.vdo; + off_t increment = ((format == VDO_METADATA_RECOVERY_JOURNAL_2) + ? sizeof(struct packed_recovery_journal_entry) + : sizeof(struct packed_recovery_journal_entry_1)); + + for (i = 0; i < entry_count; i++, entries += increment) { + struct recovery_journal_entry entry; + + if (!unpack_entry(vdo, entries, format, &entry)) + /* When recovering from read-only mode, ignore damaged entries. */ + continue; + + repair->entries[repair->block_map_entry_count] = + (struct numbered_block_mapping) { + .block_map_slot = entry.slot, + .block_map_entry = vdo_pack_block_map_entry(entry.mapping.pbn, + entry.mapping.state), + .number = repair->block_map_entry_count, + }; + repair->block_map_entry_count++; + } +} + +static journal_entry_count_t entries_per_sector(enum vdo_metadata_type format, + u8 sector_number) +{ + if (format == VDO_METADATA_RECOVERY_JOURNAL_2) + return RECOVERY_JOURNAL_ENTRIES_PER_SECTOR; + + return ((sector_number == (VDO_SECTORS_PER_BLOCK - 1)) + ? RECOVERY_JOURNAL_1_ENTRIES_IN_LAST_SECTOR + : RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR); +} + +static void extract_entries_from_block(struct repair_completion *repair, + struct recovery_journal *journal, + sequence_number_t sequence, + enum vdo_metadata_type format, + journal_entry_count_t entries) +{ + sector_count_t i; + struct recovery_block_header header = + get_recovery_journal_block_header(journal, repair->journal_data, + sequence); + + if (!is_exact_recovery_journal_block(journal, &header, sequence, format)) { + /* This block is invalid, so skip it. */ + return; + } + + entries = min(entries, header.entry_count); + for (i = 1; i < VDO_SECTORS_PER_BLOCK; i++) { + struct packed_journal_sector *sector = + get_sector(journal, repair->journal_data, sequence, i); + journal_entry_count_t sector_entries = + min(entries, entries_per_sector(format, i)); + + if (vdo_is_valid_recovery_journal_sector(&header, sector, i)) { + /* Only extract as many as the block header calls for. */ + append_sector_entries(repair, (char *) sector->entries, format, + min_t(journal_entry_count_t, + sector->entry_count, + sector_entries)); + } + + /* + * Even if the sector wasn't full, count it as full when counting up to the + * entry count the block header claims. + */ + entries -= sector_entries; + } +} + +static int parse_journal_for_rebuild(struct repair_completion *repair) +{ + int result; + sequence_number_t i; + block_count_t count; + enum vdo_metadata_type format; + struct vdo *vdo = repair->completion.vdo; + struct recovery_journal *journal = vdo->recovery_journal; + journal_entry_count_t entries_per_block = journal->entries_per_block; + + format = get_recovery_journal_block_header(journal, repair->journal_data, + repair->highest_tail).metadata_type; + if (format == VDO_METADATA_RECOVERY_JOURNAL) + entries_per_block = RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK; + + /* + * Allocate an array of numbered_block_mapping structures large enough to transcribe every + * packed_recovery_journal_entry from every valid journal block. + */ + count = ((repair->highest_tail - repair->block_map_head + 1) * entries_per_block); + result = uds_allocate(count, struct numbered_block_mapping, __func__, + &repair->entries); + if (result != VDO_SUCCESS) + return result; + + for (i = repair->block_map_head; i <= repair->highest_tail; i++) + extract_entries_from_block(repair, journal, i, format, entries_per_block); + + return VDO_SUCCESS; +} + +static int validate_heads(struct repair_completion *repair) +{ + /* Both reap heads must be behind the tail. */ + if ((repair->block_map_head <= repair->tail) && + (repair->slab_journal_head <= repair->tail)) + return VDO_SUCCESS; + + + return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Journal tail too early. block map head: %llu, slab journal head: %llu, tail: %llu", + (unsigned long long) repair->block_map_head, + (unsigned long long) repair->slab_journal_head, + (unsigned long long) repair->tail); +} + +/** + * extract_new_mappings() - Find all valid new mappings to be applied to the block map. + * + * The mappings are extracted from the journal and stored in a sortable array so that all of the + * mappings to be applied to a given block map page can be done in a single page fetch. + */ +static int extract_new_mappings(struct repair_completion *repair) +{ + int result; + struct vdo *vdo = repair->completion.vdo; + struct recovery_point recovery_point = { + .sequence_number = repair->block_map_head, + .sector_count = 1, + .entry_count = 0, + }; + + /* + * Allocate an array of numbered_block_mapping structs just large enough to transcribe + * every packed_recovery_journal_entry from every valid journal block. + */ + result = uds_allocate(repair->entry_count, struct numbered_block_mapping, + __func__, &repair->entries); + if (result != VDO_SUCCESS) + return result; + + for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); + increment_recovery_point(&recovery_point)) { + struct recovery_journal_entry entry = get_entry(repair, &recovery_point); + + result = validate_recovery_journal_entry(vdo, &entry); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(vdo, result); + return result; + } + + repair->entries[repair->block_map_entry_count] = + (struct numbered_block_mapping) { + .block_map_slot = entry.slot, + .block_map_entry = vdo_pack_block_map_entry(entry.mapping.pbn, + entry.mapping.state), + .number = repair->block_map_entry_count, + }; + repair->block_map_entry_count++; + } + + result = ASSERT((repair->block_map_entry_count <= repair->entry_count), + "approximate entry count is an upper bound"); + if (result != VDO_SUCCESS) + vdo_enter_read_only_mode(vdo, result); + + return result; +} + +/** + * compute_usages() - Compute the lbns in use and block map data blocks counts from the tail of + * the journal. + */ +static noinline int compute_usages(struct repair_completion *repair) +{ + /* + * VDO-5182: function is declared noinline to avoid what is likely a spurious valgrind + * error about this structure being uninitialized. + */ + struct recovery_point recovery_point = { + .sequence_number = repair->tail, + .sector_count = 1, + .entry_count = 0, + }; + + struct vdo *vdo = repair->completion.vdo; + struct recovery_journal *journal = vdo->recovery_journal; + struct recovery_block_header header = + get_recovery_journal_block_header(journal, repair->journal_data, + repair->tail); + + repair->logical_blocks_used = header.logical_blocks_used; + repair->block_map_data_blocks = header.block_map_data_blocks; + + for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); + increment_recovery_point(&recovery_point)) { + struct recovery_journal_entry entry = get_entry(repair, &recovery_point); + int result; + + result = validate_recovery_journal_entry(vdo, &entry); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(vdo, result); + return result; + } + + if (entry.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + repair->block_map_data_blocks++; + continue; + } + + if (vdo_is_mapped_location(&entry.mapping)) + repair->logical_blocks_used++; + + if (vdo_is_mapped_location(&entry.unmapping)) + repair->logical_blocks_used--; + } + + return VDO_SUCCESS; +} + +static int parse_journal_for_recovery(struct repair_completion *repair) +{ + int result; + sequence_number_t i, head; + bool found_entries = false; + struct recovery_journal *journal = repair->completion.vdo->recovery_journal; + + head = min(repair->block_map_head, repair->slab_journal_head); + for (i = head; i <= repair->highest_tail; i++) { + struct recovery_block_header header; + journal_entry_count_t block_entries; + u8 j; + + repair->tail = i; + repair->tail_recovery_point = (struct recovery_point) { + .sequence_number = i, + .sector_count = 0, + .entry_count = 0, + }; + + header = get_recovery_journal_block_header(journal, repair->journal_data, i); + if (header.metadata_type == VDO_METADATA_RECOVERY_JOURNAL) { + /* This is an old format block, so we need to upgrade */ + uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "Recovery journal is in the old format, a read-only rebuild is required."); + vdo_enter_read_only_mode(repair->completion.vdo, + VDO_UNSUPPORTED_VERSION); + return VDO_UNSUPPORTED_VERSION; + } + + if (!is_exact_recovery_journal_block(journal, &header, i, + VDO_METADATA_RECOVERY_JOURNAL_2)) { + /* A bad block header was found so this must be the end of the journal. */ + break; + } + + block_entries = header.entry_count; + + /* Examine each sector in turn to determine the last valid sector. */ + for (j = 1; j < VDO_SECTORS_PER_BLOCK; j++) { + struct packed_journal_sector *sector = + get_sector(journal, repair->journal_data, i, j); + journal_entry_count_t sector_entries = + min_t(journal_entry_count_t, sector->entry_count, + block_entries); + + /* A bad sector means that this block was torn. */ + if (!vdo_is_valid_recovery_journal_sector(&header, sector, j)) + break; + + if (sector_entries > 0) { + found_entries = true; + repair->tail_recovery_point.sector_count++; + repair->tail_recovery_point.entry_count = sector_entries; + block_entries -= sector_entries; + repair->entry_count += sector_entries; + } + + /* If this sector is short, the later sectors can't matter. */ + if ((sector_entries < RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) || + (block_entries == 0)) + break; + } + + /* If this block was not filled, or if it tore, no later block can matter. */ + if ((header.entry_count != journal->entries_per_block) || (block_entries > 0)) + break; + } + + if (!found_entries) + return validate_heads(repair); + + /* Set the tail to the last valid tail block, if there is one. */ + if (repair->tail_recovery_point.sector_count == 0) + repair->tail--; + + result = validate_heads(repair); + if (result != VDO_SUCCESS) + return result; + + uds_log_info("Highest-numbered recovery journal block has sequence number %llu, and the highest-numbered usable block is %llu", + (unsigned long long) repair->highest_tail, + (unsigned long long) repair->tail); + + result = extract_new_mappings(repair); + if (result != VDO_SUCCESS) + return result; + + return compute_usages(repair); +} + +static int parse_journal(struct repair_completion *repair) +{ + if (!find_recovery_journal_head_and_tail(repair)) + return VDO_SUCCESS; + + return (vdo_state_requires_read_only_rebuild(repair->completion.vdo->load_state) ? + parse_journal_for_rebuild(repair) : + parse_journal_for_recovery(repair)); +} + +static void finish_journal_load(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + if (++repair->vios_complete != repair->vio_count) + return; + + uds_log_info("Finished reading recovery journal"); + uninitialize_vios(repair); + prepare_repair_completion(repair, recover_block_map, VDO_ZONE_TYPE_LOGICAL); + vdo_continue_completion(&repair->completion, parse_journal(repair)); +} + +static void handle_journal_load_error(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + /* Preserve the error */ + vdo_set_completion_result(&repair->completion, completion->result); + vio_record_metadata_io_error(as_vio(completion)); + completion->callback(completion); +} + +static void read_journal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + + continue_vio_after_io(vio, finish_journal_load, vdo->thread_config.admin_thread); +} + +/** + * vdo_repair() - Load the recovery journal and then recover or rebuild a vdo. + * @parent: The completion to notify when the operation is complete + */ +void vdo_repair(struct vdo_completion *parent) +{ + int result; + char *ptr; + struct repair_completion *repair; + struct vdo *vdo = parent->vdo; + struct recovery_journal *journal = vdo->recovery_journal; + physical_block_number_t pbn = journal->origin; + block_count_t remaining = journal->size; + block_count_t vio_count = DIV_ROUND_UP(remaining, MAX_BLOCKS_PER_VIO); + page_count_t page_count = min_t(page_count_t, + vdo->device_config->cache_size >> 1, + MAXIMUM_SIMULTANEOUS_VDO_BLOCK_MAP_RESTORATION_READS); + + vdo_assert_on_admin_thread(vdo, __func__); + + if (vdo->load_state == VDO_FORCE_REBUILD) { + uds_log_warning("Rebuilding reference counts to clear read-only mode"); + vdo->states.vdo.read_only_recoveries++; + } else if (vdo->load_state == VDO_REBUILD_FOR_UPGRADE) { + uds_log_warning("Rebuilding reference counts for upgrade"); + } else { + uds_log_warning("Device was dirty, rebuilding reference counts"); + } + + result = uds_allocate_extended(struct repair_completion, page_count, + struct vdo_page_completion, __func__, + &repair); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + vdo_initialize_completion(&repair->completion, vdo, VDO_REPAIR_COMPLETION); + repair->completion.error_handler = abort_repair; + repair->completion.parent = parent; + prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); + repair->page_count = page_count; + + result = uds_allocate(remaining * VDO_BLOCK_SIZE, char, __func__, + &repair->journal_data); + if (abort_on_error(result, repair)) + return; + + result = uds_allocate(vio_count, struct vio, __func__, &repair->vios); + if (abort_on_error(result, repair)) + return; + + ptr = repair->journal_data; + for (repair->vio_count = 0; repair->vio_count < vio_count; repair->vio_count++) { + block_count_t blocks = min_t(block_count_t, remaining, + MAX_BLOCKS_PER_VIO); + + result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL, + VIO_PRIORITY_METADATA, + repair, blocks, ptr, + &repair->vios[repair->vio_count]); + if (abort_on_error(result, repair)) + return; + + ptr += (blocks * VDO_BLOCK_SIZE); + remaining -= blocks; + } + + for (vio_count = 0; + vio_count < repair->vio_count; + vio_count++, pbn += MAX_BLOCKS_PER_VIO) { + submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio, + handle_journal_load_error, REQ_OP_READ); + } +} diff --git a/drivers/md/dm-vdo/repair.h b/drivers/md/dm-vdo/repair.h new file mode 100644 index 000000000000..ff255cf41486 --- /dev/null +++ b/drivers/md/dm-vdo/repair.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_REPAIR_H +#define VDO_REPAIR_H + +#include "types.h" + +void vdo_replay_into_slab_journals(struct block_allocator *allocator, void *context); +void vdo_repair(struct vdo_completion *parent); + +#endif /* VDO_REPAIR_H */ From 06e932fea1aafbaa62c0c872060de5f28f717814 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:08:47 -0500 Subject: [PATCH 0581/1038] dm vdo: add the primary vdo structure Add the data and methods that manage the dm-vdo target itself. This includes the overall state of the target and its threads, the state of the logical volumes, startup, shutdown, and statistics. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/vdo.c | 1784 +++++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/vdo.h | 376 +++++++++ 2 files changed, 2160 insertions(+) create mode 100644 drivers/md/dm-vdo/vdo.c create mode 100644 drivers/md/dm-vdo/vdo.h diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c new file mode 100644 index 000000000000..b494c11e1d64 --- /dev/null +++ b/drivers/md/dm-vdo/vdo.c @@ -0,0 +1,1784 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/* + * This file contains the main entry points for normal operations on a vdo as well as functions for + * constructing and destroying vdo instances (in memory). + */ + +/** + * DOC: + * + * A read_only_notifier has a single completion which is used to perform read-only notifications, + * however, vdo_enter_read_only_mode() may be called from any thread. A pair of fields, protected + * by a spinlock, are used to control the read-only mode entry process. The first field holds the + * read-only error. The second is the state field, which may hold any of the four special values + * enumerated here. + * + * When vdo_enter_read_only_mode() is called from some vdo thread, if the read_only_error field + * already contains an error (i.e. its value is not VDO_SUCCESS), then some other error has already + * initiated the read-only process, and nothing more is done. Otherwise, the new error is stored in + * the read_only_error field, and the state field is consulted. If the state is MAY_NOTIFY, it is + * set to NOTIFYING, and the notification process begins. If the state is MAY_NOT_NOTIFY, then + * notifications are currently disallowed, generally due to the vdo being suspended. In this case, + * the nothing more will be done until the vdo is resumed, at which point the notification will be + * performed. In any other case, the vdo is already read-only, and there is nothing more to do. + */ + +#include "vdo.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" + +#include "block-map.h" +#include "completion.h" +#include "data-vio.h" +#include "dedupe.h" +#include "encodings.h" +#include "funnel-workqueue.h" +#include "io-submitter.h" +#include "logical-zone.h" +#include "packer.h" +#include "physical-zone.h" +#include "pool-sysfs.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "statistics.h" +#include "status-codes.h" +#include "vio.h" + +enum { PARANOID_THREAD_CONSISTENCY_CHECKS = 0 }; + +struct sync_completion { + struct vdo_completion vdo_completion; + struct completion completion; +}; + +/* + * We don't expect this set to ever get really large, so a linked list is adequate. We can use a + * pointer_map if we need to later. + */ +struct device_registry { + struct list_head links; + /* TODO: Convert to rcu per kernel recommendation. */ + rwlock_t lock; +}; + +static struct device_registry registry; + +/** + * vdo_initialize_device_registry_once() - Initialize the necessary structures for the device + * registry. + */ +void vdo_initialize_device_registry_once(void) +{ + INIT_LIST_HEAD(®istry.links); + rwlock_init(®istry.lock); +} + +/** vdo_is_equal() - Implements vdo_filter_fn. */ +static bool vdo_is_equal(struct vdo *vdo, const void *context) +{ + return (vdo == context); +} + +/** + * filter_vdos_locked() - Find a vdo in the registry if it exists there. + * @filter: The filter function to apply to devices. + * @context: A bit of context to provide the filter. + * + * Context: Must be called holding the lock. + * + * Return: the vdo object found, if any. + */ +static struct vdo * __must_check filter_vdos_locked(vdo_filter_fn filter, + const void *context) +{ + struct vdo *vdo; + + list_for_each_entry(vdo, ®istry.links, registration) { + if (filter(vdo, context)) + return vdo; + } + + return NULL; +} + +/** + * vdo_find_matching() - Find and return the first (if any) vdo matching a given filter function. + * @filter: The filter function to apply to vdos. + * @context: A bit of context to provide the filter. + */ +struct vdo *vdo_find_matching(vdo_filter_fn filter, const void *context) +{ + struct vdo *vdo; + + read_lock(®istry.lock); + vdo = filter_vdos_locked(filter, context); + read_unlock(®istry.lock); + + return vdo; +} + +static void start_vdo_request_queue(void *ptr) +{ + struct vdo_thread *thread = vdo_get_work_queue_owner(vdo_get_current_work_queue()); + + uds_register_allocating_thread(&thread->allocating_thread, + &thread->vdo->allocations_allowed); +} + +static void finish_vdo_request_queue(void *ptr) +{ + uds_unregister_allocating_thread(); +} + +#ifdef MODULE +#define MODULE_NAME THIS_MODULE->name +#else +#define MODULE_NAME "dm-vdo" +#endif /* MODULE */ + +static const struct vdo_work_queue_type default_queue_type = { + .start = start_vdo_request_queue, + .finish = finish_vdo_request_queue, + .max_priority = VDO_DEFAULT_Q_MAX_PRIORITY, + .default_priority = VDO_DEFAULT_Q_COMPLETION_PRIORITY, +}; + +static const struct vdo_work_queue_type bio_ack_q_type = { + .start = NULL, + .finish = NULL, + .max_priority = BIO_ACK_Q_MAX_PRIORITY, + .default_priority = BIO_ACK_Q_ACK_PRIORITY, +}; + +static const struct vdo_work_queue_type cpu_q_type = { + .start = NULL, + .finish = NULL, + .max_priority = CPU_Q_MAX_PRIORITY, + .default_priority = CPU_Q_MAX_PRIORITY, +}; + +static void uninitialize_thread_config(struct thread_config *config) +{ + uds_free(uds_forget(config->logical_threads)); + uds_free(uds_forget(config->physical_threads)); + uds_free(uds_forget(config->hash_zone_threads)); + uds_free(uds_forget(config->bio_threads)); + memset(config, 0, sizeof(struct thread_config)); +} + +static void assign_thread_ids(struct thread_config *config, + thread_id_t thread_ids[], zone_count_t count) +{ + zone_count_t zone; + + for (zone = 0; zone < count; zone++) + thread_ids[zone] = config->thread_count++; +} + +/** + * initialize_thread_config() - Initialize the thread mapping + * + * If the logical, physical, and hash zone counts are all 0, a single thread will be shared by all + * three plus the packer and recovery journal. Otherwise, there must be at least one of each type, + * and each will have its own thread, as will the packer and recovery journal. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check initialize_thread_config(struct thread_count_config counts, + struct thread_config *config) +{ + int result; + bool single = ((counts.logical_zones + counts.physical_zones + counts.hash_zones) == 0); + + config->bio_thread_count = counts.bio_threads; + if (single) { + config->logical_zone_count = 1; + config->physical_zone_count = 1; + config->hash_zone_count = 1; + } else { + config->logical_zone_count = counts.logical_zones; + config->physical_zone_count = counts.physical_zones; + config->hash_zone_count = counts.hash_zones; + } + + result = uds_allocate(config->logical_zone_count, thread_id_t, + "logical thread array", &config->logical_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + result = uds_allocate(config->physical_zone_count, thread_id_t, + "physical thread array", &config->physical_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + result = uds_allocate(config->hash_zone_count, thread_id_t, + "hash thread array", &config->hash_zone_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + result = uds_allocate(config->bio_thread_count, thread_id_t, + "bio thread array", &config->bio_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + if (single) { + config->logical_threads[0] = config->thread_count; + config->physical_threads[0] = config->thread_count; + config->hash_zone_threads[0] = config->thread_count++; + } else { + config->admin_thread = config->thread_count; + config->journal_thread = config->thread_count++; + config->packer_thread = config->thread_count++; + assign_thread_ids(config, config->logical_threads, counts.logical_zones); + assign_thread_ids(config, config->physical_threads, counts.physical_zones); + assign_thread_ids(config, config->hash_zone_threads, counts.hash_zones); + } + + config->dedupe_thread = config->thread_count++; + config->bio_ack_thread = + ((counts.bio_ack_threads > 0) ? config->thread_count++ : VDO_INVALID_THREAD_ID); + config->cpu_thread = config->thread_count++; + assign_thread_ids(config, config->bio_threads, counts.bio_threads); + return VDO_SUCCESS; +} + +/** + * vdo_read_geometry_block() - Synchronously read the geometry block from a vdo's underlying block + * device. + * @vdo: The vdo whose geometry is to be read. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check read_geometry_block(struct vdo *vdo) +{ + struct vio *vio; + char *block; + int result; + + result = uds_allocate(VDO_BLOCK_SIZE, u8, __func__, &block); + if (result != VDO_SUCCESS) + return result; + + result = create_metadata_vio(vdo, VIO_TYPE_GEOMETRY, VIO_PRIORITY_HIGH, NULL, + block, &vio); + if (result != VDO_SUCCESS) { + uds_free(block); + return result; + } + + /* + * This is only safe because, having not already loaded the geometry, the vdo's geometry's + * bio_offset field is 0, so the fact that vio_reset_bio() will subtract that offset from + * the supplied pbn is not a problem. + */ + result = vio_reset_bio(vio, block, NULL, REQ_OP_READ, + VDO_GEOMETRY_BLOCK_LOCATION); + if (result != VDO_SUCCESS) { + free_vio(uds_forget(vio)); + uds_free(block); + return result; + } + + bio_set_dev(vio->bio, vdo_get_backing_device(vdo)); + submit_bio_wait(vio->bio); + result = blk_status_to_errno(vio->bio->bi_status); + free_vio(uds_forget(vio)); + if (result != 0) { + uds_log_error_strerror(result, "synchronous read failed"); + uds_free(block); + return -EIO; + } + + result = vdo_parse_geometry_block((u8 *) block, &vdo->geometry); + uds_free(block); + return result; +} + +static bool get_zone_thread_name(const thread_id_t thread_ids[], zone_count_t count, + thread_id_t id, const char *prefix, + char *buffer, size_t buffer_length) +{ + if (id >= thread_ids[0]) { + thread_id_t index = id - thread_ids[0]; + + if (index < count) { + snprintf(buffer, buffer_length, "%s%d", prefix, index); + return true; + } + } + + return false; +} + +/** + * get_thread_name() - Format the name of the worker thread desired to support a given work queue. + * @thread_config: The thread configuration. + * @thread_id: The thread id. + * @buffer: Where to put the formatted name. + * @buffer_length: Size of the output buffer. + * + * The physical layer may add a prefix identifying the product; the output from this function + * should just identify the thread. + */ +static void get_thread_name(const struct thread_config *thread_config, + thread_id_t thread_id, char *buffer, size_t buffer_length) +{ + if (thread_id == thread_config->journal_thread) { + if (thread_config->packer_thread == thread_id) { + /* + * This is the "single thread" config where one thread is used for the + * journal, packer, logical, physical, and hash zones. In that case, it is + * known as the "request queue." + */ + snprintf(buffer, buffer_length, "reqQ"); + return; + } + + snprintf(buffer, buffer_length, "journalQ"); + return; + } else if (thread_id == thread_config->admin_thread) { + /* Theoretically this could be different from the journal thread. */ + snprintf(buffer, buffer_length, "adminQ"); + return; + } else if (thread_id == thread_config->packer_thread) { + snprintf(buffer, buffer_length, "packerQ"); + return; + } else if (thread_id == thread_config->dedupe_thread) { + snprintf(buffer, buffer_length, "dedupeQ"); + return; + } else if (thread_id == thread_config->bio_ack_thread) { + snprintf(buffer, buffer_length, "ackQ"); + return; + } else if (thread_id == thread_config->cpu_thread) { + snprintf(buffer, buffer_length, "cpuQ"); + return; + } + + if (get_zone_thread_name(thread_config->logical_threads, + thread_config->logical_zone_count, + thread_id, "logQ", buffer, buffer_length)) + return; + + if (get_zone_thread_name(thread_config->physical_threads, + thread_config->physical_zone_count, + thread_id, "physQ", buffer, buffer_length)) + return; + + if (get_zone_thread_name(thread_config->hash_zone_threads, + thread_config->hash_zone_count, + thread_id, "hashQ", buffer, buffer_length)) + return; + + if (get_zone_thread_name(thread_config->bio_threads, + thread_config->bio_thread_count, + thread_id, "bioQ", buffer, buffer_length)) + return; + + /* Some sort of misconfiguration? */ + snprintf(buffer, buffer_length, "reqQ%d", thread_id); +} + +/** + * vdo_make_thread() - Construct a single vdo work_queue and its associated thread (or threads for + * round-robin queues). + * @vdo: The vdo which owns the thread. + * @thread_id: The id of the thread to create (as determined by the thread_config). + * @type: The description of the work queue for this thread. + * @queue_count: The number of actual threads/queues contained in the "thread". + * @contexts: An array of queue_count contexts, one for each individual queue; may be NULL. + * + * Each "thread" constructed by this method is represented by a unique thread id in the thread + * config, and completions can be enqueued to the queue and run on the threads comprising this + * entity. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, + const struct vdo_work_queue_type *type, + unsigned int queue_count, void *contexts[]) +{ + struct vdo_thread *thread = &vdo->threads[thread_id]; + char queue_name[MAX_VDO_WORK_QUEUE_NAME_LEN]; + + if (type == NULL) + type = &default_queue_type; + + if (thread->queue != NULL) { + return ASSERT(vdo_work_queue_type_is(thread->queue, type), + "already constructed vdo thread %u is of the correct type", + thread_id); + } + + thread->vdo = vdo; + thread->thread_id = thread_id; + get_thread_name(&vdo->thread_config, thread_id, queue_name, sizeof(queue_name)); + return vdo_make_work_queue(vdo->thread_name_prefix, queue_name, thread, + type, queue_count, contexts, &thread->queue); +} + +/** + * register_vdo() - Register a VDO; it must not already be registered. + * @vdo: The vdo to register. + * + * Return: VDO_SUCCESS or an error. + */ +static int register_vdo(struct vdo *vdo) +{ + int result; + + write_lock(®istry.lock); + result = ASSERT(filter_vdos_locked(vdo_is_equal, vdo) == NULL, + "VDO not already registered"); + if (result == VDO_SUCCESS) { + INIT_LIST_HEAD(&vdo->registration); + list_add_tail(&vdo->registration, ®istry.links); + } + write_unlock(®istry.lock); + + return result; +} + +/** + * initialize_vdo() - Do the portion of initializing a vdo which will clean up after itself on + * error. + * @vdo: The vdo being initialized + * @config: The configuration of the vdo + * @instance: The instance number of the vdo + * @reason: The buffer to hold the failure reason on error + */ +static int initialize_vdo(struct vdo *vdo, struct device_config *config, + unsigned int instance, char **reason) +{ + int result; + zone_count_t i; + + vdo->device_config = config; + vdo->starting_sector_offset = config->owning_target->begin; + vdo->instance = instance; + vdo->allocations_allowed = true; + vdo_set_admin_state_code(&vdo->admin.state, VDO_ADMIN_STATE_NEW); + INIT_LIST_HEAD(&vdo->device_config_list); + vdo_initialize_completion(&vdo->admin.completion, vdo, VDO_ADMIN_COMPLETION); + init_completion(&vdo->admin.callback_sync); + mutex_init(&vdo->stats_mutex); + result = read_geometry_block(vdo); + if (result != VDO_SUCCESS) { + *reason = "Could not load geometry block"; + return result; + } + + result = initialize_thread_config(config->thread_counts, &vdo->thread_config); + if (result != VDO_SUCCESS) { + *reason = "Cannot create thread configuration"; + return result; + } + + uds_log_info("zones: %d logical, %d physical, %d hash; total threads: %d", + config->thread_counts.logical_zones, + config->thread_counts.physical_zones, + config->thread_counts.hash_zones, vdo->thread_config.thread_count); + + /* Compression context storage */ + result = uds_allocate(config->thread_counts.cpu_threads, char *, "LZ4 context", + &vdo->compression_context); + if (result != VDO_SUCCESS) { + *reason = "cannot allocate LZ4 context"; + return result; + } + + for (i = 0; i < config->thread_counts.cpu_threads; i++) { + result = uds_allocate(LZ4_MEM_COMPRESS, char, "LZ4 context", + &vdo->compression_context[i]); + if (result != VDO_SUCCESS) { + *reason = "cannot allocate LZ4 context"; + return result; + } + } + + result = register_vdo(vdo); + if (result != VDO_SUCCESS) { + *reason = "Cannot add VDO to device registry"; + return result; + } + + vdo_set_admin_state_code(&vdo->admin.state, VDO_ADMIN_STATE_INITIALIZED); + return result; +} + +/** + * vdo_make() - Allocate and initialize a vdo. + * @instance: Device instantiation counter. + * @config: The device configuration. + * @reason: The reason for any failure during this call. + * @vdo_ptr: A pointer to hold the created vdo. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make(unsigned int instance, struct device_config *config, char **reason, + struct vdo **vdo_ptr) +{ + int result; + struct vdo *vdo; + + /* VDO-3769 - Set a generic reason so we don't ever return garbage. */ + *reason = "Unspecified error"; + + result = uds_allocate(1, struct vdo, __func__, &vdo); + if (result != UDS_SUCCESS) { + *reason = "Cannot allocate VDO"; + return result; + } + + result = initialize_vdo(vdo, config, instance, reason); + if (result != VDO_SUCCESS) { + vdo_destroy(vdo); + return result; + } + + /* From here on, the caller will clean up if there is an error. */ + *vdo_ptr = vdo; + + snprintf(vdo->thread_name_prefix, sizeof(vdo->thread_name_prefix), + "%s%u", MODULE_NAME, instance); + BUG_ON(vdo->thread_name_prefix[0] == '\0'); + result = uds_allocate(vdo->thread_config.thread_count, + struct vdo_thread, __func__, &vdo->threads); + if (result != VDO_SUCCESS) { + *reason = "Cannot allocate thread structures"; + return result; + } + + result = vdo_make_thread(vdo, vdo->thread_config.admin_thread, + &default_queue_type, 1, NULL); + if (result != VDO_SUCCESS) { + *reason = "Cannot make admin thread"; + return result; + } + + result = vdo_make_flusher(vdo); + if (result != VDO_SUCCESS) { + *reason = "Cannot make flusher zones"; + return result; + } + + result = vdo_make_packer(vdo, DEFAULT_PACKER_BINS, &vdo->packer); + if (result != VDO_SUCCESS) { + *reason = "Cannot make packer zones"; + return result; + } + + BUG_ON(vdo->device_config->logical_block_size <= 0); + BUG_ON(vdo->device_config->owned_device == NULL); + result = make_data_vio_pool(vdo, MAXIMUM_VDO_USER_VIOS, + MAXIMUM_VDO_USER_VIOS * 3 / 4, + &vdo->data_vio_pool); + if (result != VDO_SUCCESS) { + *reason = "Cannot allocate data_vio pool"; + return result; + } + + result = vdo_make_io_submitter(config->thread_counts.bio_threads, + config->thread_counts.bio_rotation_interval, + get_data_vio_pool_request_limit(vdo->data_vio_pool), + vdo, &vdo->io_submitter); + if (result != VDO_SUCCESS) { + *reason = "bio submission initialization failed"; + return result; + } + + if (vdo_uses_bio_ack_queue(vdo)) { + result = vdo_make_thread(vdo, vdo->thread_config.bio_ack_thread, + &bio_ack_q_type, + config->thread_counts.bio_ack_threads, NULL); + if (result != VDO_SUCCESS) { + *reason = "bio ack queue initialization failed"; + return result; + } + } + + result = vdo_make_thread(vdo, vdo->thread_config.cpu_thread, &cpu_q_type, + config->thread_counts.cpu_threads, + (void **) vdo->compression_context); + if (result != VDO_SUCCESS) { + *reason = "CPU queue initialization failed"; + return result; + } + + return VDO_SUCCESS; +} + +static void finish_vdo(struct vdo *vdo) +{ + int i; + + if (vdo->threads == NULL) + return; + + vdo_cleanup_io_submitter(vdo->io_submitter); + vdo_finish_dedupe_index(vdo->hash_zones); + + for (i = 0; i < vdo->thread_config.thread_count; i++) + vdo_finish_work_queue(vdo->threads[i].queue); +} + +/** + * free_listeners() - Free the list of read-only listeners associated with a thread. + * @thread_data: The thread holding the list to free. + */ +static void free_listeners(struct vdo_thread *thread) +{ + struct read_only_listener *listener, *next; + + for (listener = uds_forget(thread->listeners); listener != NULL; listener = next) { + next = uds_forget(listener->next); + uds_free(listener); + } +} + +static void uninitialize_super_block(struct vdo_super_block *super_block) +{ + free_vio_components(&super_block->vio); + uds_free(super_block->buffer); +} + +/** + * unregister_vdo() - Remove a vdo from the device registry. + * @vdo: The vdo to remove. + */ +static void unregister_vdo(struct vdo *vdo) +{ + write_lock(®istry.lock); + if (filter_vdos_locked(vdo_is_equal, vdo) == vdo) + list_del_init(&vdo->registration); + + write_unlock(®istry.lock); +} + +/** + * vdo_destroy() - Destroy a vdo instance. + * @vdo: The vdo to destroy (may be NULL). + */ +void vdo_destroy(struct vdo *vdo) +{ + unsigned int i; + + if (vdo == NULL) + return; + + /* A running VDO should never be destroyed without suspending first. */ + BUG_ON(vdo_get_admin_state(vdo)->normal); + + vdo->allocations_allowed = true; + + /* Stop services that need to gather VDO statistics from the worker threads. */ + if (vdo->sysfs_added) { + init_completion(&vdo->stats_shutdown); + kobject_put(&vdo->stats_directory); + wait_for_completion(&vdo->stats_shutdown); + } + + finish_vdo(vdo); + unregister_vdo(vdo); + free_data_vio_pool(vdo->data_vio_pool); + vdo_free_io_submitter(uds_forget(vdo->io_submitter)); + vdo_free_flusher(uds_forget(vdo->flusher)); + vdo_free_packer(uds_forget(vdo->packer)); + vdo_free_recovery_journal(uds_forget(vdo->recovery_journal)); + vdo_free_slab_depot(uds_forget(vdo->depot)); + vdo_uninitialize_layout(&vdo->layout); + vdo_uninitialize_layout(&vdo->next_layout); + if (vdo->partition_copier) + dm_kcopyd_client_destroy(uds_forget(vdo->partition_copier)); + uninitialize_super_block(&vdo->super_block); + vdo_free_block_map(uds_forget(vdo->block_map)); + vdo_free_hash_zones(uds_forget(vdo->hash_zones)); + vdo_free_physical_zones(uds_forget(vdo->physical_zones)); + vdo_free_logical_zones(uds_forget(vdo->logical_zones)); + + if (vdo->threads != NULL) { + for (i = 0; i < vdo->thread_config.thread_count; i++) { + free_listeners(&vdo->threads[i]); + vdo_free_work_queue(uds_forget(vdo->threads[i].queue)); + } + uds_free(uds_forget(vdo->threads)); + } + + uninitialize_thread_config(&vdo->thread_config); + + if (vdo->compression_context != NULL) { + for (i = 0; i < vdo->device_config->thread_counts.cpu_threads; i++) + uds_free(uds_forget(vdo->compression_context[i])); + + uds_free(uds_forget(vdo->compression_context)); + } + + /* + * The call to kobject_put on the kobj sysfs node will decrement its reference count; when + * the count goes to zero the VDO object will be freed as a side effect. + */ + if (!vdo->sysfs_added) + uds_free(vdo); + else + kobject_put(&vdo->vdo_directory); +} + +static int initialize_super_block(struct vdo *vdo, struct vdo_super_block *super_block) +{ + int result; + + result = uds_allocate(VDO_BLOCK_SIZE, char, "encoded super block", + (char **) &vdo->super_block.buffer); + if (result != VDO_SUCCESS) + return result; + + return allocate_vio_components(vdo, VIO_TYPE_SUPER_BLOCK, + VIO_PRIORITY_METADATA, NULL, 1, + (char *) super_block->buffer, + &vdo->super_block.vio); +} + +/** + * finish_reading_super_block() - Continue after loading the super block. + * @completion: The super block vio. + * + * This callback is registered in vdo_load_super_block(). + */ +static void finish_reading_super_block(struct vdo_completion *completion) +{ + struct vdo_super_block *super_block = + container_of(as_vio(completion), struct vdo_super_block, vio); + + vdo_continue_completion(uds_forget(completion->parent), + vdo_decode_super_block(super_block->buffer)); +} + +/** + * handle_super_block_read_error() - Handle an error reading the super block. + * @completion: The super block vio. + * + * This error handler is registered in vdo_load_super_block(). + */ +static void handle_super_block_read_error(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + finish_reading_super_block(completion); +} + +static void read_super_block_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo_completion *parent = vio->completion.parent; + + continue_vio_after_io(vio, finish_reading_super_block, + parent->callback_thread_id); +} + +/** + * vdo_load_super_block() - Allocate a super block and read its contents from storage. + * @vdo: The vdo containing the super block on disk. + * @parent: The completion to notify after loading the super block. + */ +void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent) +{ + int result; + + result = initialize_super_block(vdo, &vdo->super_block); + if (result != VDO_SUCCESS) { + vdo_continue_completion(parent, result); + return; + } + + vdo->super_block.vio.completion.parent = parent; + submit_metadata_vio(&vdo->super_block.vio, + vdo_get_data_region_start(vdo->geometry), + read_super_block_endio, + handle_super_block_read_error, + REQ_OP_READ); +} + +/** + * pool_stats_release() - Signal that sysfs stats have been shut down. + * @directory: The vdo stats directory. + */ +static void pool_stats_release(struct kobject *directory) +{ + struct vdo *vdo = container_of(directory, struct vdo, stats_directory); + + complete(&vdo->stats_shutdown); +} + +ATTRIBUTE_GROUPS(vdo_pool_stats); +static const struct kobj_type stats_directory_type = { + .release = pool_stats_release, + .sysfs_ops = &vdo_pool_stats_sysfs_ops, + .default_groups = vdo_pool_stats_groups, +}; + +/** + * vdo_add_sysfs_stats_dir() - Add the stats directory to the vdo sysfs directory. + * @vdo: The vdo. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_add_sysfs_stats_dir(struct vdo *vdo) +{ + int result; + + kobject_init(&vdo->stats_directory, &stats_directory_type); + result = kobject_add(&vdo->stats_directory, &vdo->vdo_directory, "statistics"); + if (result != 0) + return VDO_CANT_ADD_SYSFS_NODE; + + return VDO_SUCCESS; +} + +/** + * vdo_get_backing_device() - Get the block device object underlying a vdo. + * @vdo: The vdo. + * + * Return: The vdo's current block device. + */ +struct block_device *vdo_get_backing_device(const struct vdo *vdo) +{ + return vdo->device_config->owned_device->bdev; +} + +/** + * vdo_get_device_name() - Get the device name associated with the vdo target. + * @target: The target device interface. + * + * Return: The block device name. + */ +const char *vdo_get_device_name(const struct dm_target *target) +{ + return dm_device_name(dm_table_get_md(target->table)); +} + +/** + * vdo_synchronous_flush() - Issue a flush request and wait for it to complete. + * @vdo: The vdo. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_synchronous_flush(struct vdo *vdo) +{ + int result; + struct bio bio; + + bio_init(&bio, vdo_get_backing_device(vdo), 0, 0, REQ_OP_WRITE | REQ_PREFLUSH); + submit_bio_wait(&bio); + result = blk_status_to_errno(bio.bi_status); + + atomic64_inc(&vdo->stats.flush_out); + if (result != 0) { + uds_log_error_strerror(result, "synchronous flush failed"); + result = -EIO; + } + + bio_uninit(&bio); + return result; +} + +/** + * vdo_get_state() - Get the current state of the vdo. + * @vdo: The vdo. + + * Context: This method may be called from any thread. + * + * Return: The current state of the vdo. + */ +enum vdo_state vdo_get_state(const struct vdo *vdo) +{ + enum vdo_state state = atomic_read(&vdo->state); + + /* pairs with barriers where state field is changed */ + smp_rmb(); + return state; +} + +/** + * vdo_set_state() - Set the current state of the vdo. + * @vdo: The vdo whose state is to be set. + * @state: The new state of the vdo. + * + * Context: This method may be called from any thread. + */ +void vdo_set_state(struct vdo *vdo, enum vdo_state state) +{ + /* pairs with barrier in vdo_get_state */ + smp_wmb(); + atomic_set(&vdo->state, state); +} + +/** + * vdo_get_admin_state() - Get the admin state of the vdo. + * @vdo: The vdo. + * + * Return: The code for the vdo's current admin state. + */ +const struct admin_state_code *vdo_get_admin_state(const struct vdo *vdo) +{ + return vdo_get_admin_state_code(&vdo->admin.state); +} + +/** + * record_vdo() - Record the state of the VDO for encoding in the super block. + */ +static void record_vdo(struct vdo *vdo) +{ + /* This is for backwards compatibility. */ + vdo->states.unused = vdo->geometry.unused; + vdo->states.vdo.state = vdo_get_state(vdo); + vdo->states.block_map = vdo_record_block_map(vdo->block_map); + vdo->states.recovery_journal = vdo_record_recovery_journal(vdo->recovery_journal); + vdo->states.slab_depot = vdo_record_slab_depot(vdo->depot); + vdo->states.layout = vdo->layout; +} + +/** + * continue_super_block_parent() - Continue the parent of a super block save operation. + * @completion: The super block vio. + * + * This callback is registered in vdo_save_components(). + */ +static void continue_super_block_parent(struct vdo_completion *completion) +{ + vdo_continue_completion(uds_forget(completion->parent), completion->result); +} + +/** + * handle_save_error() - Log a super block save error. + * @completion: The super block vio. + * + * This error handler is registered in vdo_save_components(). + */ +static void handle_save_error(struct vdo_completion *completion) +{ + struct vdo_super_block *super_block = + container_of(as_vio(completion), struct vdo_super_block, vio); + + vio_record_metadata_io_error(&super_block->vio); + uds_log_error_strerror(completion->result, "super block save failed"); + /* + * Mark the super block as unwritable so that we won't attempt to write it again. This + * avoids the case where a growth attempt fails writing the super block with the new size, + * but the subsequent attempt to write out the read-only state succeeds. In this case, + * writes which happened just before the suspend would not be visible if the VDO is + * restarted without rebuilding, but, after a read-only rebuild, the effects of those + * writes would reappear. + */ + super_block->unwritable = true; + completion->callback(completion); +} + +static void super_block_write_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo_completion *parent = vio->completion.parent; + + continue_vio_after_io(vio, continue_super_block_parent, + parent->callback_thread_id); +} + +/** + * vdo_save_components() - Encode the vdo and save the super block asynchronously. + * @vdo: The vdo whose state is being saved. + * @parent: The completion to notify when the save is complete. + */ +void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent) +{ + struct vdo_super_block *super_block = &vdo->super_block; + + if (super_block->unwritable) { + vdo_continue_completion(parent, VDO_READ_ONLY); + return; + } + + if (super_block->vio.completion.parent != NULL) { + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + return; + } + + record_vdo(vdo); + + vdo_encode_super_block(super_block->buffer, &vdo->states); + super_block->vio.completion.parent = parent; + super_block->vio.completion.callback_thread_id = parent->callback_thread_id; + submit_metadata_vio(&super_block->vio, + vdo_get_data_region_start(vdo->geometry), + super_block_write_endio, handle_save_error, + REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA); +} + +/** + * vdo_register_read_only_listener() - Register a listener to be notified when the VDO goes + * read-only. + * @vdo: The vdo to register with. + * @listener: The object to notify. + * @notification: The function to call to send the notification. + * @thread_id: The id of the thread on which to send the notification. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_register_read_only_listener(struct vdo *vdo, void *listener, + vdo_read_only_notification_fn notification, + thread_id_t thread_id) +{ + struct vdo_thread *thread = &vdo->threads[thread_id]; + struct read_only_listener *read_only_listener; + int result; + + result = ASSERT(thread_id != vdo->thread_config.dedupe_thread, + "read only listener not registered on dedupe thread"); + if (result != VDO_SUCCESS) + return result; + + result = uds_allocate(1, struct read_only_listener, __func__, + &read_only_listener); + if (result != VDO_SUCCESS) + return result; + + *read_only_listener = (struct read_only_listener) { + .listener = listener, + .notify = notification, + .next = thread->listeners, + }; + + thread->listeners = read_only_listener; + return VDO_SUCCESS; +} + +/** + * notify_vdo_of_read_only_mode() - Notify a vdo that it is going read-only. + * @listener: The vdo. + * @parent: The completion to notify in order to acknowledge the notification. + * + * This will save the read-only state to the super block. + * + * Implements vdo_read_only_notification_fn. + */ +static void notify_vdo_of_read_only_mode(void *listener, struct vdo_completion *parent) +{ + struct vdo *vdo = listener; + + if (vdo_in_read_only_mode(vdo)) + vdo_finish_completion(parent); + + vdo_set_state(vdo, VDO_READ_ONLY_MODE); + vdo_save_components(vdo, parent); +} + +/** + * vdo_enable_read_only_entry() - Enable a vdo to enter read-only mode on errors. + * @vdo: The vdo to enable. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_enable_read_only_entry(struct vdo *vdo) +{ + thread_id_t id; + bool is_read_only = vdo_in_read_only_mode(vdo); + struct read_only_notifier *notifier = &vdo->read_only_notifier; + + if (is_read_only) { + notifier->read_only_error = VDO_READ_ONLY; + notifier->state = NOTIFIED; + } else { + notifier->state = MAY_NOT_NOTIFY; + } + + spin_lock_init(¬ifier->lock); + vdo_initialize_completion(¬ifier->completion, vdo, + VDO_READ_ONLY_MODE_COMPLETION); + + for (id = 0; id < vdo->thread_config.thread_count; id++) + vdo->threads[id].is_read_only = is_read_only; + + return vdo_register_read_only_listener(vdo, vdo, notify_vdo_of_read_only_mode, + vdo->thread_config.admin_thread); +} + +/** + * vdo_wait_until_not_entering_read_only_mode() - Wait until no read-only notifications are in + * progress and prevent any subsequent + * notifications. + * @parent: The completion to notify when no threads are entering read-only mode. + * + * Notifications may be re-enabled by calling vdo_allow_read_only_mode_entry(). + */ +void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent) +{ + struct vdo *vdo = parent->vdo; + struct read_only_notifier *notifier = &vdo->read_only_notifier; + + vdo_assert_on_admin_thread(vdo, __func__); + + if (notifier->waiter != NULL) { + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + return; + } + + spin_lock(¬ifier->lock); + if (notifier->state == NOTIFYING) + notifier->waiter = parent; + else if (notifier->state == MAY_NOTIFY) + notifier->state = MAY_NOT_NOTIFY; + spin_unlock(¬ifier->lock); + + if (notifier->waiter == NULL) { + /* + * A notification was not in progress, and now they are + * disallowed. + */ + vdo_launch_completion(parent); + return; + } +} + +/** + * as_notifier() - Convert a generic vdo_completion to a read_only_notifier. + * @completion: The completion to convert. + * + * Return: The completion as a read_only_notifier. + */ +static inline struct read_only_notifier *as_notifier(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_READ_ONLY_MODE_COMPLETION); + return container_of(completion, struct read_only_notifier, completion); +} + +/** + * finish_entering_read_only_mode() - Complete the process of entering read only mode. + * @completion: The read-only mode completion. + */ +static void finish_entering_read_only_mode(struct vdo_completion *completion) +{ + struct read_only_notifier *notifier = as_notifier(completion); + + vdo_assert_on_admin_thread(completion->vdo, __func__); + + spin_lock(¬ifier->lock); + notifier->state = NOTIFIED; + spin_unlock(¬ifier->lock); + + if (notifier->waiter != NULL) + vdo_continue_completion(uds_forget(notifier->waiter), + completion->result); +} + +/** + * make_thread_read_only() - Inform each thread that the VDO is in read-only mode. + * @completion: The read-only mode completion. + */ +static void make_thread_read_only(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + thread_id_t thread_id = completion->callback_thread_id; + struct read_only_notifier *notifier = as_notifier(completion); + struct read_only_listener *listener = completion->parent; + + if (listener == NULL) { + /* This is the first call on this thread */ + struct vdo_thread *thread = &vdo->threads[thread_id]; + + thread->is_read_only = true; + listener = thread->listeners; + if (thread_id == 0) + uds_log_error_strerror(READ_ONCE(notifier->read_only_error), + "Unrecoverable error, entering read-only mode"); + } else { + /* We've just finished notifying a listener */ + listener = listener->next; + } + + if (listener != NULL) { + /* We have a listener to notify */ + vdo_prepare_completion(completion, make_thread_read_only, + make_thread_read_only, thread_id, + listener); + listener->notify(listener->listener, completion); + return; + } + + /* We're done with this thread */ + if (++thread_id == vdo->thread_config.dedupe_thread) { + /* + * We don't want to notify the dedupe thread since it may be + * blocked rebuilding the index. + */ + thread_id++; + } + + if (thread_id >= vdo->thread_config.thread_count) { + /* There are no more threads */ + vdo_prepare_completion(completion, finish_entering_read_only_mode, + finish_entering_read_only_mode, + vdo->thread_config.admin_thread, NULL); + } else { + vdo_prepare_completion(completion, make_thread_read_only, + make_thread_read_only, thread_id, NULL); + } + + vdo_launch_completion(completion); +} + +/** + * vdo_allow_read_only_mode_entry() - Allow the notifier to put the VDO into read-only mode, + * reversing the effects of + * vdo_wait_until_not_entering_read_only_mode(). + * @parent: The object to notify once the operation is complete. + * + * If some thread tried to put the vdo into read-only mode while notifications were disallowed, it + * will be done when this method is called. If that happens, the parent will not be notified until + * the vdo has actually entered read-only mode and attempted to save the super block. + * + * Context: This method may only be called from the admin thread. + */ +void vdo_allow_read_only_mode_entry(struct vdo_completion *parent) +{ + struct vdo *vdo = parent->vdo; + struct read_only_notifier *notifier = &vdo->read_only_notifier; + + vdo_assert_on_admin_thread(vdo, __func__); + + if (notifier->waiter != NULL) { + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + return; + } + + spin_lock(¬ifier->lock); + if (notifier->state == MAY_NOT_NOTIFY) { + if (notifier->read_only_error == VDO_SUCCESS) { + notifier->state = MAY_NOTIFY; + } else { + notifier->state = NOTIFYING; + notifier->waiter = parent; + } + } + spin_unlock(¬ifier->lock); + + if (notifier->waiter == NULL) { + /* We're done */ + vdo_launch_completion(parent); + return; + } + + /* Do the pending notification. */ + make_thread_read_only(¬ifier->completion); +} + +/** + * vdo_enter_read_only_mode() - Put a VDO into read-only mode and save the read-only state in the + * super block. + * @vdo: The vdo. + * @error_code: The error which caused the VDO to enter read-only mode. + * + * This method is a no-op if the VDO is already read-only. + */ +void vdo_enter_read_only_mode(struct vdo *vdo, int error_code) +{ + bool notify = false; + thread_id_t thread_id = vdo_get_callback_thread_id(); + struct read_only_notifier *notifier = &vdo->read_only_notifier; + struct vdo_thread *thread; + + if (thread_id != VDO_INVALID_THREAD_ID) { + thread = &vdo->threads[thread_id]; + if (thread->is_read_only) { + /* This thread has already gone read-only. */ + return; + } + + /* Record for this thread that the VDO is read-only. */ + thread->is_read_only = true; + } + + spin_lock(¬ifier->lock); + if (notifier->read_only_error == VDO_SUCCESS) { + WRITE_ONCE(notifier->read_only_error, error_code); + if (notifier->state == MAY_NOTIFY) { + notifier->state = NOTIFYING; + notify = true; + } + } + spin_unlock(¬ifier->lock); + + if (!notify) { + /* The notifier is already aware of a read-only error */ + return; + } + + /* Initiate a notification starting on the lowest numbered thread. */ + vdo_launch_completion_callback(¬ifier->completion, make_thread_read_only, 0); +} + +/** + * vdo_is_read_only() - Check whether the VDO is read-only. + * @vdo: The vdo. + * + * Return: true if the vdo is read-only. + * + * This method may be called from any thread, as opposed to examining the VDO's state field which + * is only safe to check from the admin thread. + */ +bool vdo_is_read_only(struct vdo *vdo) +{ + return vdo->threads[vdo_get_callback_thread_id()].is_read_only; +} + +/** + * vdo_in_read_only_mode() - Check whether a vdo is in read-only mode. + * @vdo: The vdo to query. + * + * Return: true if the vdo is in read-only mode. + */ +bool vdo_in_read_only_mode(const struct vdo *vdo) +{ + return (vdo_get_state(vdo) == VDO_READ_ONLY_MODE); +} + +/** + * vdo_in_recovery_mode() - Check whether the vdo is in recovery mode. + * @vdo: The vdo to query. + * + * Return: true if the vdo is in recovery mode. + */ +bool vdo_in_recovery_mode(const struct vdo *vdo) +{ + return (vdo_get_state(vdo) == VDO_RECOVERING); +} + +/** + * vdo_enter_recovery_mode() - Put the vdo into recovery mode. + * @vdo: The vdo. + */ +void vdo_enter_recovery_mode(struct vdo *vdo) +{ + vdo_assert_on_admin_thread(vdo, __func__); + + if (vdo_in_read_only_mode(vdo)) + return; + + uds_log_info("Entering recovery mode"); + vdo_set_state(vdo, VDO_RECOVERING); +} + +/** + * complete_synchronous_action() - Signal the waiting thread that a synchronous action is complete. + * @completion: The sync completion. + */ +static void complete_synchronous_action(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_SYNC_COMPLETION); + complete(&(container_of(completion, struct sync_completion, + vdo_completion)->completion)); +} + +/** + * perform_synchronous_action() - Launch an action on a VDO thread and wait for it to complete. + * @vdo: The vdo. + * @action: The callback to launch. + * @thread_id: The thread on which to run the action. + * @parent: The parent of the sync completion (may be NULL). + */ +static int perform_synchronous_action(struct vdo *vdo, vdo_action_fn action, + thread_id_t thread_id, void *parent) +{ + struct sync_completion sync; + + vdo_initialize_completion(&sync.vdo_completion, vdo, VDO_SYNC_COMPLETION); + init_completion(&sync.completion); + sync.vdo_completion.parent = parent; + vdo_launch_completion_callback(&sync.vdo_completion, action, thread_id); + wait_for_completion(&sync.completion); + return sync.vdo_completion.result; +} + +/** + * set_compression_callback() - Callback to turn compression on or off. + * @completion: The completion. + */ +static void set_compression_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + bool *enable = completion->parent; + bool was_enabled = vdo_get_compressing(vdo); + + if (*enable != was_enabled) { + WRITE_ONCE(vdo->compressing, *enable); + if (was_enabled) { + /* Signal the packer to flush since compression has been disabled. */ + vdo_flush_packer(vdo->packer); + } + } + + uds_log_info("compression is %s", (*enable ? "enabled" : "disabled")); + *enable = was_enabled; + complete_synchronous_action(completion); +} + +/** + * vdo_set_compressing() - Turn compression on or off. + * @vdo: The vdo. + * @enable: Whether to enable or disable compression. + * + * Return: Whether compression was previously on or off. + */ +bool vdo_set_compressing(struct vdo *vdo, bool enable) +{ + perform_synchronous_action(vdo, set_compression_callback, + vdo->thread_config.packer_thread, + &enable); + return enable; +} + +/** + * vdo_get_compressing() - Get whether compression is enabled in a vdo. + * @vdo: The vdo. + * + * Return: State of compression. + */ +bool vdo_get_compressing(struct vdo *vdo) +{ + return READ_ONCE(vdo->compressing); +} + +static size_t get_block_map_cache_size(const struct vdo *vdo) +{ + return ((size_t) vdo->device_config->cache_size) * VDO_BLOCK_SIZE; +} + +static struct error_statistics __must_check get_vdo_error_statistics(const struct vdo *vdo) +{ + /* + * The error counts can be incremented from arbitrary threads and so must be incremented + * atomically, but they are just statistics with no semantics that could rely on memory + * order, so unfenced reads are sufficient. + */ + const struct atomic_statistics *atoms = &vdo->stats; + + return (struct error_statistics) { + .invalid_advice_pbn_count = atomic64_read(&atoms->invalid_advice_pbn_count), + .no_space_error_count = atomic64_read(&atoms->no_space_error_count), + .read_only_error_count = atomic64_read(&atoms->read_only_error_count), + }; +} + +static void copy_bio_stat(struct bio_stats *b, const struct atomic_bio_stats *a) +{ + b->read = atomic64_read(&a->read); + b->write = atomic64_read(&a->write); + b->discard = atomic64_read(&a->discard); + b->flush = atomic64_read(&a->flush); + b->empty_flush = atomic64_read(&a->empty_flush); + b->fua = atomic64_read(&a->fua); +} + +static struct bio_stats subtract_bio_stats(struct bio_stats minuend, + struct bio_stats subtrahend) +{ + return (struct bio_stats) { + .read = minuend.read - subtrahend.read, + .write = minuend.write - subtrahend.write, + .discard = minuend.discard - subtrahend.discard, + .flush = minuend.flush - subtrahend.flush, + .empty_flush = minuend.empty_flush - subtrahend.empty_flush, + .fua = minuend.fua - subtrahend.fua, + }; +} + +/** + * vdo_get_physical_blocks_allocated() - Get the number of physical blocks in use by user data. + * @vdo: The vdo. + * + * Return: The number of blocks allocated for user data. + */ +static block_count_t __must_check vdo_get_physical_blocks_allocated(const struct vdo *vdo) +{ + return (vdo_get_slab_depot_allocated_blocks(vdo->depot) - + vdo_get_journal_block_map_data_blocks_used(vdo->recovery_journal)); +} + +/** + * vdo_get_physical_blocks_overhead() - Get the number of physical blocks used by vdo metadata. + * @vdo: The vdo. + * + * Return: The number of overhead blocks. + */ +static block_count_t __must_check vdo_get_physical_blocks_overhead(const struct vdo *vdo) +{ + /* + * config.physical_blocks is mutated during resize and is in a packed structure, + * but resize runs on admin thread. + * TODO: Verify that this is always safe. + */ + return (vdo->states.vdo.config.physical_blocks - + vdo_get_slab_depot_data_blocks(vdo->depot) + + vdo_get_journal_block_map_data_blocks_used(vdo->recovery_journal)); +} + +static const char *vdo_describe_state(enum vdo_state state) +{ + /* These strings should all fit in the 15 chars of VDOStatistics.mode. */ + switch (state) { + case VDO_RECOVERING: + return "recovering"; + + case VDO_READ_ONLY_MODE: + return "read-only"; + + default: + return "normal"; + } +} + +/** + * get_vdo_statistics() - Populate a vdo_statistics structure on the admin thread. + * @vdo: The vdo. + * @stats: The statistics structure to populate. + */ +static void get_vdo_statistics(const struct vdo *vdo, struct vdo_statistics *stats) +{ + struct recovery_journal *journal = vdo->recovery_journal; + enum vdo_state state = vdo_get_state(vdo); + + vdo_assert_on_admin_thread(vdo, __func__); + + /* start with a clean slate */ + memset(stats, 0, sizeof(struct vdo_statistics)); + + /* + * These are immutable properties of the vdo object, so it is safe to query them from any + * thread. + */ + stats->version = STATISTICS_VERSION; + stats->logical_blocks = vdo->states.vdo.config.logical_blocks; + /* + * config.physical_blocks is mutated during resize and is in a packed structure, but resize + * runs on the admin thread. + * TODO: verify that this is always safe + */ + stats->physical_blocks = vdo->states.vdo.config.physical_blocks; + stats->block_size = VDO_BLOCK_SIZE; + stats->complete_recoveries = vdo->states.vdo.complete_recoveries; + stats->read_only_recoveries = vdo->states.vdo.read_only_recoveries; + stats->block_map_cache_size = get_block_map_cache_size(vdo); + + /* The callees are responsible for thread-safety. */ + stats->data_blocks_used = vdo_get_physical_blocks_allocated(vdo); + stats->overhead_blocks_used = vdo_get_physical_blocks_overhead(vdo); + stats->logical_blocks_used = vdo_get_recovery_journal_logical_blocks_used(journal); + vdo_get_slab_depot_statistics(vdo->depot, stats); + stats->journal = vdo_get_recovery_journal_statistics(journal); + stats->packer = vdo_get_packer_statistics(vdo->packer); + stats->block_map = vdo_get_block_map_statistics(vdo->block_map); + vdo_get_dedupe_statistics(vdo->hash_zones, stats); + stats->errors = get_vdo_error_statistics(vdo); + stats->in_recovery_mode = (state == VDO_RECOVERING); + snprintf(stats->mode, sizeof(stats->mode), "%s", vdo_describe_state(state)); + + stats->instance = vdo->instance; + stats->current_vios_in_progress = get_data_vio_pool_active_requests(vdo->data_vio_pool); + stats->max_vios = get_data_vio_pool_maximum_requests(vdo->data_vio_pool); + + stats->flush_out = atomic64_read(&vdo->stats.flush_out); + stats->logical_block_size = vdo->device_config->logical_block_size; + copy_bio_stat(&stats->bios_in, &vdo->stats.bios_in); + copy_bio_stat(&stats->bios_in_partial, &vdo->stats.bios_in_partial); + copy_bio_stat(&stats->bios_out, &vdo->stats.bios_out); + copy_bio_stat(&stats->bios_meta, &vdo->stats.bios_meta); + copy_bio_stat(&stats->bios_journal, &vdo->stats.bios_journal); + copy_bio_stat(&stats->bios_page_cache, &vdo->stats.bios_page_cache); + copy_bio_stat(&stats->bios_out_completed, &vdo->stats.bios_out_completed); + copy_bio_stat(&stats->bios_meta_completed, &vdo->stats.bios_meta_completed); + copy_bio_stat(&stats->bios_journal_completed, + &vdo->stats.bios_journal_completed); + copy_bio_stat(&stats->bios_page_cache_completed, + &vdo->stats.bios_page_cache_completed); + copy_bio_stat(&stats->bios_acknowledged, &vdo->stats.bios_acknowledged); + copy_bio_stat(&stats->bios_acknowledged_partial, &vdo->stats.bios_acknowledged_partial); + stats->bios_in_progress = + subtract_bio_stats(stats->bios_in, stats->bios_acknowledged); + uds_get_memory_stats(&stats->memory_usage.bytes_used, + &stats->memory_usage.peak_bytes_used); +} + +/** + * vdo_fetch_statistics_callback() - Action to populate a vdo_statistics + * structure on the admin thread. + * @completion: The completion. + * + * This callback is registered in vdo_fetch_statistics(). + */ +static void vdo_fetch_statistics_callback(struct vdo_completion *completion) +{ + get_vdo_statistics(completion->vdo, completion->parent); + complete_synchronous_action(completion); +} + +/** + * vdo_fetch_statistics() - Fetch statistics on the correct thread. + * @vdo: The vdo. + * @stats: The vdo statistics are returned here. + */ +void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats) +{ + perform_synchronous_action(vdo, vdo_fetch_statistics_callback, + vdo->thread_config.admin_thread, stats); +} + +/** + * vdo_get_callback_thread_id() - Get the id of the callback thread on which a completion is + * currently running. + * + * Return: The current thread ID, or -1 if no such thread. + */ +thread_id_t vdo_get_callback_thread_id(void) +{ + struct vdo_work_queue *queue = vdo_get_current_work_queue(); + struct vdo_thread *thread; + thread_id_t thread_id; + + if (queue == NULL) + return VDO_INVALID_THREAD_ID; + + thread = vdo_get_work_queue_owner(queue); + thread_id = thread->thread_id; + + if (PARANOID_THREAD_CONSISTENCY_CHECKS) { + BUG_ON(thread_id >= thread->vdo->thread_config.thread_count); + BUG_ON(thread != &thread->vdo->threads[thread_id]); + } + + return thread_id; +} + +/** + * vdo_dump_status() - Dump status information about a vdo to the log for debugging. + * @vdo: The vdo to dump. + */ +void vdo_dump_status(const struct vdo *vdo) +{ + zone_count_t zone; + + vdo_dump_flusher(vdo->flusher); + vdo_dump_recovery_journal_statistics(vdo->recovery_journal); + vdo_dump_packer(vdo->packer); + vdo_dump_slab_depot(vdo->depot); + + for (zone = 0; zone < vdo->thread_config.logical_zone_count; zone++) + vdo_dump_logical_zone(&vdo->logical_zones->zones[zone]); + + for (zone = 0; zone < vdo->thread_config.physical_zone_count; zone++) + vdo_dump_physical_zone(&vdo->physical_zones->zones[zone]); + + vdo_dump_hash_zones(vdo->hash_zones); +} + +/** + * vdo_assert_on_admin_thread() - Assert that we are running on the admin thread. + * @vdo: The vdo. + * @name: The name of the function which should be running on the admin thread (for logging). + */ +void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.admin_thread), + "%s called on admin thread", name); +} + +/** + * vdo_assert_on_logical_zone_thread() - Assert that this function was called on the specified + * logical zone thread. + * @vdo: The vdo. + * @logical_zone: The number of the logical zone. + * @name: The name of the calling function. + */ +void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, + const char *name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == + vdo->thread_config.logical_threads[logical_zone]), + "%s called on logical thread", name); +} + +/** + * vdo_assert_on_physical_zone_thread() - Assert that this function was called on the specified + * physical zone thread. + * @vdo: The vdo. + * @physical_zone: The number of the physical zone. + * @name: The name of the calling function. + */ +void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, + zone_count_t physical_zone, const char *name) +{ + ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == + vdo->thread_config.physical_threads[physical_zone]), + "%s called on physical thread", name); +} + +/** + * vdo_get_physical_zone() - Get the physical zone responsible for a given physical block number. + * @vdo: The vdo containing the physical zones. + * @pbn: The PBN of the data block. + * @zone_ptr: A pointer to return the physical zone. + * + * Gets the physical zone responsible for a given physical block number of a data block in this vdo + * instance, or of the zero block (for which a NULL zone is returned). For any other block number + * that is not in the range of valid data block numbers in any slab, an error will be returned. + * This function is safe to call on invalid block numbers; it will not put the vdo into read-only + * mode. + * + * Return: VDO_SUCCESS or VDO_OUT_OF_RANGE if the block number is invalid or an error code for any + * other failure. + */ +int vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, + struct physical_zone **zone_ptr) +{ + struct vdo_slab *slab; + int result; + + if (pbn == VDO_ZERO_BLOCK) { + *zone_ptr = NULL; + return VDO_SUCCESS; + } + + /* + * Used because it does a more restrictive bounds check than vdo_get_slab(), and done first + * because it won't trigger read-only mode on an invalid PBN. + */ + if (!vdo_is_physical_data_block(vdo->depot, pbn)) + return VDO_OUT_OF_RANGE; + + /* With the PBN already checked, we should always succeed in finding a slab. */ + slab = vdo_get_slab(vdo->depot, pbn); + result = ASSERT(slab != NULL, "vdo_get_slab must succeed on all valid PBNs"); + if (result != VDO_SUCCESS) + return result; + + *zone_ptr = &vdo->physical_zones->zones[slab->allocator->zone_number]; + return VDO_SUCCESS; +} diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h new file mode 100644 index 000000000000..35d603066529 --- /dev/null +++ b/drivers/md/dm-vdo/vdo.h @@ -0,0 +1,376 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_H +#define VDO_H + +#include +#include +#include +#include +#include +#include +#include + +#include "admin-state.h" +#include "encodings.h" +#include "funnel-workqueue.h" +#include "packer.h" +#include "physical-zone.h" +#include "statistics.h" +#include "thread-registry.h" +#include "types.h" +#include "uds.h" + +enum notifier_state { + /** Notifications are allowed but not in progress */ + MAY_NOTIFY, + /** A notification is in progress */ + NOTIFYING, + /** Notifications are not allowed */ + MAY_NOT_NOTIFY, + /** A notification has completed */ + NOTIFIED, +}; + +/** + * typedef vdo_read_only_notification_fn - A function to notify a listener that the VDO has gone + * read-only. + * @listener: The object to notify. + * @parent: The completion to notify in order to acknowledge the notification. + */ +typedef void (*vdo_read_only_notification_fn)(void *listener, struct vdo_completion *parent); + +/* + * An object to be notified when the VDO enters read-only mode + */ +struct read_only_listener { + /* The listener */ + void *listener; + /* The method to call to notify the listener */ + vdo_read_only_notification_fn notify; + /* A pointer to the next listener */ + struct read_only_listener *next; +}; + +struct vdo_thread { + struct vdo *vdo; + thread_id_t thread_id; + struct vdo_work_queue *queue; + /* + * Each thread maintains its own notion of whether the VDO is read-only so that the + * read-only state can be checked from any base thread without worrying about + * synchronization or thread safety. This does mean that knowledge of the VDO going + * read-only does not occur simultaneously across the VDO's threads, but that does not seem + * to cause any problems. + */ + bool is_read_only; + /* + * A list of objects waiting to be notified on this thread that the VDO has entered + * read-only mode. + */ + struct read_only_listener *listeners; + struct registered_thread allocating_thread; +}; + +/* Keep struct bio statistics atomically */ +struct atomic_bio_stats { + atomic64_t read; /* Number of not REQ_WRITE bios */ + atomic64_t write; /* Number of REQ_WRITE bios */ + atomic64_t discard; /* Number of REQ_DISCARD bios */ + atomic64_t flush; /* Number of REQ_FLUSH bios */ + atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */ + atomic64_t fua; /* Number of REQ_FUA bios */ +}; + +/* Counters are atomic since updates can arrive concurrently from arbitrary threads. */ +struct atomic_statistics { + atomic64_t bios_submitted; + atomic64_t bios_completed; + atomic64_t flush_out; + atomic64_t invalid_advice_pbn_count; + atomic64_t no_space_error_count; + atomic64_t read_only_error_count; + struct atomic_bio_stats bios_in; + struct atomic_bio_stats bios_in_partial; + struct atomic_bio_stats bios_out; + struct atomic_bio_stats bios_out_completed; + struct atomic_bio_stats bios_acknowledged; + struct atomic_bio_stats bios_acknowledged_partial; + struct atomic_bio_stats bios_meta; + struct atomic_bio_stats bios_meta_completed; + struct atomic_bio_stats bios_journal; + struct atomic_bio_stats bios_journal_completed; + struct atomic_bio_stats bios_page_cache; + struct atomic_bio_stats bios_page_cache_completed; +}; + +struct read_only_notifier { + /* The completion for entering read-only mode */ + struct vdo_completion completion; + /* A completion waiting for notifications to be drained or enabled */ + struct vdo_completion *waiter; + /* Lock to protect the next two fields */ + spinlock_t lock; + /* The code of the error which put the VDO into read-only mode */ + int read_only_error; + /* The current state of the notifier (values described above) */ + enum notifier_state state; +}; + +/* + * The thread ID returned when the current thread is not a vdo thread, or can not be determined + * (usually due to being at interrupt context). + */ +#define VDO_INVALID_THREAD_ID ((thread_id_t) -1) + +struct thread_config { + zone_count_t logical_zone_count; + zone_count_t physical_zone_count; + zone_count_t hash_zone_count; + thread_count_t bio_thread_count; + thread_count_t thread_count; + thread_id_t admin_thread; + thread_id_t journal_thread; + thread_id_t packer_thread; + thread_id_t dedupe_thread; + thread_id_t bio_ack_thread; + thread_id_t cpu_thread; + thread_id_t *logical_threads; + thread_id_t *physical_threads; + thread_id_t *hash_zone_threads; + thread_id_t *bio_threads; +}; + +struct thread_count_config; + +struct vdo_super_block { + /* The vio for reading and writing the super block to disk */ + struct vio vio; + /* A buffer to hold the super block */ + u8 *buffer; + /* Whether this super block may not be written */ + bool unwritable; +}; + +struct data_vio_pool; + +struct vdo_administrator { + struct vdo_completion completion; + struct admin_state state; + atomic_t busy; + u32 phase; + struct completion callback_sync; +}; + +struct vdo { + char thread_name_prefix[MAX_VDO_WORK_QUEUE_NAME_LEN]; + struct vdo_thread *threads; + vdo_action_fn action; + struct vdo_completion *completion; + struct vio_tracer *vio_tracer; + + /* The atomic version of the state of this vdo */ + atomic_t state; + /* The full state of all components */ + struct vdo_component_states states; + /* + * A counter value to attach to thread names and log messages to identify the individual + * device. + */ + unsigned int instance; + /* The read-only notifier */ + struct read_only_notifier read_only_notifier; + /* The load-time configuration of this vdo */ + struct device_config *device_config; + /* The thread mapping */ + struct thread_config thread_config; + + /* The super block */ + struct vdo_super_block super_block; + + /* The partitioning of the underlying storage */ + struct layout layout; + struct layout next_layout; + struct dm_kcopyd_client *partition_copier; + + /* The block map */ + struct block_map *block_map; + + /* The journal for block map recovery */ + struct recovery_journal *recovery_journal; + + /* The slab depot */ + struct slab_depot *depot; + + /* The compressed-block packer */ + struct packer *packer; + /* Whether incoming data should be compressed */ + bool compressing; + + /* The handler for flush requests */ + struct flusher *flusher; + + /* The state the vdo was in when loaded (primarily for unit tests) */ + enum vdo_state load_state; + + /* The logical zones of this vdo */ + struct logical_zones *logical_zones; + + /* The physical zones of this vdo */ + struct physical_zones *physical_zones; + + /* The hash lock zones of this vdo */ + struct hash_zones *hash_zones; + + /* Bio submission manager used for sending bios to the storage device. */ + struct io_submitter *io_submitter; + + /* The pool of data_vios for servicing incoming bios */ + struct data_vio_pool *data_vio_pool; + + /* The manager for administrative operations */ + struct vdo_administrator admin; + + /* Flags controlling administrative operations */ + const struct admin_state_code *suspend_type; + bool allocations_allowed; + bool dump_on_shutdown; + atomic_t processing_message; + + /* + * Statistics + * Atomic stats counters + */ + struct atomic_statistics stats; + /* Used to gather statistics without allocating memory */ + struct vdo_statistics stats_buffer; + /* Protects the stats_buffer */ + struct mutex stats_mutex; + /* true if sysfs directory is set up */ + bool sysfs_added; + /* Used when shutting down the sysfs statistics */ + struct completion stats_shutdown; + + + /* A list of all device_configs referencing this vdo */ + struct list_head device_config_list; + + /* This VDO's list entry for the device registry */ + struct list_head registration; + + /* Underlying block device info. */ + u64 starting_sector_offset; + struct volume_geometry geometry; + + /* For sysfs */ + struct kobject vdo_directory; + struct kobject stats_directory; + + /* N blobs of context data for LZ4 code, one per CPU thread. */ + char **compression_context; +}; + + +/** + * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue + * for acknowledging received and processed bios. + * @vdo: The vdo. + * + * Note that this directly controls the handling of write operations, but the compile-time flag + * VDO_USE_BIO_ACK_QUEUE_FOR_READ is also checked for read operations. + * + * Return: Whether a bio-acknowledgement work queue is in use. + */ +static inline bool vdo_uses_bio_ack_queue(struct vdo *vdo) +{ + return vdo->device_config->thread_counts.bio_ack_threads > 0; +} + +/** + * typedef vdo_filter_fn - Method type for vdo matching methods. + * + * A filter function returns false if the vdo doesn't match. + */ +typedef bool (*vdo_filter_fn)(struct vdo *vdo, const void *context); + +void vdo_initialize_device_registry_once(void); +struct vdo * __must_check vdo_find_matching(vdo_filter_fn filter, const void *context); + +int __must_check vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, + const struct vdo_work_queue_type *type, + unsigned int queue_count, void *contexts[]); + +static inline int __must_check vdo_make_default_thread(struct vdo *vdo, + thread_id_t thread_id) +{ + return vdo_make_thread(vdo, thread_id, NULL, 1, NULL); +} + +int __must_check vdo_make(unsigned int instance, struct device_config *config, + char **reason, struct vdo **vdo_ptr); + +void vdo_destroy(struct vdo *vdo); + +void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent); + +int __must_check vdo_add_sysfs_stats_dir(struct vdo *vdo); + +struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo); + +const char * __must_check vdo_get_device_name(const struct dm_target *target); + +int __must_check vdo_synchronous_flush(struct vdo *vdo); + +const struct admin_state_code * __must_check vdo_get_admin_state(const struct vdo *vdo); + +bool vdo_set_compressing(struct vdo *vdo, bool enable); + +bool vdo_get_compressing(struct vdo *vdo); + +void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats); + +thread_id_t vdo_get_callback_thread_id(void); + +enum vdo_state __must_check vdo_get_state(const struct vdo *vdo); + +void vdo_set_state(struct vdo *vdo, enum vdo_state state); + +void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent); + +int vdo_register_read_only_listener(struct vdo *vdo, void *listener, + vdo_read_only_notification_fn notification, + thread_id_t thread_id); + +int vdo_enable_read_only_entry(struct vdo *vdo); + +void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent); + +void vdo_allow_read_only_mode_entry(struct vdo_completion *parent); + +void vdo_enter_read_only_mode(struct vdo *vdo, int error_code); + +bool __must_check vdo_is_read_only(struct vdo *vdo); + +bool __must_check vdo_in_read_only_mode(const struct vdo *vdo); + +bool __must_check vdo_in_recovery_mode(const struct vdo *vdo); + +void vdo_enter_recovery_mode(struct vdo *vdo); + +void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name); + +void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, + const char *name); + +void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone, + const char *name); + +int __must_check vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, + struct physical_zone **zone_ptr); + +void vdo_dump_status(const struct vdo *vdo); + +#endif /* VDO_H */ From 827c6389c6e496a9d8bd2f94667dd7638c75fe3e Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:09:25 -0500 Subject: [PATCH 0582/1038] dm vdo: add the on-disk formats and marshalling of vdo structures Add data and methods for marshalling and unmarshalling the persistent metadata. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/encodings.c | 1481 +++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/encodings.h | 1299 +++++++++++++++++++++++++++++ 2 files changed, 2780 insertions(+) create mode 100644 drivers/md/dm-vdo/encodings.c create mode 100644 drivers/md/dm-vdo/encodings.h diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c new file mode 100644 index 000000000000..2d9d8645f66b --- /dev/null +++ b/drivers/md/dm-vdo/encodings.c @@ -0,0 +1,1481 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "encodings.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "constants.h" +#include "status-codes.h" +#include "types.h" + +struct geometry_block { + char magic_number[VDO_GEOMETRY_MAGIC_NUMBER_SIZE]; + struct packed_header header; + u32 checksum; +} __packed; + +static const struct header GEOMETRY_BLOCK_HEADER_5_0 = { + .id = VDO_GEOMETRY_BLOCK, + .version = { + .major_version = 5, + .minor_version = 0, + }, + /* + * Note: this size isn't just the payload size following the header, like it is everywhere + * else in VDO. + */ + .size = sizeof(struct geometry_block) + sizeof(struct volume_geometry), +}; + +static const struct header GEOMETRY_BLOCK_HEADER_4_0 = { + .id = VDO_GEOMETRY_BLOCK, + .version = { + .major_version = 4, + .minor_version = 0, + }, + /* + * Note: this size isn't just the payload size following the header, like it is everywhere + * else in VDO. + */ + .size = sizeof(struct geometry_block) + sizeof(struct volume_geometry_4_0), +}; + +const u8 VDO_GEOMETRY_MAGIC_NUMBER[VDO_GEOMETRY_MAGIC_NUMBER_SIZE + 1] = "dmvdo001"; + +enum { + PAGE_HEADER_4_1_SIZE = 8 + 8 + 8 + 1 + 1 + 1 + 1, +}; + +static const struct version_number BLOCK_MAP_4_1 = { + .major_version = 4, + .minor_version = 1, +}; + +const struct header VDO_BLOCK_MAP_HEADER_2_0 = { + .id = VDO_BLOCK_MAP, + .version = { + .major_version = 2, + .minor_version = 0, + }, + .size = sizeof(struct block_map_state_2_0), +}; + +const struct header VDO_RECOVERY_JOURNAL_HEADER_7_0 = { + .id = VDO_RECOVERY_JOURNAL, + .version = { + .major_version = 7, + .minor_version = 0, + }, + .size = sizeof(struct recovery_journal_state_7_0), +}; + +const struct header VDO_SLAB_DEPOT_HEADER_2_0 = { + .id = VDO_SLAB_DEPOT, + .version = { + .major_version = 2, + .minor_version = 0, + }, + .size = sizeof(struct slab_depot_state_2_0), +}; + +const struct header VDO_LAYOUT_HEADER_3_0 = { + .id = VDO_LAYOUT, + .version = { + .major_version = 3, + .minor_version = 0, + }, + .size = sizeof(struct layout_3_0) + (sizeof(struct partition_3_0) * VDO_PARTITION_COUNT), +}; + +static const enum partition_id REQUIRED_PARTITIONS[] = { + VDO_BLOCK_MAP_PARTITION, + VDO_SLAB_DEPOT_PARTITION, + VDO_RECOVERY_JOURNAL_PARTITION, + VDO_SLAB_SUMMARY_PARTITION, +}; + +/* + * The current version for the data encoded in the super block. This must be changed any time there + * is a change to encoding of the component data of any VDO component. + */ +static const struct version_number VDO_COMPONENT_DATA_41_0 = { + .major_version = 41, + .minor_version = 0, +}; + +const struct version_number VDO_VOLUME_VERSION_67_0 = { + .major_version = 67, + .minor_version = 0, +}; + +static const struct header SUPER_BLOCK_HEADER_12_0 = { + .id = VDO_SUPER_BLOCK, + .version = { + .major_version = 12, + .minor_version = 0, + }, + + /* This is the minimum size, if the super block contains no components. */ + .size = VDO_SUPER_BLOCK_FIXED_SIZE - VDO_ENCODED_HEADER_SIZE, +}; + +/** + * validate_version() - Check whether a version matches an expected version. + * @expected_version: The expected version. + * @actual_version: The version being validated. + * @component_name: The name of the component or the calling function (for error logging). + * + * Logs an error describing a mismatch. + * + * Return: VDO_SUCCESS if the versions are the same, + * VDO_UNSUPPORTED_VERSION if the versions don't match. + */ +static int __must_check validate_version(struct version_number expected_version, + struct version_number actual_version, + const char *component_name) +{ + if (!vdo_are_same_version(expected_version, actual_version)) { + return uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "%s version mismatch, expected %d.%d, got %d.%d", + component_name, + expected_version.major_version, + expected_version.minor_version, + actual_version.major_version, + actual_version.minor_version); + } + + return VDO_SUCCESS; +} + +/** + * vdo_validate_header() - Check whether a header matches expectations. + * @expected_header: The expected header. + * @actual_header: The header being validated. + * @exact_size: If true, the size fields of the two headers must be the same, otherwise it is + * required that actual_header.size >= expected_header.size. + * @name: The name of the component or the calling function (for error logging). + * + * Logs an error describing the first mismatch found. + * + * Return: VDO_SUCCESS if the header meets expectations, + * VDO_INCORRECT_COMPONENT if the component ids don't match, + * VDO_UNSUPPORTED_VERSION if the versions or sizes don't match. + */ +int vdo_validate_header(const struct header *expected_header, + const struct header *actual_header, bool exact_size, + const char *name) +{ + int result; + + if (expected_header->id != actual_header->id) { + return uds_log_error_strerror(VDO_INCORRECT_COMPONENT, + "%s ID mismatch, expected %d, got %d", + name, expected_header->id, + actual_header->id); + } + + result = validate_version(expected_header->version, actual_header->version, + name); + if (result != VDO_SUCCESS) + return result; + + if ((expected_header->size > actual_header->size) || + (exact_size && (expected_header->size < actual_header->size))) { + return uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "%s size mismatch, expected %zu, got %zu", + name, expected_header->size, + actual_header->size); + } + + return VDO_SUCCESS; +} + +static void encode_version_number(u8 *buffer, size_t *offset, + struct version_number version) +{ + struct packed_version_number packed = vdo_pack_version_number(version); + + memcpy(buffer + *offset, &packed, sizeof(packed)); + *offset += sizeof(packed); +} + +void vdo_encode_header(u8 *buffer, size_t *offset, const struct header *header) +{ + struct packed_header packed = vdo_pack_header(header); + + memcpy(buffer + *offset, &packed, sizeof(packed)); + *offset += sizeof(packed); +} + +static void decode_version_number(u8 *buffer, size_t *offset, + struct version_number *version) +{ + struct packed_version_number packed; + + memcpy(&packed, buffer + *offset, sizeof(packed)); + *offset += sizeof(packed); + *version = vdo_unpack_version_number(packed); +} + +void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header) +{ + struct packed_header packed; + + memcpy(&packed, buffer + *offset, sizeof(packed)); + *offset += sizeof(packed); + + *header = vdo_unpack_header(&packed); +} + +/** + * decode_volume_geometry() - Decode the on-disk representation of a volume geometry from a buffer. + * @buffer: A buffer to decode from. + * @offset: The offset in the buffer at which to decode. + * @geometry: The structure to receive the decoded fields. + * @version: The geometry block version to decode. + */ +static void decode_volume_geometry(u8 *buffer, size_t *offset, + struct volume_geometry *geometry, u32 version) +{ + u32 unused, mem; + enum volume_region_id id; + nonce_t nonce; + block_count_t bio_offset = 0; + bool sparse; + + /* This is for backwards compatibility. */ + decode_u32_le(buffer, offset, &unused); + geometry->unused = unused; + + decode_u64_le(buffer, offset, &nonce); + geometry->nonce = nonce; + + memcpy((unsigned char *) &geometry->uuid, buffer + *offset, sizeof(uuid_t)); + *offset += sizeof(uuid_t); + + if (version > 4) + decode_u64_le(buffer, offset, &bio_offset); + geometry->bio_offset = bio_offset; + + for (id = 0; id < VDO_VOLUME_REGION_COUNT; id++) { + physical_block_number_t start_block; + enum volume_region_id saved_id; + + decode_u32_le(buffer, offset, &saved_id); + decode_u64_le(buffer, offset, &start_block); + + geometry->regions[id] = (struct volume_region) { + .id = saved_id, + .start_block = start_block, + }; + } + + decode_u32_le(buffer, offset, &mem); + *offset += sizeof(u32); + sparse = buffer[(*offset)++]; + + geometry->index_config = (struct index_config) { + .mem = mem, + .sparse = sparse, + }; +} + +/** + * vdo_parse_geometry_block() - Decode and validate an encoded geometry block. + * @block: The encoded geometry block. + * @geometry: The structure to receive the decoded fields. + */ +int __must_check vdo_parse_geometry_block(u8 *block, struct volume_geometry *geometry) +{ + u32 checksum, saved_checksum; + struct header header; + size_t offset = 0; + int result; + + if (memcmp(block, VDO_GEOMETRY_MAGIC_NUMBER, VDO_GEOMETRY_MAGIC_NUMBER_SIZE) != 0) + return VDO_BAD_MAGIC; + offset += VDO_GEOMETRY_MAGIC_NUMBER_SIZE; + + vdo_decode_header(block, &offset, &header); + if (header.version.major_version <= 4) { + result = vdo_validate_header(&GEOMETRY_BLOCK_HEADER_4_0, &header, + true, __func__); + } else { + result = vdo_validate_header(&GEOMETRY_BLOCK_HEADER_5_0, &header, + true, __func__); + } + if (result != VDO_SUCCESS) + return result; + + decode_volume_geometry(block, &offset, geometry, header.version.major_version); + + result = ASSERT(header.size == offset + sizeof(u32), + "should have decoded up to the geometry checksum"); + if (result != VDO_SUCCESS) + return result; + + /* Decode and verify the checksum. */ + checksum = vdo_crc32(block, offset); + decode_u32_le(block, &offset, &saved_checksum); + + return ((checksum == saved_checksum) ? VDO_SUCCESS : VDO_CHECKSUM_MISMATCH); +} + +struct block_map_page *vdo_format_block_map_page(void *buffer, nonce_t nonce, + physical_block_number_t pbn, + bool initialized) +{ + struct block_map_page *page = buffer; + + memset(buffer, 0, VDO_BLOCK_SIZE); + page->version = vdo_pack_version_number(BLOCK_MAP_4_1); + page->header.nonce = __cpu_to_le64(nonce); + page->header.pbn = __cpu_to_le64(pbn); + page->header.initialized = initialized; + return page; +} + +enum block_map_page_validity vdo_validate_block_map_page(struct block_map_page *page, + nonce_t nonce, + physical_block_number_t pbn) +{ + BUILD_BUG_ON(sizeof(struct block_map_page_header) != PAGE_HEADER_4_1_SIZE); + + if (!vdo_are_same_version(BLOCK_MAP_4_1, + vdo_unpack_version_number(page->version)) || + !page->header.initialized || (nonce != __le64_to_cpu(page->header.nonce))) + return VDO_BLOCK_MAP_PAGE_INVALID; + + if (pbn != vdo_get_block_map_page_pbn(page)) + return VDO_BLOCK_MAP_PAGE_BAD; + + return VDO_BLOCK_MAP_PAGE_VALID; +} + +static int decode_block_map_state_2_0(u8 *buffer, size_t *offset, + struct block_map_state_2_0 *state) +{ + size_t initial_offset; + block_count_t flat_page_count, root_count; + physical_block_number_t flat_page_origin, root_origin; + struct header header; + int result; + + vdo_decode_header(buffer, offset, &header); + result = vdo_validate_header(&VDO_BLOCK_MAP_HEADER_2_0, &header, true, __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + + decode_u64_le(buffer, offset, &flat_page_origin); + result = ASSERT(flat_page_origin == VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + "Flat page origin must be %u (recorded as %llu)", + VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + (unsigned long long) state->flat_page_origin); + if (result != UDS_SUCCESS) + return result; + + decode_u64_le(buffer, offset, &flat_page_count); + result = ASSERT(flat_page_count == 0, + "Flat page count must be 0 (recorded as %llu)", + (unsigned long long) state->flat_page_count); + if (result != UDS_SUCCESS) + return result; + + decode_u64_le(buffer, offset, &root_origin); + decode_u64_le(buffer, offset, &root_count); + + result = ASSERT(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, + "decoded block map component size must match header size"); + if (result != VDO_SUCCESS) + return result; + + *state = (struct block_map_state_2_0) { + .flat_page_origin = flat_page_origin, + .flat_page_count = flat_page_count, + .root_origin = root_origin, + .root_count = root_count, + }; + + return VDO_SUCCESS; +} + +static void encode_block_map_state_2_0(u8 *buffer, size_t *offset, + struct block_map_state_2_0 state) +{ + size_t initial_offset; + + vdo_encode_header(buffer, offset, &VDO_BLOCK_MAP_HEADER_2_0); + + initial_offset = *offset; + encode_u64_le(buffer, offset, state.flat_page_origin); + encode_u64_le(buffer, offset, state.flat_page_count); + encode_u64_le(buffer, offset, state.root_origin); + encode_u64_le(buffer, offset, state.root_count); + + ASSERT_LOG_ONLY(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, + "encoded block map component size must match header size"); +} + +/** + * vdo_compute_new_forest_pages() - Compute the number of pages which must be allocated at each + * level in order to grow the forest to a new number of entries. + * @entries: The new number of entries the block map must address. + * + * Return: The total number of non-leaf pages required. + */ +block_count_t vdo_compute_new_forest_pages(root_count_t root_count, + struct boundary *old_sizes, + block_count_t entries, + struct boundary *new_sizes) +{ + page_count_t leaf_pages = max(vdo_compute_block_map_page_count(entries), 1U); + page_count_t level_size = DIV_ROUND_UP(leaf_pages, root_count); + block_count_t total_pages = 0; + height_t height; + + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT; height++) { + block_count_t new_pages; + + level_size = DIV_ROUND_UP(level_size, VDO_BLOCK_MAP_ENTRIES_PER_PAGE); + new_sizes->levels[height] = level_size; + new_pages = level_size; + if (old_sizes != NULL) + new_pages -= old_sizes->levels[height]; + total_pages += (new_pages * root_count); + } + + return total_pages; +} + +/** + * encode_recovery_journal_state_7_0() - Encode the state of a recovery journal. + * + * Return: VDO_SUCCESS or an error code. + */ +static void encode_recovery_journal_state_7_0(u8 *buffer, size_t *offset, + struct recovery_journal_state_7_0 state) +{ + size_t initial_offset; + + vdo_encode_header(buffer, offset, &VDO_RECOVERY_JOURNAL_HEADER_7_0); + + initial_offset = *offset; + encode_u64_le(buffer, offset, state.journal_start); + encode_u64_le(buffer, offset, state.logical_blocks_used); + encode_u64_le(buffer, offset, state.block_map_data_blocks); + + ASSERT_LOG_ONLY(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, + "encoded recovery journal component size must match header size"); +} + +/** + * decode_recovery_journal_state_7_0() - Decode the state of a recovery journal saved in a buffer. + * @buffer: The buffer containing the saved state. + * @state: A pointer to a recovery journal state to hold the result of a successful decode. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check decode_recovery_journal_state_7_0(u8 *buffer, size_t *offset, + struct recovery_journal_state_7_0 *state) +{ + struct header header; + int result; + size_t initial_offset; + sequence_number_t journal_start; + block_count_t logical_blocks_used, block_map_data_blocks; + + vdo_decode_header(buffer, offset, &header); + result = vdo_validate_header(&VDO_RECOVERY_JOURNAL_HEADER_7_0, &header, true, + __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + decode_u64_le(buffer, offset, &journal_start); + decode_u64_le(buffer, offset, &logical_blocks_used); + decode_u64_le(buffer, offset, &block_map_data_blocks); + + result = ASSERT(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, + "decoded recovery journal component size must match header size"); + if (result != UDS_SUCCESS) + return result; + + *state = (struct recovery_journal_state_7_0) { + .journal_start = journal_start, + .logical_blocks_used = logical_blocks_used, + .block_map_data_blocks = block_map_data_blocks, + }; + + return VDO_SUCCESS; +} + +/** + * vdo_get_journal_operation_name() - Get the name of a journal operation. + * @operation: The operation to name. + * + * Return: The name of the operation. + */ +const char *vdo_get_journal_operation_name(enum journal_operation operation) +{ + switch (operation) { + case VDO_JOURNAL_DATA_REMAPPING: + return "data remapping"; + + case VDO_JOURNAL_BLOCK_MAP_REMAPPING: + return "block map remapping"; + + default: + return "unknown journal operation"; + } +} + +/** + * encode_slab_depot_state_2_0() - Encode the state of a slab depot into a buffer. + * + * Return: UDS_SUCCESS or an error. + */ +static void encode_slab_depot_state_2_0(u8 *buffer, size_t *offset, + struct slab_depot_state_2_0 state) +{ + size_t initial_offset; + + vdo_encode_header(buffer, offset, &VDO_SLAB_DEPOT_HEADER_2_0); + + initial_offset = *offset; + encode_u64_le(buffer, offset, state.slab_config.slab_blocks); + encode_u64_le(buffer, offset, state.slab_config.data_blocks); + encode_u64_le(buffer, offset, state.slab_config.reference_count_blocks); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_blocks); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_flushing_threshold); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_blocking_threshold); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_scrubbing_threshold); + encode_u64_le(buffer, offset, state.first_block); + encode_u64_le(buffer, offset, state.last_block); + buffer[(*offset)++] = state.zone_count; + + ASSERT_LOG_ONLY(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, + "encoded block map component size must match header size"); +} + +/** + * decode_slab_depot_state_2_0() - Decode slab depot component state version 2.0 from a buffer. + * + * Return: UDS_SUCCESS or an error code. + */ +static int decode_slab_depot_state_2_0(u8 *buffer, size_t *offset, + struct slab_depot_state_2_0 *state) +{ + struct header header; + int result; + size_t initial_offset; + struct slab_config slab_config; + block_count_t count; + physical_block_number_t first_block, last_block; + zone_count_t zone_count; + + vdo_decode_header(buffer, offset, &header); + result = vdo_validate_header(&VDO_SLAB_DEPOT_HEADER_2_0, &header, true, + __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + decode_u64_le(buffer, offset, &count); + slab_config.slab_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.data_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.reference_count_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_flushing_threshold = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_blocking_threshold = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_scrubbing_threshold = count; + + decode_u64_le(buffer, offset, &first_block); + decode_u64_le(buffer, offset, &last_block); + zone_count = buffer[(*offset)++]; + + result = ASSERT(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, + "decoded slab depot component size must match header size"); + if (result != UDS_SUCCESS) + return result; + + *state = (struct slab_depot_state_2_0) { + .slab_config = slab_config, + .first_block = first_block, + .last_block = last_block, + .zone_count = zone_count, + }; + + return VDO_SUCCESS; +} + +/** + * vdo_configure_slab_depot() - Configure the slab depot. + * @partition: The slab depot partition + * @slab_config: The configuration of a single slab. + * @zone_count: The number of zones the depot will use. + * @state: The state structure to be configured. + * + * Configures the slab_depot for the specified storage capacity, finding the number of data blocks + * that will fit and still leave room for the depot metadata, then return the saved state for that + * configuration. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_configure_slab_depot(const struct partition *partition, + struct slab_config slab_config, zone_count_t zone_count, + struct slab_depot_state_2_0 *state) +{ + block_count_t total_slab_blocks, total_data_blocks; + size_t slab_count; + physical_block_number_t last_block; + block_count_t slab_size = slab_config.slab_blocks; + + uds_log_debug("slabDepot %s(block_count=%llu, first_block=%llu, slab_size=%llu, zone_count=%u)", + __func__, (unsigned long long) partition->count, + (unsigned long long) partition->offset, + (unsigned long long) slab_size, zone_count); + + /* We do not allow runt slabs, so we waste up to a slab's worth. */ + slab_count = (partition->count / slab_size); + if (slab_count == 0) + return VDO_NO_SPACE; + + if (slab_count > MAX_VDO_SLABS) + return VDO_TOO_MANY_SLABS; + + total_slab_blocks = slab_count * slab_config.slab_blocks; + total_data_blocks = slab_count * slab_config.data_blocks; + last_block = partition->offset + total_slab_blocks; + + *state = (struct slab_depot_state_2_0) { + .slab_config = slab_config, + .first_block = partition->offset, + .last_block = last_block, + .zone_count = zone_count, + }; + + uds_log_debug("slab_depot last_block=%llu, total_data_blocks=%llu, slab_count=%zu, left_over=%llu", + (unsigned long long) last_block, + (unsigned long long) total_data_blocks, slab_count, + (unsigned long long) (partition->count - (last_block - partition->offset))); + + return VDO_SUCCESS; +} + +/** + * vdo_configure_slab() - Measure and initialize the configuration to use for each slab. + * @slab_size: The number of blocks per slab. + * @slab_journal_blocks: The number of blocks for the slab journal. + * @slab_config: The slab configuration to initialize. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_configure_slab(block_count_t slab_size, block_count_t slab_journal_blocks, + struct slab_config *slab_config) +{ + block_count_t ref_blocks, meta_blocks, data_blocks; + block_count_t flushing_threshold, remaining, blocking_threshold; + block_count_t minimal_extra_space, scrubbing_threshold; + + if (slab_journal_blocks >= slab_size) + return VDO_BAD_CONFIGURATION; + + /* + * This calculation should technically be a recurrence, but the total number of metadata + * blocks is currently less than a single block of ref_counts, so we'd gain at most one + * data block in each slab with more iteration. + */ + ref_blocks = vdo_get_saved_reference_count_size(slab_size - slab_journal_blocks); + meta_blocks = (ref_blocks + slab_journal_blocks); + + /* Make sure test code hasn't configured slabs to be too small. */ + if (meta_blocks >= slab_size) + return VDO_BAD_CONFIGURATION; + + /* + * If the slab size is very small, assume this must be a unit test and override the number + * of data blocks to be a power of two (wasting blocks in the slab). Many tests need their + * data_blocks fields to be the exact capacity of the configured volume, and that used to + * fall out since they use a power of two for the number of data blocks, the slab size was + * a power of two, and every block in a slab was a data block. + * + * TODO: Try to figure out some way of structuring testParameters and unit tests so this + * hack isn't needed without having to edit several unit tests every time the metadata size + * changes by one block. + */ + data_blocks = slab_size - meta_blocks; + if ((slab_size < 1024) && !is_power_of_2(data_blocks)) + data_blocks = ((block_count_t) 1 << ilog2(data_blocks)); + + /* + * Configure the slab journal thresholds. The flush threshold is 168 of 224 blocks in + * production, or 3/4ths, so we use this ratio for all sizes. + */ + flushing_threshold = ((slab_journal_blocks * 3) + 3) / 4; + /* + * The blocking threshold should be far enough from the flushing threshold to not produce + * delays, but far enough from the end of the journal to allow multiple successive recovery + * failures. + */ + remaining = slab_journal_blocks - flushing_threshold; + blocking_threshold = flushing_threshold + ((remaining * 5) / 7); + /* The scrubbing threshold should be at least 2048 entries before the end of the journal. */ + minimal_extra_space = 1 + (MAXIMUM_VDO_USER_VIOS / VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK); + scrubbing_threshold = blocking_threshold; + if (slab_journal_blocks > minimal_extra_space) + scrubbing_threshold = slab_journal_blocks - minimal_extra_space; + if (blocking_threshold > scrubbing_threshold) + blocking_threshold = scrubbing_threshold; + + *slab_config = (struct slab_config) { + .slab_blocks = slab_size, + .data_blocks = data_blocks, + .reference_count_blocks = ref_blocks, + .slab_journal_blocks = slab_journal_blocks, + .slab_journal_flushing_threshold = flushing_threshold, + .slab_journal_blocking_threshold = blocking_threshold, + .slab_journal_scrubbing_threshold = scrubbing_threshold}; + return VDO_SUCCESS; +} + +/** + * vdo_decode_slab_journal_entry() - Decode a slab journal entry. + * @block: The journal block holding the entry. + * @entry_count: The number of the entry. + * + * Return: The decoded entry. + */ +struct slab_journal_entry vdo_decode_slab_journal_entry(struct packed_slab_journal_block *block, + journal_entry_count_t entry_count) +{ + struct slab_journal_entry entry = + vdo_unpack_slab_journal_entry(&block->payload.entries[entry_count]); + + if (block->header.has_block_map_increments && + ((block->payload.full_entries.entry_types[entry_count / 8] & + ((u8) 1 << (entry_count % 8))) != 0)) + entry.operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING; + + return entry; +} + +/** + * allocate_partition() - Allocate a partition and add it to a layout. + * @layout: The layout containing the partition. + * @id: The id of the partition. + * @offset: The offset into the layout at which the partition begins. + * @size: The size of the partition in blocks. + * + * Return: VDO_SUCCESS or an error. + */ +static int allocate_partition(struct layout *layout, u8 id, + physical_block_number_t offset, block_count_t size) +{ + struct partition *partition; + int result; + + result = uds_allocate(1, struct partition, __func__, &partition); + if (result != UDS_SUCCESS) + return result; + + partition->id = id; + partition->offset = offset; + partition->count = size; + partition->next = layout->head; + layout->head = partition; + + return VDO_SUCCESS; +} + +/** + * make_partition() - Create a new partition from the beginning or end of the unused space in a + * layout. + * @layout: The layout. + * @id: The id of the partition to make. + * @size: The number of blocks to carve out; if 0, all remaining space will be used. + * @beginning: True if the partition should start at the beginning of the unused space. + * + * Return: A success or error code, particularly VDO_NO_SPACE if there are fewer than size blocks + * remaining. + */ +static int __must_check make_partition(struct layout *layout, enum partition_id id, + block_count_t size, bool beginning) +{ + int result; + physical_block_number_t offset; + block_count_t free_blocks = layout->last_free - layout->first_free; + + if (size == 0) { + if (free_blocks == 0) + return VDO_NO_SPACE; + size = free_blocks; + } else if (size > free_blocks) { + return VDO_NO_SPACE; + } + + result = vdo_get_partition(layout, id, NULL); + if (result != VDO_UNKNOWN_PARTITION) + return VDO_PARTITION_EXISTS; + + offset = beginning ? layout->first_free : (layout->last_free - size); + + result = allocate_partition(layout, id, offset, size); + if (result != VDO_SUCCESS) + return result; + + layout->num_partitions++; + if (beginning) + layout->first_free += size; + else + layout->last_free = layout->last_free - size; + + return VDO_SUCCESS; +} + +/** + * vdo_initialize_layout() - Lay out the partitions of a vdo. + * @size: The entire size of the vdo. + * @origin: The start of the layout on the underlying storage in blocks. + * @block_map_blocks: The size of the block map partition. + * @journal_blocks: The size of the journal partition. + * @summary_blocks: The size of the slab summary partition. + * @layout: The layout to initialize. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_initialize_layout(block_count_t size, physical_block_number_t offset, + block_count_t block_map_blocks, block_count_t journal_blocks, + block_count_t summary_blocks, struct layout *layout) +{ + int result; + block_count_t necessary_size = + (offset + block_map_blocks + journal_blocks + summary_blocks); + + if (necessary_size > size) + return uds_log_error_strerror(VDO_NO_SPACE, + "Not enough space to make a VDO"); + + *layout = (struct layout) { + .start = offset, + .size = size, + .first_free = offset, + .last_free = size, + .num_partitions = 0, + .head = NULL, + }; + + result = make_partition(layout, VDO_BLOCK_MAP_PARTITION, block_map_blocks, true); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + + result = make_partition(layout, VDO_SLAB_SUMMARY_PARTITION, summary_blocks, + false); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + + result = make_partition(layout, VDO_RECOVERY_JOURNAL_PARTITION, journal_blocks, + false); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + + result = make_partition(layout, VDO_SLAB_DEPOT_PARTITION, 0, true); + if (result != VDO_SUCCESS) + vdo_uninitialize_layout(layout); + + return result; +} + +/** + * vdo_uninitialize_layout() - Clean up a layout. + * @layout: The layout to clean up. + * + * All partitions created by this layout become invalid pointers. + */ +void vdo_uninitialize_layout(struct layout *layout) +{ + while (layout->head != NULL) { + struct partition *part = layout->head; + + layout->head = part->next; + uds_free(part); + } + + memset(layout, 0, sizeof(struct layout)); +} + +/** + * vdo_get_partition() - Get a partition by id. + * @layout: The layout from which to get a partition. + * @id: The id of the partition. + * @partition_ptr: A pointer to hold the partition. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_get_partition(struct layout *layout, enum partition_id id, + struct partition **partition_ptr) +{ + struct partition *partition; + + for (partition = layout->head; partition != NULL; partition = partition->next) { + if (partition->id == id) { + if (partition_ptr != NULL) + *partition_ptr = partition; + return VDO_SUCCESS; + } + } + + return VDO_UNKNOWN_PARTITION; +} + +/** + * vdo_get_known_partition() - Get a partition by id from a validated layout. + * @layout: The layout from which to get a partition. + * @id: The id of the partition. + * + * Return: the partition + */ +struct partition *vdo_get_known_partition(struct layout *layout, enum partition_id id) +{ + struct partition *partition; + int result = vdo_get_partition(layout, id, &partition); + + ASSERT_LOG_ONLY(result == VDO_SUCCESS, "layout has expected partition: %u", id); + + return partition; +} + +static void encode_layout(u8 *buffer, size_t *offset, const struct layout *layout) +{ + const struct partition *partition; + size_t initial_offset; + struct header header = VDO_LAYOUT_HEADER_3_0; + + BUILD_BUG_ON(sizeof(enum partition_id) != sizeof(u8)); + ASSERT_LOG_ONLY(layout->num_partitions <= U8_MAX, + "layout partition count must fit in a byte"); + + vdo_encode_header(buffer, offset, &header); + + initial_offset = *offset; + encode_u64_le(buffer, offset, layout->first_free); + encode_u64_le(buffer, offset, layout->last_free); + buffer[(*offset)++] = layout->num_partitions; + + ASSERT_LOG_ONLY(sizeof(struct layout_3_0) == *offset - initial_offset, + "encoded size of a layout header must match structure"); + + for (partition = layout->head; partition != NULL; partition = partition->next) { + buffer[(*offset)++] = partition->id; + encode_u64_le(buffer, offset, partition->offset); + /* This field only exists for backwards compatibility */ + encode_u64_le(buffer, offset, 0); + encode_u64_le(buffer, offset, partition->count); + } + + ASSERT_LOG_ONLY(header.size == *offset - initial_offset, + "encoded size of a layout must match header size"); +} + +static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t start, + block_count_t size, struct layout *layout) +{ + struct header header; + struct layout_3_0 layout_header; + struct partition *partition; + size_t initial_offset; + physical_block_number_t first_free, last_free; + u8 partition_count; + u8 i; + int result; + + vdo_decode_header(buffer, offset, &header); + /* Layout is variable size, so only do a minimum size check here. */ + result = vdo_validate_header(&VDO_LAYOUT_HEADER_3_0, &header, false, __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + decode_u64_le(buffer, offset, &first_free); + decode_u64_le(buffer, offset, &last_free); + partition_count = buffer[(*offset)++]; + layout_header = (struct layout_3_0) { + .first_free = first_free, + .last_free = last_free, + .partition_count = partition_count, + }; + + result = ASSERT(sizeof(struct layout_3_0) == *offset - initial_offset, + "decoded size of a layout header must match structure"); + if (result != VDO_SUCCESS) + return result; + + layout->start = start; + layout->size = size; + layout->first_free = layout_header.first_free; + layout->last_free = layout_header.last_free; + layout->num_partitions = layout_header.partition_count; + + if (layout->num_partitions > VDO_PARTITION_COUNT) { + return uds_log_error_strerror(VDO_UNKNOWN_PARTITION, + "layout has extra partitions"); + } + + for (i = 0; i < layout->num_partitions; i++) { + u8 id; + u64 partition_offset, count; + + id = buffer[(*offset)++]; + decode_u64_le(buffer, offset, &partition_offset); + *offset += sizeof(u64); + decode_u64_le(buffer, offset, &count); + + result = allocate_partition(layout, id, partition_offset, count); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + } + + /* Validate that the layout has all (and only) the required partitions */ + for (i = 0; i < VDO_PARTITION_COUNT; i++) { + result = vdo_get_partition(layout, REQUIRED_PARTITIONS[i], &partition); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return uds_log_error_strerror(result, + "layout is missing required partition %u", + REQUIRED_PARTITIONS[i]); + } + + start += partition->count; + } + + if (start != size) { + vdo_uninitialize_layout(layout); + return uds_log_error_strerror(UDS_BAD_STATE, + "partitions do not cover the layout"); + } + + return VDO_SUCCESS; +} + +/** + * pack_vdo_config() - Convert a vdo_config to its packed on-disk representation. + * @config: The vdo config to convert. + * + * Return: The platform-independent representation of the config. + */ +static struct packed_vdo_config pack_vdo_config(struct vdo_config config) +{ + return (struct packed_vdo_config) { + .logical_blocks = __cpu_to_le64(config.logical_blocks), + .physical_blocks = __cpu_to_le64(config.physical_blocks), + .slab_size = __cpu_to_le64(config.slab_size), + .recovery_journal_size = __cpu_to_le64(config.recovery_journal_size), + .slab_journal_blocks = __cpu_to_le64(config.slab_journal_blocks), + }; +} + +/** + * pack_vdo_component() - Convert a vdo_component to its packed on-disk representation. + * @component: The VDO component data to convert. + * + * Return: The platform-independent representation of the component. + */ +static struct packed_vdo_component_41_0 pack_vdo_component(const struct vdo_component component) +{ + return (struct packed_vdo_component_41_0) { + .state = __cpu_to_le32(component.state), + .complete_recoveries = __cpu_to_le64(component.complete_recoveries), + .read_only_recoveries = __cpu_to_le64(component.read_only_recoveries), + .config = pack_vdo_config(component.config), + .nonce = __cpu_to_le64(component.nonce), + }; +} + +static void encode_vdo_component(u8 *buffer, size_t *offset, + struct vdo_component component) +{ + struct packed_vdo_component_41_0 packed; + + encode_version_number(buffer, offset, VDO_COMPONENT_DATA_41_0); + packed = pack_vdo_component(component); + memcpy(buffer + *offset, &packed, sizeof(packed)); + *offset += sizeof(packed); +} + +/** + * unpack_vdo_config() - Convert a packed_vdo_config to its native in-memory representation. + * @config: The packed vdo config to convert. + * + * Return: The native in-memory representation of the vdo config. + */ +static struct vdo_config unpack_vdo_config(struct packed_vdo_config config) +{ + return (struct vdo_config) { + .logical_blocks = __le64_to_cpu(config.logical_blocks), + .physical_blocks = __le64_to_cpu(config.physical_blocks), + .slab_size = __le64_to_cpu(config.slab_size), + .recovery_journal_size = __le64_to_cpu(config.recovery_journal_size), + .slab_journal_blocks = __le64_to_cpu(config.slab_journal_blocks), + }; +} + +/** + * unpack_vdo_component_41_0() - Convert a packed_vdo_component_41_0 to its native in-memory + * representation. + * @component: The packed vdo component data to convert. + * + * Return: The native in-memory representation of the component. + */ +static struct vdo_component unpack_vdo_component_41_0(struct packed_vdo_component_41_0 component) +{ + return (struct vdo_component) { + .state = __le32_to_cpu(component.state), + .complete_recoveries = __le64_to_cpu(component.complete_recoveries), + .read_only_recoveries = __le64_to_cpu(component.read_only_recoveries), + .config = unpack_vdo_config(component.config), + .nonce = __le64_to_cpu(component.nonce), + }; +} + +/** + * vdo_decode_component() - Decode the component data for the vdo itself out of the super block. + * + * Return: VDO_SUCCESS or an error. + */ +static int decode_vdo_component(u8 *buffer, size_t *offset, struct vdo_component *component) +{ + struct version_number version; + struct packed_vdo_component_41_0 packed; + int result; + + decode_version_number(buffer, offset, &version); + result = validate_version(version, VDO_COMPONENT_DATA_41_0, + "VDO component data"); + if (result != VDO_SUCCESS) + return result; + + memcpy(&packed, buffer + *offset, sizeof(packed)); + *offset += sizeof(packed); + *component = unpack_vdo_component_41_0(packed); + return VDO_SUCCESS; +} + +/** + * vdo_validate_config() - Validate constraints on a VDO config. + * @config: The VDO config. + * @physical_block_count: The minimum block count of the underlying storage. + * @logical_block_count: The expected logical size of the VDO, or 0 if the logical size may be + * unspecified. + * + * Return: A success or error code. + */ +int vdo_validate_config(const struct vdo_config *config, + block_count_t physical_block_count, + block_count_t logical_block_count) +{ + struct slab_config slab_config; + int result; + + result = ASSERT(config->slab_size > 0, "slab size unspecified"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(is_power_of_2(config->slab_size), + "slab size must be a power of two"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(config->slab_size <= (1 << MAX_VDO_SLAB_BITS), + "slab size must be less than or equal to 2^%d", + MAX_VDO_SLAB_BITS); + if (result != VDO_SUCCESS) + return result; + + result = ASSERT(config->slab_journal_blocks >= MINIMUM_VDO_SLAB_JOURNAL_BLOCKS, + "slab journal size meets minimum size"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(config->slab_journal_blocks <= config->slab_size, + "slab journal size is within expected bound"); + if (result != UDS_SUCCESS) + return result; + + result = vdo_configure_slab(config->slab_size, config->slab_journal_blocks, + &slab_config); + if (result != VDO_SUCCESS) + return result; + + result = ASSERT((slab_config.data_blocks >= 1), + "slab must be able to hold at least one block"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(config->physical_blocks > 0, "physical blocks unspecified"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(config->physical_blocks <= MAXIMUM_VDO_PHYSICAL_BLOCKS, + "physical block count %llu exceeds maximum %llu", + (unsigned long long) config->physical_blocks, + (unsigned long long) MAXIMUM_VDO_PHYSICAL_BLOCKS); + if (result != UDS_SUCCESS) + return VDO_OUT_OF_RANGE; + + if (physical_block_count != config->physical_blocks) { + uds_log_error("A physical size of %llu blocks was specified, not the %llu blocks configured in the vdo super block", + (unsigned long long) physical_block_count, + (unsigned long long) config->physical_blocks); + return VDO_PARAMETER_MISMATCH; + } + + if (logical_block_count > 0) { + result = ASSERT((config->logical_blocks > 0), + "logical blocks unspecified"); + if (result != UDS_SUCCESS) + return result; + + if (logical_block_count != config->logical_blocks) { + uds_log_error("A logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", + (unsigned long long) logical_block_count, + (unsigned long long) config->logical_blocks); + return VDO_PARAMETER_MISMATCH; + } + } + + result = ASSERT(config->logical_blocks <= MAXIMUM_VDO_LOGICAL_BLOCKS, + "logical blocks too large"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(config->recovery_journal_size > 0, + "recovery journal size unspecified"); + if (result != UDS_SUCCESS) + return result; + + result = ASSERT(is_power_of_2(config->recovery_journal_size), + "recovery journal size must be a power of two"); + if (result != UDS_SUCCESS) + return result; + + return result; +} + +/** + * vdo_destroy_component_states() - Clean up any allocations in a vdo_component_states. + * @states: The component states to destroy. + */ +void vdo_destroy_component_states(struct vdo_component_states *states) +{ + if (states == NULL) + return; + + vdo_uninitialize_layout(&states->layout); +} + +/** + * decode_components() - Decode the components now that we know the component data is a version we + * understand. + * @buffer: The buffer being decoded. + * @offset: The offset to start decoding from. + * @geometry: The vdo geometry + * @states: An object to hold the successfully decoded state. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check decode_components(u8 *buffer, size_t *offset, + struct volume_geometry *geometry, + struct vdo_component_states *states) +{ + int result; + + decode_vdo_component(buffer, offset, &states->vdo); + + result = decode_layout(buffer, offset, vdo_get_data_region_start(*geometry) + 1, + states->vdo.config.physical_blocks, &states->layout); + if (result != VDO_SUCCESS) + return result; + + result = decode_recovery_journal_state_7_0(buffer, offset, + &states->recovery_journal); + if (result != VDO_SUCCESS) + return result; + + result = decode_slab_depot_state_2_0(buffer, offset, &states->slab_depot); + if (result != VDO_SUCCESS) + return result; + + result = decode_block_map_state_2_0(buffer, offset, &states->block_map); + if (result != VDO_SUCCESS) + return result; + + ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, + "All decoded component data was used"); + return VDO_SUCCESS; +} + +/** + * vdo_decode_component_states() - Decode the payload of a super block. + * @buffer: The buffer containing the encoded super block contents. + * @geometry: The vdo geometry + * @states: A pointer to hold the decoded states. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_decode_component_states(u8 *buffer, struct volume_geometry *geometry, + struct vdo_component_states *states) +{ + int result; + size_t offset = VDO_COMPONENT_DATA_OFFSET; + + /* This is for backwards compatibility. */ + decode_u32_le(buffer, &offset, &states->unused); + + /* Check the VDO volume version */ + decode_version_number(buffer, &offset, &states->volume_version); + result = validate_version(VDO_VOLUME_VERSION_67_0, states->volume_version, + "volume"); + if (result != VDO_SUCCESS) + return result; + + result = decode_components(buffer, &offset, geometry, states); + if (result != VDO_SUCCESS) + vdo_uninitialize_layout(&states->layout); + + return result; +} + +/** + * vdo_validate_component_states() - Validate the decoded super block configuration. + * @states: The state decoded from the super block. + * @geometry_nonce: The nonce from the geometry block. + * @physical_size: The minimum block count of the underlying storage. + * @logical_size: The expected logical size of the VDO, or 0 if the logical size may be + * unspecified. + * + * Return: VDO_SUCCESS or an error if the configuration is invalid. + */ +int vdo_validate_component_states(struct vdo_component_states *states, + nonce_t geometry_nonce, block_count_t physical_size, + block_count_t logical_size) +{ + if (geometry_nonce != states->vdo.nonce) { + return uds_log_error_strerror(VDO_BAD_NONCE, + "Geometry nonce %llu does not match superblock nonce %llu", + (unsigned long long) geometry_nonce, + (unsigned long long) states->vdo.nonce); + } + + return vdo_validate_config(&states->vdo.config, physical_size, logical_size); +} + +/** + * vdo_encode_component_states() - Encode the state of all vdo components in the super block. + */ +static void vdo_encode_component_states(u8 *buffer, size_t *offset, + const struct vdo_component_states *states) +{ + /* This is for backwards compatibility. */ + encode_u32_le(buffer, offset, states->unused); + encode_version_number(buffer, offset, states->volume_version); + encode_vdo_component(buffer, offset, states->vdo); + encode_layout(buffer, offset, &states->layout); + encode_recovery_journal_state_7_0(buffer, offset, states->recovery_journal); + encode_slab_depot_state_2_0(buffer, offset, states->slab_depot); + encode_block_map_state_2_0(buffer, offset, states->block_map); + + ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, + "All super block component data was encoded"); +} + +/** + * vdo_encode_super_block() - Encode a super block into its on-disk representation. + */ +void vdo_encode_super_block(u8 *buffer, struct vdo_component_states *states) +{ + u32 checksum; + struct header header = SUPER_BLOCK_HEADER_12_0; + size_t offset = 0; + + header.size += VDO_COMPONENT_DATA_SIZE; + vdo_encode_header(buffer, &offset, &header); + vdo_encode_component_states(buffer, &offset, states); + + checksum = vdo_crc32(buffer, offset); + encode_u32_le(buffer, &offset, checksum); + + /* + * Even though the buffer is a full block, to avoid the potential corruption from a torn + * write, the entire encoding must fit in the first sector. + */ + ASSERT_LOG_ONLY(offset <= VDO_SECTOR_SIZE, + "entire superblock must fit in one sector"); +} + +/** + * vdo_decode_super_block() - Decode a super block from its on-disk representation. + */ +int vdo_decode_super_block(u8 *buffer) +{ + struct header header; + int result; + u32 checksum, saved_checksum; + size_t offset = 0; + + /* Decode and validate the header. */ + vdo_decode_header(buffer, &offset, &header); + result = vdo_validate_header(&SUPER_BLOCK_HEADER_12_0, &header, false, __func__); + if (result != VDO_SUCCESS) + return result; + + if (header.size > VDO_COMPONENT_DATA_SIZE + sizeof(u32)) { + /* + * We can't check release version or checksum until we know the content size, so we + * have to assume a version mismatch on unexpected values. + */ + return uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "super block contents too large: %zu", + header.size); + } + + /* Skip past the component data for now, to verify the checksum. */ + offset += VDO_COMPONENT_DATA_SIZE; + + checksum = vdo_crc32(buffer, offset); + decode_u32_le(buffer, &offset, &saved_checksum); + + result = ASSERT(offset == VDO_SUPER_BLOCK_FIXED_SIZE + VDO_COMPONENT_DATA_SIZE, + "must have decoded entire superblock payload"); + if (result != VDO_SUCCESS) + return result; + + return ((checksum != saved_checksum) ? VDO_CHECKSUM_MISMATCH : VDO_SUCCESS); +} diff --git a/drivers/md/dm-vdo/encodings.h b/drivers/md/dm-vdo/encodings.h new file mode 100644 index 000000000000..260822a3958a --- /dev/null +++ b/drivers/md/dm-vdo/encodings.h @@ -0,0 +1,1299 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_ENCODINGS_H +#define VDO_ENCODINGS_H + +#include +#include +#include +#include + +#include "numeric.h" +#include "uds.h" + +#include "constants.h" +#include "types.h" + +/* + * An in-memory representation of a version number for versioned structures on disk. + * + * A version number consists of two portions, a major version and a minor version. Any format + * change which does not require an explicit upgrade step from the previous version should + * increment the minor version. Any format change which either requires an explicit upgrade step, + * or is wholly incompatible (i.e. can not be upgraded to), should increment the major version, and + * set the minor version to 0. + */ +struct version_number { + u32 major_version; + u32 minor_version; +}; + +/* + * A packed, machine-independent, on-disk representation of a version_number. Both fields are + * stored in little-endian byte order. + */ +struct packed_version_number { + __le32 major_version; + __le32 minor_version; +} __packed; + +/* The registry of component ids for use in headers */ +#define VDO_SUPER_BLOCK 0 +#define VDO_LAYOUT 1 +#define VDO_RECOVERY_JOURNAL 2 +#define VDO_SLAB_DEPOT 3 +#define VDO_BLOCK_MAP 4 +#define VDO_GEOMETRY_BLOCK 5 + +/* The header for versioned data stored on disk. */ +struct header { + u32 id; /* The component this is a header for */ + struct version_number version; /* The version of the data format */ + size_t size; /* The size of the data following this header */ +}; + +/* A packed, machine-independent, on-disk representation of a component header. */ +struct packed_header { + __le32 id; + struct packed_version_number version; + __le64 size; +} __packed; + +enum { + VDO_GEOMETRY_BLOCK_LOCATION = 0, + VDO_GEOMETRY_MAGIC_NUMBER_SIZE = 8, + VDO_DEFAULT_GEOMETRY_BLOCK_VERSION = 5, +}; + +struct index_config { + u32 mem; + u32 unused; + bool sparse; +} __packed; + +enum volume_region_id { + VDO_INDEX_REGION = 0, + VDO_DATA_REGION = 1, + VDO_VOLUME_REGION_COUNT, +}; + +struct volume_region { + /* The ID of the region */ + enum volume_region_id id; + /* + * The absolute starting offset on the device. The region continues until the next region + * begins. + */ + physical_block_number_t start_block; +} __packed; + +struct volume_geometry { + /* For backwards compatibility */ + u32 unused; + /* The nonce of this volume */ + nonce_t nonce; + /* The uuid of this volume */ + uuid_t uuid; + /* The block offset to be applied to bios */ + block_count_t bio_offset; + /* The regions in ID order */ + struct volume_region regions[VDO_VOLUME_REGION_COUNT]; + /* The index config */ + struct index_config index_config; +} __packed; + +/* This volume geometry struct is used for sizing only */ +struct volume_geometry_4_0 { + /* For backwards compatibility */ + u32 unused; + /* The nonce of this volume */ + nonce_t nonce; + /* The uuid of this volume */ + uuid_t uuid; + /* The regions in ID order */ + struct volume_region regions[VDO_VOLUME_REGION_COUNT]; + /* The index config */ + struct index_config index_config; +} __packed; + +extern const u8 VDO_GEOMETRY_MAGIC_NUMBER[VDO_GEOMETRY_MAGIC_NUMBER_SIZE + 1]; + +/** + * DOC: Block map entries + * + * The entry for each logical block in the block map is encoded into five bytes, which saves space + * in both the on-disk and in-memory layouts. It consists of the 36 low-order bits of a + * physical_block_number_t (addressing 256 terabytes with a 4KB block size) and a 4-bit encoding of + * a block_mapping_state. + * + * Of the 8 high bits of the 5-byte structure: + * + * Bits 7..4: The four highest bits of the 36-bit physical block number + * Bits 3..0: The 4-bit block_mapping_state + * + * The following 4 bytes are the low order bytes of the physical block number, in little-endian + * order. + * + * Conversion functions to and from a data location are provided. + */ +struct block_map_entry { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned mapping_state : 4; + unsigned pbn_high_nibble : 4; +#else + unsigned pbn_high_nibble : 4; + unsigned mapping_state : 4; +#endif + + __le32 pbn_low_word; +} __packed; + +struct block_map_page_header { + __le64 nonce; + __le64 pbn; + + /** May be non-zero on disk */ + u8 unused_long_word[8]; + + /* Whether this page has been written twice to disk */ + bool initialized; + + /* Always zero on disk */ + u8 unused_byte1; + + /* May be non-zero on disk */ + u8 unused_byte2; + u8 unused_byte3; +} __packed; + +struct block_map_page { + struct packed_version_number version; + struct block_map_page_header header; + struct block_map_entry entries[]; +} __packed; + +enum block_map_page_validity { + VDO_BLOCK_MAP_PAGE_VALID, + VDO_BLOCK_MAP_PAGE_INVALID, + /* Valid page found in the wrong location on disk */ + VDO_BLOCK_MAP_PAGE_BAD, +}; + +struct block_map_state_2_0 { + physical_block_number_t flat_page_origin; + block_count_t flat_page_count; + physical_block_number_t root_origin; + block_count_t root_count; +} __packed; + +struct boundary { + page_number_t levels[VDO_BLOCK_MAP_TREE_HEIGHT]; +}; + +extern const struct header VDO_BLOCK_MAP_HEADER_2_0; + +/* The state of the recovery journal as encoded in the VDO super block. */ +struct recovery_journal_state_7_0 { + /** Sequence number to start the journal */ + sequence_number_t journal_start; + /** Number of logical blocks used by VDO */ + block_count_t logical_blocks_used; + /** Number of block map pages allocated */ + block_count_t block_map_data_blocks; +} __packed; + +extern const struct header VDO_RECOVERY_JOURNAL_HEADER_7_0; + +typedef u16 journal_entry_count_t; + +/* + * A recovery journal entry stores three physical locations: a data location that is the value of a + * single mapping in the block map tree, and the two locations of the block map pages and slots + * that are acquiring and releasing a reference to the location. The journal entry also stores an + * operation code that says whether the mapping is for a logical block or for the block map tree + * itself. + */ +struct recovery_journal_entry { + struct block_map_slot slot; + struct data_location mapping; + struct data_location unmapping; + enum journal_operation operation; +}; + +/* The packed, on-disk representation of a recovery journal entry. */ +struct packed_recovery_journal_entry { + /* + * In little-endian bit order: + * Bits 15..12: The four highest bits of the 36-bit physical block number of the block map + * tree page + * Bits 11..2: The 10-bit block map page slot number + * Bit 1..0: The journal_operation of the entry (this actually only requires 1 bit, but + * it is convenient to keep the extra bit as part of this field. + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned operation : 2; + unsigned slot_low : 6; + unsigned slot_high : 4; + unsigned pbn_high_nibble : 4; +#else + unsigned slot_low : 6; + unsigned operation : 2; + unsigned pbn_high_nibble : 4; + unsigned slot_high : 4; +#endif + + /* + * Bits 47..16: The 32 low-order bits of the block map page PBN, in little-endian byte + * order + */ + __le32 pbn_low_word; + + /* + * Bits 87..48: The five-byte block map entry encoding the location that will be stored in + * the block map page slot + */ + struct block_map_entry mapping; + + /* + * Bits 127..88: The five-byte block map entry encoding the location that was stored in the + * block map page slot + */ + struct block_map_entry unmapping; +} __packed; + +/* The packed, on-disk representation of an old format recovery journal entry. */ +struct packed_recovery_journal_entry_1 { + /* + * In little-endian bit order: + * Bits 15..12: The four highest bits of the 36-bit physical block number of the block map + * tree page + * Bits 11..2: The 10-bit block map page slot number + * Bits 1..0: The 2-bit journal_operation of the entry + * + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned operation : 2; + unsigned slot_low : 6; + unsigned slot_high : 4; + unsigned pbn_high_nibble : 4; +#else + unsigned slot_low : 6; + unsigned operation : 2; + unsigned pbn_high_nibble : 4; + unsigned slot_high : 4; +#endif + + /* + * Bits 47..16: The 32 low-order bits of the block map page PBN, in little-endian byte + * order + */ + __le32 pbn_low_word; + + /* + * Bits 87..48: The five-byte block map entry encoding the location that was or will be + * stored in the block map page slot + */ + struct block_map_entry block_map_entry; +} __packed; + +enum journal_operation_1 { + VDO_JOURNAL_DATA_DECREMENT = 0, + VDO_JOURNAL_DATA_INCREMENT = 1, + VDO_JOURNAL_BLOCK_MAP_DECREMENT = 2, + VDO_JOURNAL_BLOCK_MAP_INCREMENT = 3, +} __packed; + +struct recovery_block_header { + sequence_number_t block_map_head; /* Block map head sequence number */ + sequence_number_t slab_journal_head; /* Slab journal head seq. number */ + sequence_number_t sequence_number; /* Sequence number for this block */ + nonce_t nonce; /* A given VDO instance's nonce */ + block_count_t logical_blocks_used; /* Logical blocks in use */ + block_count_t block_map_data_blocks; /* Allocated block map pages */ + journal_entry_count_t entry_count; /* Number of entries written */ + u8 check_byte; /* The protection check byte */ + u8 recovery_count; /* Number of recoveries completed */ + enum vdo_metadata_type metadata_type; /* Metadata type */ +}; + +/* + * The packed, on-disk representation of a recovery journal block header. All fields are kept in + * little-endian byte order. + */ +struct packed_journal_header { + /* Block map head 64-bit sequence number */ + __le64 block_map_head; + + /* Slab journal head 64-bit sequence number */ + __le64 slab_journal_head; + + /* The 64-bit sequence number for this block */ + __le64 sequence_number; + + /* A given VDO instance's 64-bit nonce */ + __le64 nonce; + + /* 8-bit metadata type (should always be one for the recovery journal) */ + u8 metadata_type; + + /* 16-bit count of the entries encoded in the block */ + __le16 entry_count; + + /* 64-bit count of the logical blocks used when this block was opened */ + __le64 logical_blocks_used; + + /* 64-bit count of the block map blocks used when this block was opened */ + __le64 block_map_data_blocks; + + /* The protection check byte */ + u8 check_byte; + + /* The number of recoveries completed */ + u8 recovery_count; +} __packed; + +struct packed_journal_sector { + /* The protection check byte */ + u8 check_byte; + + /* The number of recoveries completed */ + u8 recovery_count; + + /* The number of entries in this sector */ + u8 entry_count; + + /* Journal entries for this sector */ + struct packed_recovery_journal_entry entries[]; +} __packed; + +enum { + /* The number of entries in each sector (except the last) when filled */ + RECOVERY_JOURNAL_ENTRIES_PER_SECTOR = + ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_sector)) / + sizeof(struct packed_recovery_journal_entry)), + RECOVERY_JOURNAL_ENTRIES_PER_BLOCK = RECOVERY_JOURNAL_ENTRIES_PER_SECTOR * 7, + /* The number of entries in a v1 recovery journal block. */ + RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK = 311, + /* The number of entries in each v1 sector (except the last) when filled */ + RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR = + ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_sector)) / + sizeof(struct packed_recovery_journal_entry_1)), + /* The number of entries in the last sector when a block is full */ + RECOVERY_JOURNAL_1_ENTRIES_IN_LAST_SECTOR = + (RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK % RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR), +}; + +/* A type representing a reference count of a block. */ +typedef u8 vdo_refcount_t; + +/* The absolute position of an entry in a recovery journal or slab journal. */ +struct journal_point { + sequence_number_t sequence_number; + journal_entry_count_t entry_count; +}; + +/* A packed, platform-independent encoding of a struct journal_point. */ +struct packed_journal_point { + /* + * The packed representation is the little-endian 64-bit representation of the low-order 48 + * bits of the sequence number, shifted up 16 bits, or'ed with the 16-bit entry count. + * + * Very long-term, the top 16 bits of the sequence number may not always be zero, as this + * encoding assumes--see BZ 1523240. + */ + __le64 encoded_point; +} __packed; + +/* Special vdo_refcount_t values. */ +#define EMPTY_REFERENCE_COUNT 0 +enum { + MAXIMUM_REFERENCE_COUNT = 254, + PROVISIONAL_REFERENCE_COUNT = 255, +}; + +enum { + COUNTS_PER_SECTOR = + ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_point)) / sizeof(vdo_refcount_t)), + COUNTS_PER_BLOCK = COUNTS_PER_SECTOR * VDO_SECTORS_PER_BLOCK, +}; + +/* The format of each sector of a reference_block on disk. */ +struct packed_reference_sector { + struct packed_journal_point commit_point; + vdo_refcount_t counts[COUNTS_PER_SECTOR]; +} __packed; + +struct packed_reference_block { + struct packed_reference_sector sectors[VDO_SECTORS_PER_BLOCK]; +}; + +struct slab_depot_state_2_0 { + struct slab_config slab_config; + physical_block_number_t first_block; + physical_block_number_t last_block; + zone_count_t zone_count; +} __packed; + +extern const struct header VDO_SLAB_DEPOT_HEADER_2_0; + +/* + * vdo_slab journal blocks may have one of two formats, depending upon whether or not any of the + * entries in the block are block map increments. Since the steady state for a VDO is that all of + * the necessary block map pages will be allocated, most slab journal blocks will have only data + * entries. Such blocks can hold more entries, hence the two formats. + */ + +/* A single slab journal entry */ +struct slab_journal_entry { + slab_block_number sbn; + enum journal_operation operation; + bool increment; +}; + +/* A single slab journal entry in its on-disk form */ +typedef struct { + u8 offset_low8; + u8 offset_mid8; + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned offset_high7 : 7; + unsigned increment : 1; +#else + unsigned increment : 1; + unsigned offset_high7 : 7; +#endif +} __packed packed_slab_journal_entry; + +/* The unpacked representation of the header of a slab journal block */ +struct slab_journal_block_header { + /* Sequence number for head of journal */ + sequence_number_t head; + /* Sequence number for this block */ + sequence_number_t sequence_number; + /* The nonce for a given VDO instance */ + nonce_t nonce; + /* Recovery journal point for last entry */ + struct journal_point recovery_point; + /* Metadata type */ + enum vdo_metadata_type metadata_type; + /* Whether this block contains block map increments */ + bool has_block_map_increments; + /* The number of entries in the block */ + journal_entry_count_t entry_count; +}; + +/* + * The packed, on-disk representation of a slab journal block header. All fields are kept in + * little-endian byte order. + */ +struct packed_slab_journal_block_header { + /* 64-bit sequence number for head of journal */ + __le64 head; + /* 64-bit sequence number for this block */ + __le64 sequence_number; + /* Recovery journal point for the last entry, packed into 64 bits */ + struct packed_journal_point recovery_point; + /* The 64-bit nonce for a given VDO instance */ + __le64 nonce; + /* 8-bit metadata type (should always be two, for the slab journal) */ + u8 metadata_type; + /* Whether this block contains block map increments */ + bool has_block_map_increments; + /* 16-bit count of the entries encoded in the block */ + __le16 entry_count; +} __packed; + +enum { + VDO_SLAB_JOURNAL_PAYLOAD_SIZE = + VDO_BLOCK_SIZE - sizeof(struct packed_slab_journal_block_header), + VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK = (VDO_SLAB_JOURNAL_PAYLOAD_SIZE * 8) / 25, + VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE = + ((VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK - 1) / 8) + 1, + VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK = + (VDO_SLAB_JOURNAL_PAYLOAD_SIZE / sizeof(packed_slab_journal_entry)), +}; + +/* The payload of a slab journal block which has block map increments */ +struct full_slab_journal_entries { + /* The entries themselves */ + packed_slab_journal_entry entries[VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK]; + /* The bit map indicating which entries are block map increments */ + u8 entry_types[VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE]; +} __packed; + +typedef union { + /* Entries which include block map increments */ + struct full_slab_journal_entries full_entries; + /* Entries which are only data updates */ + packed_slab_journal_entry entries[VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK]; + /* Ensure the payload fills to the end of the block */ + u8 space[VDO_SLAB_JOURNAL_PAYLOAD_SIZE]; +} __packed slab_journal_payload; + +struct packed_slab_journal_block { + struct packed_slab_journal_block_header header; + slab_journal_payload payload; +} __packed; + +/* The offset of a slab journal tail block. */ +typedef u8 tail_block_offset_t; + +struct slab_summary_entry { + /* Bits 7..0: The offset of the tail block within the slab journal */ + tail_block_offset_t tail_block_offset; + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* Bits 13..8: A hint about the fullness of the slab */ + unsigned int fullness_hint : 6; + /* Bit 14: Whether the ref_counts must be loaded from the layer */ + unsigned int load_ref_counts : 1; + /* Bit 15: The believed cleanliness of this slab */ + unsigned int is_dirty : 1; +#else + /* Bit 15: The believed cleanliness of this slab */ + unsigned int is_dirty : 1; + /* Bit 14: Whether the ref_counts must be loaded from the layer */ + unsigned int load_ref_counts : 1; + /* Bits 13..8: A hint about the fullness of the slab */ + unsigned int fullness_hint : 6; +#endif +} __packed; + +enum { + VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS = 6, + VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK = VDO_BLOCK_SIZE / sizeof(struct slab_summary_entry), + VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE = MAX_VDO_SLABS / VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK, + VDO_SLAB_SUMMARY_BLOCKS = VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * MAX_VDO_PHYSICAL_ZONES, +}; + +struct layout { + physical_block_number_t start; + block_count_t size; + physical_block_number_t first_free; + physical_block_number_t last_free; + size_t num_partitions; + struct partition *head; +}; + +struct partition { + enum partition_id id; /* The id of this partition */ + physical_block_number_t offset; /* The offset into the layout of this partition */ + block_count_t count; /* The number of blocks in the partition */ + struct partition *next; /* A pointer to the next partition in the layout */ +}; + +struct layout_3_0 { + physical_block_number_t first_free; + physical_block_number_t last_free; + u8 partition_count; +} __packed; + +struct partition_3_0 { + enum partition_id id; + physical_block_number_t offset; + physical_block_number_t base; /* unused but retained for backwards compatibility */ + block_count_t count; +} __packed; + +/* + * The configuration of the VDO service. + */ +struct vdo_config { + block_count_t logical_blocks; /* number of logical blocks */ + block_count_t physical_blocks; /* number of physical blocks */ + block_count_t slab_size; /* number of blocks in a slab */ + block_count_t recovery_journal_size; /* number of recovery journal blocks */ + block_count_t slab_journal_blocks; /* number of slab journal blocks */ +}; + +/* This is the structure that captures the vdo fields saved as a super block component. */ +struct vdo_component { + enum vdo_state state; + u64 complete_recoveries; + u64 read_only_recoveries; + struct vdo_config config; + nonce_t nonce; +}; + +/* + * A packed, machine-independent, on-disk representation of the vdo_config in the VDO component + * data in the super block. + */ +struct packed_vdo_config { + __le64 logical_blocks; + __le64 physical_blocks; + __le64 slab_size; + __le64 recovery_journal_size; + __le64 slab_journal_blocks; +} __packed; + +/* + * A packed, machine-independent, on-disk representation of version 41.0 of the VDO component data + * in the super block. + */ +struct packed_vdo_component_41_0 { + __le32 state; + __le64 complete_recoveries; + __le64 read_only_recoveries; + struct packed_vdo_config config; + __le64 nonce; +} __packed; + +/* + * The version of the on-disk format of a VDO volume. This should be incremented any time the + * on-disk representation of any VDO structure changes. Changes which require only online upgrade + * steps should increment the minor version. Changes which require an offline upgrade or which can + * not be upgraded to at all should increment the major version and set the minor version to 0. + */ +extern const struct version_number VDO_VOLUME_VERSION_67_0; + +enum { + VDO_ENCODED_HEADER_SIZE = sizeof(struct packed_header), + BLOCK_MAP_COMPONENT_ENCODED_SIZE = + VDO_ENCODED_HEADER_SIZE + sizeof(struct block_map_state_2_0), + RECOVERY_JOURNAL_COMPONENT_ENCODED_SIZE = + VDO_ENCODED_HEADER_SIZE + sizeof(struct recovery_journal_state_7_0), + SLAB_DEPOT_COMPONENT_ENCODED_SIZE = + VDO_ENCODED_HEADER_SIZE + sizeof(struct slab_depot_state_2_0), + VDO_PARTITION_COUNT = 4, + VDO_LAYOUT_ENCODED_SIZE = (VDO_ENCODED_HEADER_SIZE + + sizeof(struct layout_3_0) + + (sizeof(struct partition_3_0) * VDO_PARTITION_COUNT)), + VDO_SUPER_BLOCK_FIXED_SIZE = VDO_ENCODED_HEADER_SIZE + sizeof(u32), + VDO_MAX_COMPONENT_DATA_SIZE = VDO_SECTOR_SIZE - VDO_SUPER_BLOCK_FIXED_SIZE, + VDO_COMPONENT_ENCODED_SIZE = + (sizeof(struct packed_version_number) + sizeof(struct packed_vdo_component_41_0)), + VDO_COMPONENT_DATA_OFFSET = VDO_ENCODED_HEADER_SIZE, + VDO_COMPONENT_DATA_SIZE = (sizeof(u32) + + sizeof(struct packed_version_number) + + VDO_COMPONENT_ENCODED_SIZE + + VDO_LAYOUT_ENCODED_SIZE + + RECOVERY_JOURNAL_COMPONENT_ENCODED_SIZE + + SLAB_DEPOT_COMPONENT_ENCODED_SIZE + + BLOCK_MAP_COMPONENT_ENCODED_SIZE), +}; + +/* The entirety of the component data encoded in the VDO super block. */ +struct vdo_component_states { + /* For backwards compatibility */ + u32 unused; + + /* The VDO volume version */ + struct version_number volume_version; + + /* Components */ + struct vdo_component vdo; + struct block_map_state_2_0 block_map; + struct recovery_journal_state_7_0 recovery_journal; + struct slab_depot_state_2_0 slab_depot; + + /* Our partitioning of the underlying storage */ + struct layout layout; +}; + +/** + * vdo_are_same_version() - Check whether two version numbers are the same. + * @version_a: The first version. + * @version_b: The second version. + * + * Return: true if the two versions are the same. + */ +static inline bool vdo_are_same_version(struct version_number version_a, + struct version_number version_b) +{ + return ((version_a.major_version == version_b.major_version) && + (version_a.minor_version == version_b.minor_version)); +} + +/** + * vdo_is_upgradable_version() - Check whether an actual version is upgradable to an expected + * version. + * @expected_version: The expected version. + * @actual_version: The version being validated. + * + * An actual version is upgradable if its major number is expected but its minor number differs, + * and the expected version's minor number is greater than the actual version's minor number. + * + * Return: true if the actual version is upgradable. + */ +static inline bool vdo_is_upgradable_version(struct version_number expected_version, + struct version_number actual_version) +{ + return ((expected_version.major_version == actual_version.major_version) && + (expected_version.minor_version > actual_version.minor_version)); +} + +int __must_check vdo_validate_header(const struct header *expected_header, + const struct header *actual_header, bool exact_size, + const char *component_name); + +void vdo_encode_header(u8 *buffer, size_t *offset, const struct header *header); +void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header); + +/** + * vdo_pack_version_number() - Convert a version_number to its packed on-disk representation. + * @version: The version number to convert. + * + * Return: the platform-independent representation of the version + */ +static inline struct packed_version_number vdo_pack_version_number(struct version_number version) +{ + return (struct packed_version_number) { + .major_version = __cpu_to_le32(version.major_version), + .minor_version = __cpu_to_le32(version.minor_version), + }; +} + +/** + * vdo_unpack_version_number() - Convert a packed_version_number to its native in-memory + * representation. + * @version: The version number to convert. + * + * Return: The platform-independent representation of the version. + */ +static inline struct version_number vdo_unpack_version_number(struct packed_version_number version) +{ + return (struct version_number) { + .major_version = __le32_to_cpu(version.major_version), + .minor_version = __le32_to_cpu(version.minor_version), + }; +} + +/** + * vdo_pack_header() - Convert a component header to its packed on-disk representation. + * @header: The header to convert. + * + * Return: the platform-independent representation of the header + */ +static inline struct packed_header vdo_pack_header(const struct header *header) +{ + return (struct packed_header) { + .id = __cpu_to_le32(header->id), + .version = vdo_pack_version_number(header->version), + .size = __cpu_to_le64(header->size), + }; +} + +/** + * vdo_unpack_header() - Convert a packed_header to its native in-memory representation. + * @header: The header to convert. + * + * Return: The platform-independent representation of the version. + */ +static inline struct header vdo_unpack_header(const struct packed_header *header) +{ + return (struct header) { + .id = __le32_to_cpu(header->id), + .version = vdo_unpack_version_number(header->version), + .size = __le64_to_cpu(header->size), + }; +} + +/** + * vdo_get_index_region_start() - Get the start of the index region from a geometry. + * @geometry: The geometry. + * + * Return: The start of the index region. + */ +static inline physical_block_number_t __must_check +vdo_get_index_region_start(struct volume_geometry geometry) +{ + return geometry.regions[VDO_INDEX_REGION].start_block; +} + +/** + * vdo_get_data_region_start() - Get the start of the data region from a geometry. + * @geometry: The geometry. + * + * Return: The start of the data region. + */ +static inline physical_block_number_t __must_check +vdo_get_data_region_start(struct volume_geometry geometry) +{ + return geometry.regions[VDO_DATA_REGION].start_block; +} + +/** + * vdo_get_index_region_size() - Get the size of the index region from a geometry. + * @geometry: The geometry. + * + * Return: The size of the index region. + */ +static inline physical_block_number_t __must_check +vdo_get_index_region_size(struct volume_geometry geometry) +{ + return vdo_get_data_region_start(geometry) - + vdo_get_index_region_start(geometry); +} + +int __must_check vdo_parse_geometry_block(unsigned char *block, + struct volume_geometry *geometry); + +static inline bool vdo_is_state_compressed(const enum block_mapping_state mapping_state) +{ + return (mapping_state > VDO_MAPPING_STATE_UNCOMPRESSED); +} + +static inline struct block_map_entry +vdo_pack_block_map_entry(physical_block_number_t pbn, enum block_mapping_state mapping_state) +{ + return (struct block_map_entry) { + .mapping_state = (mapping_state & 0x0F), + .pbn_high_nibble = ((pbn >> 32) & 0x0F), + .pbn_low_word = __cpu_to_le32(pbn & UINT_MAX), + }; +} + +static inline struct data_location vdo_unpack_block_map_entry(const struct block_map_entry *entry) +{ + physical_block_number_t low32 = __le32_to_cpu(entry->pbn_low_word); + physical_block_number_t high4 = entry->pbn_high_nibble; + + return (struct data_location) { + .pbn = ((high4 << 32) | low32), + .state = entry->mapping_state, + }; +} + +static inline bool vdo_is_mapped_location(const struct data_location *location) +{ + return (location->state != VDO_MAPPING_STATE_UNMAPPED); +} + +static inline bool vdo_is_valid_location(const struct data_location *location) +{ + if (location->pbn == VDO_ZERO_BLOCK) + return !vdo_is_state_compressed(location->state); + else + return vdo_is_mapped_location(location); +} + +static inline physical_block_number_t __must_check +vdo_get_block_map_page_pbn(const struct block_map_page *page) +{ + return __le64_to_cpu(page->header.pbn); +} + +struct block_map_page *vdo_format_block_map_page(void *buffer, nonce_t nonce, + physical_block_number_t pbn, + bool initialized); + +enum block_map_page_validity __must_check vdo_validate_block_map_page(struct block_map_page *page, + nonce_t nonce, + physical_block_number_t pbn); + +static inline page_count_t vdo_compute_block_map_page_count(block_count_t entries) +{ + return DIV_ROUND_UP(entries, VDO_BLOCK_MAP_ENTRIES_PER_PAGE); +} + +block_count_t __must_check vdo_compute_new_forest_pages(root_count_t root_count, + struct boundary *old_sizes, + block_count_t entries, + struct boundary *new_sizes); + +/** + * vdo_pack_recovery_journal_entry() - Return the packed, on-disk representation of a recovery + * journal entry. + * @entry: The journal entry to pack. + * + * Return: The packed representation of the journal entry. + */ +static inline struct packed_recovery_journal_entry +vdo_pack_recovery_journal_entry(const struct recovery_journal_entry *entry) +{ + return (struct packed_recovery_journal_entry) { + .operation = entry->operation, + .slot_low = entry->slot.slot & 0x3F, + .slot_high = (entry->slot.slot >> 6) & 0x0F, + .pbn_high_nibble = (entry->slot.pbn >> 32) & 0x0F, + .pbn_low_word = __cpu_to_le32(entry->slot.pbn & UINT_MAX), + .mapping = vdo_pack_block_map_entry(entry->mapping.pbn, + entry->mapping.state), + .unmapping = vdo_pack_block_map_entry(entry->unmapping.pbn, + entry->unmapping.state), + }; +} + +/** + * vdo_unpack_recovery_journal_entry() - Unpack the on-disk representation of a recovery journal + * entry. + * @entry: The recovery journal entry to unpack. + * + * Return: The unpacked entry. + */ +static inline struct recovery_journal_entry +vdo_unpack_recovery_journal_entry(const struct packed_recovery_journal_entry *entry) +{ + physical_block_number_t low32 = __le32_to_cpu(entry->pbn_low_word); + physical_block_number_t high4 = entry->pbn_high_nibble; + + return (struct recovery_journal_entry) { + .operation = entry->operation, + .slot = { + .pbn = ((high4 << 32) | low32), + .slot = (entry->slot_low | (entry->slot_high << 6)), + }, + .mapping = vdo_unpack_block_map_entry(&entry->mapping), + .unmapping = vdo_unpack_block_map_entry(&entry->unmapping), + }; +} + +const char * __must_check vdo_get_journal_operation_name(enum journal_operation operation); + +/** + * vdo_is_valid_recovery_journal_sector() - Determine whether the header of the given sector could + * describe a valid sector for the given journal block + * header. + * @header: The unpacked block header to compare against. + * @sector: The packed sector to check. + * @sector_number: The number of the sector being checked. + * + * Return: true if the sector matches the block header. + */ +static inline bool __must_check +vdo_is_valid_recovery_journal_sector(const struct recovery_block_header *header, + const struct packed_journal_sector *sector, + u8 sector_number) +{ + if ((header->check_byte != sector->check_byte) || + (header->recovery_count != sector->recovery_count)) + return false; + + if (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL_2) + return sector->entry_count <= RECOVERY_JOURNAL_ENTRIES_PER_SECTOR; + + if (sector_number == 7) + return sector->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_IN_LAST_SECTOR; + + return sector->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR; +} + +/** + * vdo_compute_recovery_journal_block_number() - Compute the physical block number of the recovery + * journal block which would have a given sequence + * number. + * @journal_size: The size of the journal. + * @sequence_number: The sequence number. + * + * Return: The pbn of the journal block which would the specified sequence number. + */ +static inline physical_block_number_t __must_check +vdo_compute_recovery_journal_block_number(block_count_t journal_size, + sequence_number_t sequence_number) +{ + /* + * Since journal size is a power of two, the block number modulus can just be extracted + * from the low-order bits of the sequence. + */ + return (sequence_number & (journal_size - 1)); +} + +/** + * vdo_get_journal_block_sector() - Find the recovery journal sector from the block header and + * sector number. + * @header: The header of the recovery journal block. + * @sector_number: The index of the sector (1-based). + * + * Return: A packed recovery journal sector. + */ +static inline struct packed_journal_sector * __must_check +vdo_get_journal_block_sector(struct packed_journal_header *header, int sector_number) +{ + char *sector_data = ((char *) header) + (VDO_SECTOR_SIZE * sector_number); + + return (struct packed_journal_sector *) sector_data; +} + +/** + * vdo_pack_recovery_block_header() - Generate the packed representation of a recovery block + * header. + * @header: The header containing the values to encode. + * @packed: The header into which to pack the values. + */ +static inline void vdo_pack_recovery_block_header(const struct recovery_block_header *header, + struct packed_journal_header *packed) +{ + *packed = (struct packed_journal_header) { + .block_map_head = __cpu_to_le64(header->block_map_head), + .slab_journal_head = __cpu_to_le64(header->slab_journal_head), + .sequence_number = __cpu_to_le64(header->sequence_number), + .nonce = __cpu_to_le64(header->nonce), + .logical_blocks_used = __cpu_to_le64(header->logical_blocks_used), + .block_map_data_blocks = __cpu_to_le64(header->block_map_data_blocks), + .entry_count = __cpu_to_le16(header->entry_count), + .check_byte = header->check_byte, + .recovery_count = header->recovery_count, + .metadata_type = header->metadata_type, + }; +} + +/** + * vdo_unpack_recovery_block_header() - Decode the packed representation of a recovery block + * header. + * @packed: The packed header to decode. + * + * Return: The unpacked header. + */ +static inline struct recovery_block_header +vdo_unpack_recovery_block_header(const struct packed_journal_header *packed) +{ + return (struct recovery_block_header) { + .block_map_head = __le64_to_cpu(packed->block_map_head), + .slab_journal_head = __le64_to_cpu(packed->slab_journal_head), + .sequence_number = __le64_to_cpu(packed->sequence_number), + .nonce = __le64_to_cpu(packed->nonce), + .logical_blocks_used = __le64_to_cpu(packed->logical_blocks_used), + .block_map_data_blocks = __le64_to_cpu(packed->block_map_data_blocks), + .entry_count = __le16_to_cpu(packed->entry_count), + .check_byte = packed->check_byte, + .recovery_count = packed->recovery_count, + .metadata_type = packed->metadata_type, + }; +} + +/** + * vdo_compute_slab_count() - Compute the number of slabs a depot with given parameters would have. + * @first_block: PBN of the first data block. + * @last_block: PBN of the last data block. + * @slab_size_shift: Exponent for the number of blocks per slab. + * + * Return: The number of slabs. + */ +static inline slab_count_t vdo_compute_slab_count(physical_block_number_t first_block, + physical_block_number_t last_block, + unsigned int slab_size_shift) +{ + return (slab_count_t) ((last_block - first_block) >> slab_size_shift); +} + +int __must_check vdo_configure_slab_depot(const struct partition *partition, + struct slab_config slab_config, + zone_count_t zone_count, + struct slab_depot_state_2_0 *state); + +int __must_check vdo_configure_slab(block_count_t slab_size, + block_count_t slab_journal_blocks, + struct slab_config *slab_config); + +/** + * vdo_get_saved_reference_count_size() - Get the number of blocks required to save a reference + * counts state covering the specified number of data + * blocks. + * @block_count: The number of physical data blocks that can be referenced. + * + * Return: The number of blocks required to save reference counts with the given block count. + */ +static inline block_count_t vdo_get_saved_reference_count_size(block_count_t block_count) +{ + return DIV_ROUND_UP(block_count, COUNTS_PER_BLOCK); +} + +/** + * vdo_get_slab_journal_start_block() - Get the physical block number of the start of the slab + * journal relative to the start block allocator partition. + * @slab_config: The slab configuration of the VDO. + * @origin: The first block of the slab. + */ +static inline physical_block_number_t __must_check +vdo_get_slab_journal_start_block(const struct slab_config *slab_config, + physical_block_number_t origin) +{ + return origin + slab_config->data_blocks + slab_config->reference_count_blocks; +} + +/** + * vdo_advance_journal_point() - Move the given journal point forward by one entry. + * @point: The journal point to adjust. + * @entries_per_block: The number of entries in one full block. + */ +static inline void vdo_advance_journal_point(struct journal_point *point, + journal_entry_count_t entries_per_block) +{ + point->entry_count++; + if (point->entry_count == entries_per_block) { + point->sequence_number++; + point->entry_count = 0; + } +} + +/** + * vdo_before_journal_point() - Check whether the first point precedes the second point. + * @first: The first journal point. + * @second: The second journal point. + * + * Return: true if the first point precedes the second point. + */ +static inline bool vdo_before_journal_point(const struct journal_point *first, + const struct journal_point *second) +{ + return ((first->sequence_number < second->sequence_number) || + ((first->sequence_number == second->sequence_number) && + (first->entry_count < second->entry_count))); +} + +/** + * vdo_pack_journal_point() - Encode the journal location represented by a + * journal_point into a packed_journal_point. + * @unpacked: The unpacked input point. + * @packed: The packed output point. + */ +static inline void vdo_pack_journal_point(const struct journal_point *unpacked, + struct packed_journal_point *packed) +{ + packed->encoded_point = + __cpu_to_le64((unpacked->sequence_number << 16) | unpacked->entry_count); +} + +/** + * vdo_unpack_journal_point() - Decode the journal location represented by a packed_journal_point + * into a journal_point. + * @packed: The packed input point. + * @unpacked: The unpacked output point. + */ +static inline void vdo_unpack_journal_point(const struct packed_journal_point *packed, + struct journal_point *unpacked) +{ + u64 native = __le64_to_cpu(packed->encoded_point); + + unpacked->sequence_number = (native >> 16); + unpacked->entry_count = (native & 0xffff); +} + +/** + * vdo_pack_slab_journal_block_header() - Generate the packed representation of a slab block + * header. + * @header: The header containing the values to encode. + * @packed: The header into which to pack the values. + */ +static inline void +vdo_pack_slab_journal_block_header(const struct slab_journal_block_header *header, + struct packed_slab_journal_block_header *packed) +{ + packed->head = __cpu_to_le64(header->head); + packed->sequence_number = __cpu_to_le64(header->sequence_number); + packed->nonce = __cpu_to_le64(header->nonce); + packed->entry_count = __cpu_to_le16(header->entry_count); + packed->metadata_type = header->metadata_type; + packed->has_block_map_increments = header->has_block_map_increments; + + vdo_pack_journal_point(&header->recovery_point, &packed->recovery_point); +} + +/** + * vdo_unpack_slab_journal_block_header() - Decode the packed representation of a slab block + * header. + * @packed: The packed header to decode. + * @header: The header into which to unpack the values. + */ +static inline void +vdo_unpack_slab_journal_block_header(const struct packed_slab_journal_block_header *packed, + struct slab_journal_block_header *header) +{ + *header = (struct slab_journal_block_header) { + .head = __le64_to_cpu(packed->head), + .sequence_number = __le64_to_cpu(packed->sequence_number), + .nonce = __le64_to_cpu(packed->nonce), + .entry_count = __le16_to_cpu(packed->entry_count), + .metadata_type = packed->metadata_type, + .has_block_map_increments = packed->has_block_map_increments, + }; + vdo_unpack_journal_point(&packed->recovery_point, &header->recovery_point); +} + +/** + * vdo_pack_slab_journal_entry() - Generate the packed encoding of a slab journal entry. + * @packed: The entry into which to pack the values. + * @sbn: The slab block number of the entry to encode. + * @is_increment: The increment flag. + */ +static inline void vdo_pack_slab_journal_entry(packed_slab_journal_entry *packed, + slab_block_number sbn, bool is_increment) +{ + packed->offset_low8 = (sbn & 0x0000FF); + packed->offset_mid8 = (sbn & 0x00FF00) >> 8; + packed->offset_high7 = (sbn & 0x7F0000) >> 16; + packed->increment = is_increment ? 1 : 0; +} + +/** + * vdo_unpack_slab_journal_entry() - Decode the packed representation of a slab journal entry. + * @packed: The packed entry to decode. + * + * Return: The decoded slab journal entry. + */ +static inline struct slab_journal_entry __must_check +vdo_unpack_slab_journal_entry(const packed_slab_journal_entry *packed) +{ + struct slab_journal_entry entry; + + entry.sbn = packed->offset_high7; + entry.sbn <<= 8; + entry.sbn |= packed->offset_mid8; + entry.sbn <<= 8; + entry.sbn |= packed->offset_low8; + entry.operation = VDO_JOURNAL_DATA_REMAPPING; + entry.increment = packed->increment; + return entry; +} + +struct slab_journal_entry __must_check +vdo_decode_slab_journal_entry(struct packed_slab_journal_block *block, + journal_entry_count_t entry_count); + +/** + * vdo_get_slab_summary_hint_shift() - Compute the shift for slab summary hints. + * @slab_size_shift: Exponent for the number of blocks per slab. + * + * Return: The hint shift. + */ +static inline u8 __must_check vdo_get_slab_summary_hint_shift(unsigned int slab_size_shift) +{ + return ((slab_size_shift > VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS) ? + (slab_size_shift - VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS) : + 0); +} + +int __must_check vdo_initialize_layout(block_count_t size, + physical_block_number_t offset, + block_count_t block_map_blocks, + block_count_t journal_blocks, + block_count_t summary_blocks, + struct layout *layout); + +void vdo_uninitialize_layout(struct layout *layout); + +int __must_check vdo_get_partition(struct layout *layout, enum partition_id id, + struct partition **partition_ptr); + +struct partition * __must_check vdo_get_known_partition(struct layout *layout, + enum partition_id id); + +int vdo_validate_config(const struct vdo_config *config, + block_count_t physical_block_count, + block_count_t logical_block_count); + +void vdo_destroy_component_states(struct vdo_component_states *states); + +int __must_check vdo_decode_component_states(u8 *buffer, + struct volume_geometry *geometry, + struct vdo_component_states *states); + +int __must_check vdo_validate_component_states(struct vdo_component_states *states, + nonce_t geometry_nonce, + block_count_t physical_size, + block_count_t logical_size); + +void vdo_encode_super_block(u8 *buffer, struct vdo_component_states *states); +int __must_check vdo_decode_super_block(u8 *buffer); + +/* We start with 0L and postcondition with ~0L to match our historical usage in userspace. */ +static inline u32 vdo_crc32(const void *buf, unsigned long len) +{ + return (crc32(0L, buf, len) ^ ~0L); +} + +#endif /* VDO_ENCODINGS_H */ From a9457ab9d084047d6ac6a920d95bf431255672b0 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:10:00 -0500 Subject: [PATCH 0583/1038] dm vdo: add statistics reporting Add data and methods to report statisics. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/message-stats.c | 1215 +++++++++++++++++++++++++++++ drivers/md/dm-vdo/message-stats.h | 13 + drivers/md/dm-vdo/statistics.h | 278 +++++++ 3 files changed, 1506 insertions(+) create mode 100644 drivers/md/dm-vdo/message-stats.c create mode 100644 drivers/md/dm-vdo/message-stats.h create mode 100644 drivers/md/dm-vdo/statistics.h diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c new file mode 100644 index 000000000000..5be8503ed96e --- /dev/null +++ b/drivers/md/dm-vdo/message-stats.c @@ -0,0 +1,1215 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "dedupe.h" +#include "logger.h" +#include "memory-alloc.h" +#include "message-stats.h" +#include "statistics.h" +#include "thread-device.h" +#include "vdo.h" + +static int write_u64(char *prefix, + u64 value, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int count = scnprintf(*buf, *maxlen, "%s%llu%s", + prefix == NULL ? "" : prefix, + value, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; + if (count >= *maxlen) + return VDO_UNEXPECTED_EOF; + return VDO_SUCCESS; +} + +static int write_u32(char *prefix, + u32 value, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int count = scnprintf(*buf, *maxlen, "%s%u%s", + prefix == NULL ? "" : prefix, + value, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; + if (count >= *maxlen) + return VDO_UNEXPECTED_EOF; + return VDO_SUCCESS; +} + +static int write_block_count_t(char *prefix, + block_count_t value, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int count = scnprintf(*buf, *maxlen, "%s%llu%s", + prefix == NULL ? "" : prefix, + value, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; + if (count >= *maxlen) + return VDO_UNEXPECTED_EOF; + return VDO_SUCCESS; +} + +static int write_string(char *prefix, + char *value, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int count = scnprintf(*buf, *maxlen, "%s%s%s", + prefix == NULL ? "" : prefix, + value, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; + if (count >= *maxlen) + return VDO_UNEXPECTED_EOF; + return VDO_SUCCESS; +} + +static int write_bool(char *prefix, + bool value, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int count = scnprintf(*buf, *maxlen, "%s%d%s", + prefix == NULL ? "" : prefix, + value, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; + if (count >= *maxlen) + return VDO_UNEXPECTED_EOF; + return VDO_SUCCESS; +} + +static int write_u8(char *prefix, + u8 value, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int count = scnprintf(*buf, *maxlen, "%s%u%s", + prefix == NULL ? "" : prefix, + value, + suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; + if (count >= *maxlen) + return VDO_UNEXPECTED_EOF; + return VDO_SUCCESS; +} + +static int write_block_allocator_statistics(char *prefix, + struct block_allocator_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* The total number of slabs from which blocks may be allocated */ + result = write_u64("slabCount : ", + stats->slab_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The total number of slabs from which blocks have ever been allocated */ + result = write_u64("slabsOpened : ", + stats->slabs_opened, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The number of times since loading that a slab has been re-opened */ + result = write_u64("slabsReopened : ", + stats->slabs_reopened, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_commit_statistics(char *prefix, + struct commit_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* The total number of items on which processing has started */ + result = write_u64("started : ", + stats->started, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The total number of items for which a write operation has been issued */ + result = write_u64("written : ", + stats->written, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The total number of items for which a write operation has completed */ + result = write_u64("committed : ", + stats->committed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_recovery_journal_statistics(char *prefix, + struct recovery_journal_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the on-disk journal was full */ + result = write_u64("diskFull : ", + stats->disk_full, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the recovery journal requested slab journal commits. */ + result = write_u64("slabJournalCommitsRequested : ", + stats->slab_journal_commits_requested, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Write/Commit totals for individual journal entries */ + result = write_commit_statistics("entries : ", + &stats->entries, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Write/Commit totals for journal blocks */ + result = write_commit_statistics("blocks : ", + &stats->blocks, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_packer_statistics(char *prefix, + struct packer_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of compressed data items written since startup */ + result = write_u64("compressedFragmentsWritten : ", + stats->compressed_fragments_written, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of blocks containing compressed items written since startup */ + result = write_u64("compressedBlocksWritten : ", + stats->compressed_blocks_written, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of VIOs that are pending in the packer */ + result = write_u64("compressedFragmentsInPacker : ", + stats->compressed_fragments_in_packer, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_slab_journal_statistics(char *prefix, + struct slab_journal_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the on-disk journal was full */ + result = write_u64("diskFullCount : ", + stats->disk_full_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times an entry was added over the flush threshold */ + result = write_u64("flushCount : ", + stats->flush_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times an entry was added over the block threshold */ + result = write_u64("blockedCount : ", + stats->blocked_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times a tail block was written */ + result = write_u64("blocksWritten : ", + stats->blocks_written, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times we had to wait for the tail to write */ + result = write_u64("tailBusyCount : ", + stats->tail_busy_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_slab_summary_statistics(char *prefix, + struct slab_summary_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of blocks written */ + result = write_u64("blocksWritten : ", + stats->blocks_written, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_ref_counts_statistics(char *prefix, + struct ref_counts_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of reference blocks written */ + result = write_u64("blocksWritten : ", + stats->blocks_written, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_block_map_statistics(char *prefix, + struct block_map_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of dirty (resident) pages */ + result = write_u32("dirtyPages : ", + stats->dirty_pages, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of clean (resident) pages */ + result = write_u32("cleanPages : ", + stats->clean_pages, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of free pages */ + result = write_u32("freePages : ", + stats->free_pages, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of pages in failed state */ + result = write_u32("failedPages : ", + stats->failed_pages, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of pages incoming */ + result = write_u32("incomingPages : ", + stats->incoming_pages, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of pages outgoing */ + result = write_u32("outgoingPages : ", + stats->outgoing_pages, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* how many times free page not avail */ + result = write_u32("cachePressure : ", + stats->cache_pressure, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of get_vdo_page() calls for read */ + result = write_u64("readCount : ", + stats->read_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of get_vdo_page() calls for write */ + result = write_u64("writeCount : ", + stats->write_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of times pages failed to read */ + result = write_u64("failedReads : ", + stats->failed_reads, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of times pages failed to write */ + result = write_u64("failedWrites : ", + stats->failed_writes, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of gets that are reclaimed */ + result = write_u64("reclaimed : ", + stats->reclaimed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of gets for outgoing pages */ + result = write_u64("readOutgoing : ", + stats->read_outgoing, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of gets that were already there */ + result = write_u64("foundInCache : ", + stats->found_in_cache, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of gets requiring discard */ + result = write_u64("discardRequired : ", + stats->discard_required, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of gets enqueued for their page */ + result = write_u64("waitForPage : ", + stats->wait_for_page, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of gets that have to fetch */ + result = write_u64("fetchRequired : ", + stats->fetch_required, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of page fetches */ + result = write_u64("pagesLoaded : ", + stats->pages_loaded, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of page saves */ + result = write_u64("pagesSaved : ", + stats->pages_saved, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* the number of flushes issued */ + result = write_u64("flushCount : ", + stats->flush_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_hash_lock_statistics(char *prefix, + struct hash_lock_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the UDS advice proved correct */ + result = write_u64("dedupeAdviceValid : ", + stats->dedupe_advice_valid, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the UDS advice proved incorrect */ + result = write_u64("dedupeAdviceStale : ", + stats->dedupe_advice_stale, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of writes with the same data as another in-flight write */ + result = write_u64("concurrentDataMatches : ", + stats->concurrent_data_matches, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of writes whose hash collided with an in-flight write */ + result = write_u64("concurrentHashCollisions : ", + stats->concurrent_hash_collisions, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Current number of dedupe queries that are in flight */ + result = write_u32("currDedupeQueries : ", + stats->curr_dedupe_queries, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_error_statistics(char *prefix, + struct error_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of times VDO got an invalid dedupe advice PBN from UDS */ + result = write_u64("invalidAdvicePBNCount : ", + stats->invalid_advice_pbn_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of times a VIO completed with a VDO_NO_SPACE error */ + result = write_u64("noSpaceErrorCount : ", + stats->no_space_error_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of times a VIO completed with a VDO_READ_ONLY error */ + result = write_u64("readOnlyErrorCount : ", + stats->read_only_error_count, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_bio_stats(char *prefix, + struct bio_stats *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of REQ_OP_READ bios */ + result = write_u64("read : ", + stats->read, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of REQ_OP_WRITE bios with data */ + result = write_u64("write : ", + stats->write, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of bios tagged with REQ_PREFLUSH and containing no data */ + result = write_u64("emptyFlush : ", + stats->empty_flush, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of REQ_OP_DISCARD bios */ + result = write_u64("discard : ", + stats->discard, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of bios tagged with REQ_PREFLUSH */ + result = write_u64("flush : ", + stats->flush, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of bios tagged with REQ_FUA */ + result = write_u64("fua : ", + stats->fua, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_memory_usage(char *prefix, + struct memory_usage *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Tracked bytes currently allocated. */ + result = write_u64("bytesUsed : ", + stats->bytes_used, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Maximum tracked bytes allocated. */ + result = write_u64("peakBytesUsed : ", + stats->peak_bytes_used, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_index_statistics(char *prefix, + struct index_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of records stored in the index */ + result = write_u64("entriesIndexed : ", + stats->entries_indexed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of post calls that found an existing entry */ + result = write_u64("postsFound : ", + stats->posts_found, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of post calls that added a new entry */ + result = write_u64("postsNotFound : ", + stats->posts_not_found, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of query calls that found an existing entry */ + result = write_u64("queriesFound : ", + stats->queries_found, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of query calls that added a new entry */ + result = write_u64("queriesNotFound : ", + stats->queries_not_found, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of update calls that found an existing entry */ + result = write_u64("updatesFound : ", + stats->updates_found, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of update calls that added a new entry */ + result = write_u64("updatesNotFound : ", + stats->updates_not_found, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of entries discarded */ + result = write_u64("entriesDiscarded : ", + stats->entries_discarded, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +static int write_vdo_statistics(char *prefix, + struct vdo_statistics *stats, + char *suffix, + char **buf, + unsigned int *maxlen) +{ + int result; + + result = write_string(prefix, "{ ", NULL, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_u32("version : ", + stats->version, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of blocks used for data */ + result = write_u64("dataBlocksUsed : ", + stats->data_blocks_used, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of blocks used for VDO metadata */ + result = write_u64("overheadBlocksUsed : ", + stats->overhead_blocks_used, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of logical blocks that are currently mapped to physical blocks */ + result = write_u64("logicalBlocksUsed : ", + stats->logical_blocks_used, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of physical blocks */ + result = write_block_count_t("physicalBlocks : ", + stats->physical_blocks, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* number of logical blocks */ + result = write_block_count_t("logicalBlocks : ", + stats->logical_blocks, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Size of the block map page cache, in bytes */ + result = write_u64("blockMapCacheSize : ", + stats->block_map_cache_size, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The physical block size */ + result = write_u64("blockSize : ", + stats->block_size, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the VDO has successfully recovered */ + result = write_u64("completeRecoveries : ", + stats->complete_recoveries, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the VDO has recovered from read-only mode */ + result = write_u64("readOnlyRecoveries : ", + stats->read_only_recoveries, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* String describing the operating mode of the VDO */ + result = write_string("mode : ", + stats->mode, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Whether the VDO is in recovery mode */ + result = write_bool("inRecoveryMode : ", + stats->in_recovery_mode, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* What percentage of recovery mode work has been completed */ + result = write_u8("recoveryPercentage : ", + stats->recovery_percentage, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The statistics for the compressed block packer */ + result = write_packer_statistics("packer : ", + &stats->packer, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Counters for events in the block allocator */ + result = write_block_allocator_statistics("allocator : ", + &stats->allocator, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Counters for events in the recovery journal */ + result = write_recovery_journal_statistics("journal : ", + &stats->journal, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The statistics for the slab journals */ + result = write_slab_journal_statistics("slabJournal : ", + &stats->slab_journal, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The statistics for the slab summary */ + result = write_slab_summary_statistics("slabSummary : ", + &stats->slab_summary, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The statistics for the reference counts */ + result = write_ref_counts_statistics("refCounts : ", + &stats->ref_counts, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The statistics for the block map */ + result = write_block_map_statistics("blockMap : ", + &stats->block_map, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The dedupe statistics from hash locks */ + result = write_hash_lock_statistics("hashLock : ", + &stats->hash_lock, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Counts of error conditions */ + result = write_error_statistics("errors : ", + &stats->errors, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The VDO instance */ + result = write_u32("instance : ", + stats->instance, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Current number of active VIOs */ + result = write_u32("currentVIOsInProgress : ", + stats->current_vios_in_progress, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Maximum number of active VIOs */ + result = write_u32("maxVIOs : ", + stats->max_vios, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of times the UDS index was too slow in responding */ + result = write_u64("dedupeAdviceTimeouts : ", + stats->dedupe_advice_timeouts, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Number of flush requests submitted to the storage device */ + result = write_u64("flushOut : ", + stats->flush_out, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Logical block size */ + result = write_u64("logicalBlockSize : ", + stats->logical_block_size, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Bios submitted into VDO from above */ + result = write_bio_stats("biosIn : ", + &stats->bios_in, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosInPartial : ", + &stats->bios_in_partial, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Bios submitted onward for user data */ + result = write_bio_stats("biosOut : ", + &stats->bios_out, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Bios submitted onward for metadata */ + result = write_bio_stats("biosMeta : ", + &stats->bios_meta, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosJournal : ", + &stats->bios_journal, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosPageCache : ", + &stats->bios_page_cache, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosOutCompleted : ", + &stats->bios_out_completed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosMetaCompleted : ", + &stats->bios_meta_completed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosJournalCompleted : ", + &stats->bios_journal_completed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosPageCacheCompleted : ", + &stats->bios_page_cache_completed, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosAcknowledged : ", + &stats->bios_acknowledged, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosAcknowledgedPartial : ", + &stats->bios_acknowledged_partial, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Current number of bios in progress */ + result = write_bio_stats("biosInProgress : ", + &stats->bios_in_progress, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* Memory usage stats. */ + result = write_memory_usage("memoryUsage : ", + &stats->memory_usage, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + /* The statistics for the UDS index */ + result = write_index_statistics("index : ", + &stats->index, + ", ", + buf, + maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_string(NULL, "}", suffix, buf, maxlen); + if (result != VDO_SUCCESS) + return result; + return VDO_SUCCESS; +} + +int vdo_write_stats(struct vdo *vdo, + char *buf, + unsigned int maxlen) +{ + struct vdo_statistics *stats; + int result; + + result = uds_allocate(1, struct vdo_statistics, __func__, &stats); + if (result != VDO_SUCCESS) + return result; + + vdo_fetch_statistics(vdo, stats); + result = write_vdo_statistics(NULL, stats, NULL, &buf, &maxlen); + uds_free(stats); + return result; +} diff --git a/drivers/md/dm-vdo/message-stats.h b/drivers/md/dm-vdo/message-stats.h new file mode 100644 index 000000000000..f7fceca9acab --- /dev/null +++ b/drivers/md/dm-vdo/message-stats.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_MESSAGE_STATS_H +#define VDO_MESSAGE_STATS_H + +#include "types.h" + +int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen); + +#endif /* VDO_MESSAGE_STATS_H */ diff --git a/drivers/md/dm-vdo/statistics.h b/drivers/md/dm-vdo/statistics.h new file mode 100644 index 000000000000..da8b6a7fe24f --- /dev/null +++ b/drivers/md/dm-vdo/statistics.h @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef STATISTICS_H +#define STATISTICS_H + +#include "types.h" + +enum { + STATISTICS_VERSION = 36, +}; + +struct block_allocator_statistics { + /** The total number of slabs from which blocks may be allocated */ + u64 slab_count; + /** The total number of slabs from which blocks have ever been allocated */ + u64 slabs_opened; + /** The number of times since loading that a slab has been re-opened */ + u64 slabs_reopened; +}; + +/** + * Counters for tracking the number of items written (blocks, requests, etc.) + * that keep track of totals at steps in the write pipeline. Three counters + * allow the number of buffered, in-memory items and the number of in-flight, + * unacknowledged writes to be derived, while still tracking totals for + * reporting purposes + */ +struct commit_statistics { + /** The total number of items on which processing has started */ + u64 started; + /** The total number of items for which a write operation has been issued */ + u64 written; + /** The total number of items for which a write operation has completed */ + u64 committed; +}; + +/** Counters for events in the recovery journal */ +struct recovery_journal_statistics { + /** Number of times the on-disk journal was full */ + u64 disk_full; + /** Number of times the recovery journal requested slab journal commits. */ + u64 slab_journal_commits_requested; + /** Write/Commit totals for individual journal entries */ + struct commit_statistics entries; + /** Write/Commit totals for journal blocks */ + struct commit_statistics blocks; +}; + +/** The statistics for the compressed block packer. */ +struct packer_statistics { + /** Number of compressed data items written since startup */ + u64 compressed_fragments_written; + /** Number of blocks containing compressed items written since startup */ + u64 compressed_blocks_written; + /** Number of VIOs that are pending in the packer */ + u64 compressed_fragments_in_packer; +}; + +/** The statistics for the slab journals. */ +struct slab_journal_statistics { + /** Number of times the on-disk journal was full */ + u64 disk_full_count; + /** Number of times an entry was added over the flush threshold */ + u64 flush_count; + /** Number of times an entry was added over the block threshold */ + u64 blocked_count; + /** Number of times a tail block was written */ + u64 blocks_written; + /** Number of times we had to wait for the tail to write */ + u64 tail_busy_count; +}; + +/** The statistics for the slab summary. */ +struct slab_summary_statistics { + /** Number of blocks written */ + u64 blocks_written; +}; + +/** The statistics for the reference counts. */ +struct ref_counts_statistics { + /** Number of reference blocks written */ + u64 blocks_written; +}; + +/** The statistics for the block map. */ +struct block_map_statistics { + /** number of dirty (resident) pages */ + u32 dirty_pages; + /** number of clean (resident) pages */ + u32 clean_pages; + /** number of free pages */ + u32 free_pages; + /** number of pages in failed state */ + u32 failed_pages; + /** number of pages incoming */ + u32 incoming_pages; + /** number of pages outgoing */ + u32 outgoing_pages; + /** how many times free page not avail */ + u32 cache_pressure; + /** number of get_vdo_page() calls for read */ + u64 read_count; + /** number of get_vdo_page() calls for write */ + u64 write_count; + /** number of times pages failed to read */ + u64 failed_reads; + /** number of times pages failed to write */ + u64 failed_writes; + /** number of gets that are reclaimed */ + u64 reclaimed; + /** number of gets for outgoing pages */ + u64 read_outgoing; + /** number of gets that were already there */ + u64 found_in_cache; + /** number of gets requiring discard */ + u64 discard_required; + /** number of gets enqueued for their page */ + u64 wait_for_page; + /** number of gets that have to fetch */ + u64 fetch_required; + /** number of page fetches */ + u64 pages_loaded; + /** number of page saves */ + u64 pages_saved; + /** the number of flushes issued */ + u64 flush_count; +}; + +/** The dedupe statistics from hash locks */ +struct hash_lock_statistics { + /** Number of times the UDS advice proved correct */ + u64 dedupe_advice_valid; + /** Number of times the UDS advice proved incorrect */ + u64 dedupe_advice_stale; + /** Number of writes with the same data as another in-flight write */ + u64 concurrent_data_matches; + /** Number of writes whose hash collided with an in-flight write */ + u64 concurrent_hash_collisions; + /** Current number of dedupe queries that are in flight */ + u32 curr_dedupe_queries; +}; + +/** Counts of error conditions in VDO. */ +struct error_statistics { + /** number of times VDO got an invalid dedupe advice PBN from UDS */ + u64 invalid_advice_pbn_count; + /** number of times a VIO completed with a VDO_NO_SPACE error */ + u64 no_space_error_count; + /** number of times a VIO completed with a VDO_READ_ONLY error */ + u64 read_only_error_count; +}; + +struct bio_stats { + /** Number of REQ_OP_READ bios */ + u64 read; + /** Number of REQ_OP_WRITE bios with data */ + u64 write; + /** Number of bios tagged with REQ_PREFLUSH and containing no data */ + u64 empty_flush; + /** Number of REQ_OP_DISCARD bios */ + u64 discard; + /** Number of bios tagged with REQ_PREFLUSH */ + u64 flush; + /** Number of bios tagged with REQ_FUA */ + u64 fua; +}; + +struct memory_usage { + /** Tracked bytes currently allocated. */ + u64 bytes_used; + /** Maximum tracked bytes allocated. */ + u64 peak_bytes_used; +}; + +/** UDS index statistics */ +struct index_statistics { + /** Number of records stored in the index */ + u64 entries_indexed; + /** Number of post calls that found an existing entry */ + u64 posts_found; + /** Number of post calls that added a new entry */ + u64 posts_not_found; + /** Number of query calls that found an existing entry */ + u64 queries_found; + /** Number of query calls that added a new entry */ + u64 queries_not_found; + /** Number of update calls that found an existing entry */ + u64 updates_found; + /** Number of update calls that added a new entry */ + u64 updates_not_found; + /** Number of entries discarded */ + u64 entries_discarded; +}; + +/** The statistics of the vdo service. */ +struct vdo_statistics { + u32 version; + /** Number of blocks used for data */ + u64 data_blocks_used; + /** Number of blocks used for VDO metadata */ + u64 overhead_blocks_used; + /** Number of logical blocks that are currently mapped to physical blocks */ + u64 logical_blocks_used; + /** number of physical blocks */ + block_count_t physical_blocks; + /** number of logical blocks */ + block_count_t logical_blocks; + /** Size of the block map page cache, in bytes */ + u64 block_map_cache_size; + /** The physical block size */ + u64 block_size; + /** Number of times the VDO has successfully recovered */ + u64 complete_recoveries; + /** Number of times the VDO has recovered from read-only mode */ + u64 read_only_recoveries; + /** String describing the operating mode of the VDO */ + char mode[15]; + /** Whether the VDO is in recovery mode */ + bool in_recovery_mode; + /** What percentage of recovery mode work has been completed */ + u8 recovery_percentage; + /** The statistics for the compressed block packer */ + struct packer_statistics packer; + /** Counters for events in the block allocator */ + struct block_allocator_statistics allocator; + /** Counters for events in the recovery journal */ + struct recovery_journal_statistics journal; + /** The statistics for the slab journals */ + struct slab_journal_statistics slab_journal; + /** The statistics for the slab summary */ + struct slab_summary_statistics slab_summary; + /** The statistics for the reference counts */ + struct ref_counts_statistics ref_counts; + /** The statistics for the block map */ + struct block_map_statistics block_map; + /** The dedupe statistics from hash locks */ + struct hash_lock_statistics hash_lock; + /** Counts of error conditions */ + struct error_statistics errors; + /** The VDO instance */ + u32 instance; + /** Current number of active VIOs */ + u32 current_vios_in_progress; + /** Maximum number of active VIOs */ + u32 max_vios; + /** Number of times the UDS index was too slow in responding */ + u64 dedupe_advice_timeouts; + /** Number of flush requests submitted to the storage device */ + u64 flush_out; + /** Logical block size */ + u64 logical_block_size; + /** Bios submitted into VDO from above */ + struct bio_stats bios_in; + struct bio_stats bios_in_partial; + /** Bios submitted onward for user data */ + struct bio_stats bios_out; + /** Bios submitted onward for metadata */ + struct bio_stats bios_meta; + struct bio_stats bios_journal; + struct bio_stats bios_page_cache; + struct bio_stats bios_out_completed; + struct bio_stats bios_meta_completed; + struct bio_stats bios_journal_completed; + struct bio_stats bios_page_cache_completed; + struct bio_stats bios_acknowledged; + struct bio_stats bios_acknowledged_partial; + /** Current number of bios in progress */ + struct bio_stats bios_in_progress; + /** Memory usage stats. */ + struct memory_usage memory_usage; + /** The statistics for the UDS index */ + struct index_statistics index; +}; + +#endif /* not STATISTICS_H */ From 92f8d7a94fe743831728943aa8988ddf1dddac22 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:11:23 -0500 Subject: [PATCH 0584/1038] dm vdo: add sysfs support for setting parameters and fetching stats Add data and methods setting run time parameters via sysfs, and to make state and statistics information available through sysfs. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/pool-sysfs-stats.c | 2063 ++++++++++++++++++++++++++ drivers/md/dm-vdo/pool-sysfs.c | 198 +++ drivers/md/dm-vdo/pool-sysfs.h | 19 + drivers/md/dm-vdo/sysfs.c | 82 + 4 files changed, 2362 insertions(+) create mode 100644 drivers/md/dm-vdo/pool-sysfs-stats.c create mode 100644 drivers/md/dm-vdo/pool-sysfs.c create mode 100644 drivers/md/dm-vdo/pool-sysfs.h create mode 100644 drivers/md/dm-vdo/sysfs.c diff --git a/drivers/md/dm-vdo/pool-sysfs-stats.c b/drivers/md/dm-vdo/pool-sysfs-stats.c new file mode 100644 index 000000000000..ae3838894a1c --- /dev/null +++ b/drivers/md/dm-vdo/pool-sysfs-stats.c @@ -0,0 +1,2063 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include + +#include "logger.h" +#include "string-utils.h" + +#include "dedupe.h" +#include "pool-sysfs.h" +#include "statistics.h" +#include "vdo.h" + +struct pool_stats_attribute { + struct attribute attr; + ssize_t (*print)(struct vdo_statistics *stats, char *buf); +}; + +static ssize_t pool_stats_attr_show(struct kobject *directory, + struct attribute *attr, + char *buf) +{ + ssize_t size; + struct pool_stats_attribute *pool_stats_attr = + container_of(attr, struct pool_stats_attribute, attr); + struct vdo *vdo = container_of(directory, struct vdo, stats_directory); + + if (pool_stats_attr->print == NULL) + return -EINVAL; + + mutex_lock(&vdo->stats_mutex); + vdo_fetch_statistics(vdo, &vdo->stats_buffer); + size = pool_stats_attr->print(&vdo->stats_buffer, buf); + mutex_unlock(&vdo->stats_mutex); + + return size; +} + +const struct sysfs_ops vdo_pool_stats_sysfs_ops = { + .show = pool_stats_attr_show, + .store = NULL, +}; + +/* Number of blocks used for data */ +static ssize_t +pool_stats_print_data_blocks_used(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->data_blocks_used); +} + +static struct pool_stats_attribute pool_stats_attr_data_blocks_used = { + .attr = { .name = "data_blocks_used", .mode = 0444, }, + .print = pool_stats_print_data_blocks_used, +}; + +/* Number of blocks used for VDO metadata */ +static ssize_t +pool_stats_print_overhead_blocks_used(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->overhead_blocks_used); +} + +static struct pool_stats_attribute pool_stats_attr_overhead_blocks_used = { + .attr = { .name = "overhead_blocks_used", .mode = 0444, }, + .print = pool_stats_print_overhead_blocks_used, +}; + +/* Number of logical blocks that are currently mapped to physical blocks */ +static ssize_t +pool_stats_print_logical_blocks_used(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->logical_blocks_used); +} + +static struct pool_stats_attribute pool_stats_attr_logical_blocks_used = { + .attr = { .name = "logical_blocks_used", .mode = 0444, }, + .print = pool_stats_print_logical_blocks_used, +}; + +/* number of physical blocks */ +static ssize_t +pool_stats_print_physical_blocks(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->physical_blocks); +} + +static struct pool_stats_attribute pool_stats_attr_physical_blocks = { + .attr = { .name = "physical_blocks", .mode = 0444, }, + .print = pool_stats_print_physical_blocks, +}; + +/* number of logical blocks */ +static ssize_t +pool_stats_print_logical_blocks(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->logical_blocks); +} + +static struct pool_stats_attribute pool_stats_attr_logical_blocks = { + .attr = { .name = "logical_blocks", .mode = 0444, }, + .print = pool_stats_print_logical_blocks, +}; + +/* Size of the block map page cache, in bytes */ +static ssize_t +pool_stats_print_block_map_cache_size(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map_cache_size); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_cache_size = { + .attr = { .name = "block_map_cache_size", .mode = 0444, }, + .print = pool_stats_print_block_map_cache_size, +}; + +/* The physical block size */ +static ssize_t +pool_stats_print_block_size(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_size); +} + +static struct pool_stats_attribute pool_stats_attr_block_size = { + .attr = { .name = "block_size", .mode = 0444, }, + .print = pool_stats_print_block_size, +}; + +/* Number of times the VDO has successfully recovered */ +static ssize_t +pool_stats_print_complete_recoveries(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->complete_recoveries); +} + +static struct pool_stats_attribute pool_stats_attr_complete_recoveries = { + .attr = { .name = "complete_recoveries", .mode = 0444, }, + .print = pool_stats_print_complete_recoveries, +}; + +/* Number of times the VDO has recovered from read-only mode */ +static ssize_t +pool_stats_print_read_only_recoveries(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->read_only_recoveries); +} + +static struct pool_stats_attribute pool_stats_attr_read_only_recoveries = { + .attr = { .name = "read_only_recoveries", .mode = 0444, }, + .print = pool_stats_print_read_only_recoveries, +}; + +/* String describing the operating mode of the VDO */ +static ssize_t +pool_stats_print_mode(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%s\n", stats->mode); +} + +static struct pool_stats_attribute pool_stats_attr_mode = { + .attr = { .name = "mode", .mode = 0444, }, + .print = pool_stats_print_mode, +}; + +/* Whether the VDO is in recovery mode */ +static ssize_t +pool_stats_print_in_recovery_mode(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%d\n", stats->in_recovery_mode); +} + +static struct pool_stats_attribute pool_stats_attr_in_recovery_mode = { + .attr = { .name = "in_recovery_mode", .mode = 0444, }, + .print = pool_stats_print_in_recovery_mode, +}; + +/* What percentage of recovery mode work has been completed */ +static ssize_t +pool_stats_print_recovery_percentage(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->recovery_percentage); +} + +static struct pool_stats_attribute pool_stats_attr_recovery_percentage = { + .attr = { .name = "recovery_percentage", .mode = 0444, }, + .print = pool_stats_print_recovery_percentage, +}; + +/* Number of compressed data items written since startup */ +static ssize_t +pool_stats_print_packer_compressed_fragments_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->packer.compressed_fragments_written); +} + +static struct pool_stats_attribute pool_stats_attr_packer_compressed_fragments_written = { + .attr = { .name = "packer_compressed_fragments_written", .mode = 0444, }, + .print = pool_stats_print_packer_compressed_fragments_written, +}; + +/* Number of blocks containing compressed items written since startup */ +static ssize_t +pool_stats_print_packer_compressed_blocks_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->packer.compressed_blocks_written); +} + +static struct pool_stats_attribute pool_stats_attr_packer_compressed_blocks_written = { + .attr = { .name = "packer_compressed_blocks_written", .mode = 0444, }, + .print = pool_stats_print_packer_compressed_blocks_written, +}; + +/* Number of VIOs that are pending in the packer */ +static ssize_t +pool_stats_print_packer_compressed_fragments_in_packer(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->packer.compressed_fragments_in_packer); +} + +static struct pool_stats_attribute pool_stats_attr_packer_compressed_fragments_in_packer = { + .attr = { .name = "packer_compressed_fragments_in_packer", .mode = 0444, }, + .print = pool_stats_print_packer_compressed_fragments_in_packer, +}; + +/* The total number of slabs from which blocks may be allocated */ +static ssize_t +pool_stats_print_allocator_slab_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->allocator.slab_count); +} + +static struct pool_stats_attribute pool_stats_attr_allocator_slab_count = { + .attr = { .name = "allocator_slab_count", .mode = 0444, }, + .print = pool_stats_print_allocator_slab_count, +}; + +/* The total number of slabs from which blocks have ever been allocated */ +static ssize_t +pool_stats_print_allocator_slabs_opened(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->allocator.slabs_opened); +} + +static struct pool_stats_attribute pool_stats_attr_allocator_slabs_opened = { + .attr = { .name = "allocator_slabs_opened", .mode = 0444, }, + .print = pool_stats_print_allocator_slabs_opened, +}; + +/* The number of times since loading that a slab has been re-opened */ +static ssize_t +pool_stats_print_allocator_slabs_reopened(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->allocator.slabs_reopened); +} + +static struct pool_stats_attribute pool_stats_attr_allocator_slabs_reopened = { + .attr = { .name = "allocator_slabs_reopened", .mode = 0444, }, + .print = pool_stats_print_allocator_slabs_reopened, +}; + +/* Number of times the on-disk journal was full */ +static ssize_t +pool_stats_print_journal_disk_full(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.disk_full); +} + +static struct pool_stats_attribute pool_stats_attr_journal_disk_full = { + .attr = { .name = "journal_disk_full", .mode = 0444, }, + .print = pool_stats_print_journal_disk_full, +}; + +/* Number of times the recovery journal requested slab journal commits. */ +static ssize_t +pool_stats_print_journal_slab_journal_commits_requested(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.slab_journal_commits_requested); +} + +static struct pool_stats_attribute pool_stats_attr_journal_slab_journal_commits_requested = { + .attr = { .name = "journal_slab_journal_commits_requested", .mode = 0444, }, + .print = pool_stats_print_journal_slab_journal_commits_requested, +}; + +/* The total number of items on which processing has started */ +static ssize_t +pool_stats_print_journal_entries_started(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.entries.started); +} + +static struct pool_stats_attribute pool_stats_attr_journal_entries_started = { + .attr = { .name = "journal_entries_started", .mode = 0444, }, + .print = pool_stats_print_journal_entries_started, +}; + +/* The total number of items for which a write operation has been issued */ +static ssize_t +pool_stats_print_journal_entries_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.entries.written); +} + +static struct pool_stats_attribute pool_stats_attr_journal_entries_written = { + .attr = { .name = "journal_entries_written", .mode = 0444, }, + .print = pool_stats_print_journal_entries_written, +}; + +/* The total number of items for which a write operation has completed */ +static ssize_t +pool_stats_print_journal_entries_committed(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.entries.committed); +} + +static struct pool_stats_attribute pool_stats_attr_journal_entries_committed = { + .attr = { .name = "journal_entries_committed", .mode = 0444, }, + .print = pool_stats_print_journal_entries_committed, +}; + +/* The total number of items on which processing has started */ +static ssize_t +pool_stats_print_journal_blocks_started(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.blocks.started); +} + +static struct pool_stats_attribute pool_stats_attr_journal_blocks_started = { + .attr = { .name = "journal_blocks_started", .mode = 0444, }, + .print = pool_stats_print_journal_blocks_started, +}; + +/* The total number of items for which a write operation has been issued */ +static ssize_t +pool_stats_print_journal_blocks_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.blocks.written); +} + +static struct pool_stats_attribute pool_stats_attr_journal_blocks_written = { + .attr = { .name = "journal_blocks_written", .mode = 0444, }, + .print = pool_stats_print_journal_blocks_written, +}; + +/* The total number of items for which a write operation has completed */ +static ssize_t +pool_stats_print_journal_blocks_committed(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->journal.blocks.committed); +} + +static struct pool_stats_attribute pool_stats_attr_journal_blocks_committed = { + .attr = { .name = "journal_blocks_committed", .mode = 0444, }, + .print = pool_stats_print_journal_blocks_committed, +}; + +/* Number of times the on-disk journal was full */ +static ssize_t +pool_stats_print_slab_journal_disk_full_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->slab_journal.disk_full_count); +} + +static struct pool_stats_attribute pool_stats_attr_slab_journal_disk_full_count = { + .attr = { .name = "slab_journal_disk_full_count", .mode = 0444, }, + .print = pool_stats_print_slab_journal_disk_full_count, +}; + +/* Number of times an entry was added over the flush threshold */ +static ssize_t +pool_stats_print_slab_journal_flush_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->slab_journal.flush_count); +} + +static struct pool_stats_attribute pool_stats_attr_slab_journal_flush_count = { + .attr = { .name = "slab_journal_flush_count", .mode = 0444, }, + .print = pool_stats_print_slab_journal_flush_count, +}; + +/* Number of times an entry was added over the block threshold */ +static ssize_t +pool_stats_print_slab_journal_blocked_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->slab_journal.blocked_count); +} + +static struct pool_stats_attribute pool_stats_attr_slab_journal_blocked_count = { + .attr = { .name = "slab_journal_blocked_count", .mode = 0444, }, + .print = pool_stats_print_slab_journal_blocked_count, +}; + +/* Number of times a tail block was written */ +static ssize_t +pool_stats_print_slab_journal_blocks_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->slab_journal.blocks_written); +} + +static struct pool_stats_attribute pool_stats_attr_slab_journal_blocks_written = { + .attr = { .name = "slab_journal_blocks_written", .mode = 0444, }, + .print = pool_stats_print_slab_journal_blocks_written, +}; + +/* Number of times we had to wait for the tail to write */ +static ssize_t +pool_stats_print_slab_journal_tail_busy_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->slab_journal.tail_busy_count); +} + +static struct pool_stats_attribute pool_stats_attr_slab_journal_tail_busy_count = { + .attr = { .name = "slab_journal_tail_busy_count", .mode = 0444, }, + .print = pool_stats_print_slab_journal_tail_busy_count, +}; + +/* Number of blocks written */ +static ssize_t +pool_stats_print_slab_summary_blocks_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->slab_summary.blocks_written); +} + +static struct pool_stats_attribute pool_stats_attr_slab_summary_blocks_written = { + .attr = { .name = "slab_summary_blocks_written", .mode = 0444, }, + .print = pool_stats_print_slab_summary_blocks_written, +}; + +/* Number of reference blocks written */ +static ssize_t +pool_stats_print_ref_counts_blocks_written(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->ref_counts.blocks_written); +} + +static struct pool_stats_attribute pool_stats_attr_ref_counts_blocks_written = { + .attr = { .name = "ref_counts_blocks_written", .mode = 0444, }, + .print = pool_stats_print_ref_counts_blocks_written, +}; + +/* number of dirty (resident) pages */ +static ssize_t +pool_stats_print_block_map_dirty_pages(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.dirty_pages); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_dirty_pages = { + .attr = { .name = "block_map_dirty_pages", .mode = 0444, }, + .print = pool_stats_print_block_map_dirty_pages, +}; + +/* number of clean (resident) pages */ +static ssize_t +pool_stats_print_block_map_clean_pages(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.clean_pages); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_clean_pages = { + .attr = { .name = "block_map_clean_pages", .mode = 0444, }, + .print = pool_stats_print_block_map_clean_pages, +}; + +/* number of free pages */ +static ssize_t +pool_stats_print_block_map_free_pages(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.free_pages); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_free_pages = { + .attr = { .name = "block_map_free_pages", .mode = 0444, }, + .print = pool_stats_print_block_map_free_pages, +}; + +/* number of pages in failed state */ +static ssize_t +pool_stats_print_block_map_failed_pages(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.failed_pages); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_failed_pages = { + .attr = { .name = "block_map_failed_pages", .mode = 0444, }, + .print = pool_stats_print_block_map_failed_pages, +}; + +/* number of pages incoming */ +static ssize_t +pool_stats_print_block_map_incoming_pages(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.incoming_pages); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_incoming_pages = { + .attr = { .name = "block_map_incoming_pages", .mode = 0444, }, + .print = pool_stats_print_block_map_incoming_pages, +}; + +/* number of pages outgoing */ +static ssize_t +pool_stats_print_block_map_outgoing_pages(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.outgoing_pages); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_outgoing_pages = { + .attr = { .name = "block_map_outgoing_pages", .mode = 0444, }, + .print = pool_stats_print_block_map_outgoing_pages, +}; + +/* how many times free page not avail */ +static ssize_t +pool_stats_print_block_map_cache_pressure(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->block_map.cache_pressure); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_cache_pressure = { + .attr = { .name = "block_map_cache_pressure", .mode = 0444, }, + .print = pool_stats_print_block_map_cache_pressure, +}; + +/* number of get_vdo_page() calls for read */ +static ssize_t +pool_stats_print_block_map_read_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.read_count); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_read_count = { + .attr = { .name = "block_map_read_count", .mode = 0444, }, + .print = pool_stats_print_block_map_read_count, +}; + +/* number of get_vdo_page() calls for write */ +static ssize_t +pool_stats_print_block_map_write_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.write_count); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_write_count = { + .attr = { .name = "block_map_write_count", .mode = 0444, }, + .print = pool_stats_print_block_map_write_count, +}; + +/* number of times pages failed to read */ +static ssize_t +pool_stats_print_block_map_failed_reads(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.failed_reads); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_failed_reads = { + .attr = { .name = "block_map_failed_reads", .mode = 0444, }, + .print = pool_stats_print_block_map_failed_reads, +}; + +/* number of times pages failed to write */ +static ssize_t +pool_stats_print_block_map_failed_writes(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.failed_writes); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_failed_writes = { + .attr = { .name = "block_map_failed_writes", .mode = 0444, }, + .print = pool_stats_print_block_map_failed_writes, +}; + +/* number of gets that are reclaimed */ +static ssize_t +pool_stats_print_block_map_reclaimed(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.reclaimed); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_reclaimed = { + .attr = { .name = "block_map_reclaimed", .mode = 0444, }, + .print = pool_stats_print_block_map_reclaimed, +}; + +/* number of gets for outgoing pages */ +static ssize_t +pool_stats_print_block_map_read_outgoing(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.read_outgoing); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_read_outgoing = { + .attr = { .name = "block_map_read_outgoing", .mode = 0444, }, + .print = pool_stats_print_block_map_read_outgoing, +}; + +/* number of gets that were already there */ +static ssize_t +pool_stats_print_block_map_found_in_cache(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.found_in_cache); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_found_in_cache = { + .attr = { .name = "block_map_found_in_cache", .mode = 0444, }, + .print = pool_stats_print_block_map_found_in_cache, +}; + +/* number of gets requiring discard */ +static ssize_t +pool_stats_print_block_map_discard_required(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.discard_required); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_discard_required = { + .attr = { .name = "block_map_discard_required", .mode = 0444, }, + .print = pool_stats_print_block_map_discard_required, +}; + +/* number of gets enqueued for their page */ +static ssize_t +pool_stats_print_block_map_wait_for_page(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.wait_for_page); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_wait_for_page = { + .attr = { .name = "block_map_wait_for_page", .mode = 0444, }, + .print = pool_stats_print_block_map_wait_for_page, +}; + +/* number of gets that have to fetch */ +static ssize_t +pool_stats_print_block_map_fetch_required(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.fetch_required); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_fetch_required = { + .attr = { .name = "block_map_fetch_required", .mode = 0444, }, + .print = pool_stats_print_block_map_fetch_required, +}; + +/* number of page fetches */ +static ssize_t +pool_stats_print_block_map_pages_loaded(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.pages_loaded); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_pages_loaded = { + .attr = { .name = "block_map_pages_loaded", .mode = 0444, }, + .print = pool_stats_print_block_map_pages_loaded, +}; + +/* number of page saves */ +static ssize_t +pool_stats_print_block_map_pages_saved(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.pages_saved); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_pages_saved = { + .attr = { .name = "block_map_pages_saved", .mode = 0444, }, + .print = pool_stats_print_block_map_pages_saved, +}; + +/* the number of flushes issued */ +static ssize_t +pool_stats_print_block_map_flush_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->block_map.flush_count); +} + +static struct pool_stats_attribute pool_stats_attr_block_map_flush_count = { + .attr = { .name = "block_map_flush_count", .mode = 0444, }, + .print = pool_stats_print_block_map_flush_count, +}; + +/* Number of times the UDS advice proved correct */ +static ssize_t +pool_stats_print_hash_lock_dedupe_advice_valid(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->hash_lock.dedupe_advice_valid); +} + +static struct pool_stats_attribute pool_stats_attr_hash_lock_dedupe_advice_valid = { + .attr = { .name = "hash_lock_dedupe_advice_valid", .mode = 0444, }, + .print = pool_stats_print_hash_lock_dedupe_advice_valid, +}; + +/* Number of times the UDS advice proved incorrect */ +static ssize_t +pool_stats_print_hash_lock_dedupe_advice_stale(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->hash_lock.dedupe_advice_stale); +} + +static struct pool_stats_attribute pool_stats_attr_hash_lock_dedupe_advice_stale = { + .attr = { .name = "hash_lock_dedupe_advice_stale", .mode = 0444, }, + .print = pool_stats_print_hash_lock_dedupe_advice_stale, +}; + +/* Number of writes with the same data as another in-flight write */ +static ssize_t +pool_stats_print_hash_lock_concurrent_data_matches(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->hash_lock.concurrent_data_matches); +} + +static struct pool_stats_attribute pool_stats_attr_hash_lock_concurrent_data_matches = { + .attr = { .name = "hash_lock_concurrent_data_matches", .mode = 0444, }, + .print = pool_stats_print_hash_lock_concurrent_data_matches, +}; + +/* Number of writes whose hash collided with an in-flight write */ +static ssize_t +pool_stats_print_hash_lock_concurrent_hash_collisions(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->hash_lock.concurrent_hash_collisions); +} + +static struct pool_stats_attribute pool_stats_attr_hash_lock_concurrent_hash_collisions = { + .attr = { .name = "hash_lock_concurrent_hash_collisions", .mode = 0444, }, + .print = pool_stats_print_hash_lock_concurrent_hash_collisions, +}; + +/* Current number of dedupe queries that are in flight */ +static ssize_t +pool_stats_print_hash_lock_curr_dedupe_queries(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->hash_lock.curr_dedupe_queries); +} + +static struct pool_stats_attribute pool_stats_attr_hash_lock_curr_dedupe_queries = { + .attr = { .name = "hash_lock_curr_dedupe_queries", .mode = 0444, }, + .print = pool_stats_print_hash_lock_curr_dedupe_queries, +}; + +/* number of times VDO got an invalid dedupe advice PBN from UDS */ +static ssize_t +pool_stats_print_errors_invalid_advice_pbn_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->errors.invalid_advice_pbn_count); +} + +static struct pool_stats_attribute pool_stats_attr_errors_invalid_advice_pbn_count = { + .attr = { .name = "errors_invalid_advice_pbn_count", .mode = 0444, }, + .print = pool_stats_print_errors_invalid_advice_pbn_count, +}; + +/* number of times a VIO completed with a VDO_NO_SPACE error */ +static ssize_t +pool_stats_print_errors_no_space_error_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->errors.no_space_error_count); +} + +static struct pool_stats_attribute pool_stats_attr_errors_no_space_error_count = { + .attr = { .name = "errors_no_space_error_count", .mode = 0444, }, + .print = pool_stats_print_errors_no_space_error_count, +}; + +/* number of times a VIO completed with a VDO_READ_ONLY error */ +static ssize_t +pool_stats_print_errors_read_only_error_count(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->errors.read_only_error_count); +} + +static struct pool_stats_attribute pool_stats_attr_errors_read_only_error_count = { + .attr = { .name = "errors_read_only_error_count", .mode = 0444, }, + .print = pool_stats_print_errors_read_only_error_count, +}; + +/* The VDO instance */ +static ssize_t +pool_stats_print_instance(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->instance); +} + +static struct pool_stats_attribute pool_stats_attr_instance = { + .attr = { .name = "instance", .mode = 0444, }, + .print = pool_stats_print_instance, +}; + +/* Current number of active VIOs */ +static ssize_t +pool_stats_print_current_vios_in_progress(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->current_vios_in_progress); +} + +static struct pool_stats_attribute pool_stats_attr_current_vios_in_progress = { + .attr = { .name = "current_vios_in_progress", .mode = 0444, }, + .print = pool_stats_print_current_vios_in_progress, +}; + +/* Maximum number of active VIOs */ +static ssize_t +pool_stats_print_max_vios(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%u\n", stats->max_vios); +} + +static struct pool_stats_attribute pool_stats_attr_max_vios = { + .attr = { .name = "max_vios", .mode = 0444, }, + .print = pool_stats_print_max_vios, +}; + +/* Number of times the UDS index was too slow in responding */ +static ssize_t +pool_stats_print_dedupe_advice_timeouts(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->dedupe_advice_timeouts); +} + +static struct pool_stats_attribute pool_stats_attr_dedupe_advice_timeouts = { + .attr = { .name = "dedupe_advice_timeouts", .mode = 0444, }, + .print = pool_stats_print_dedupe_advice_timeouts, +}; + +/* Number of flush requests submitted to the storage device */ +static ssize_t +pool_stats_print_flush_out(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->flush_out); +} + +static struct pool_stats_attribute pool_stats_attr_flush_out = { + .attr = { .name = "flush_out", .mode = 0444, }, + .print = pool_stats_print_flush_out, +}; + +/* Logical block size */ +static ssize_t +pool_stats_print_logical_block_size(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->logical_block_size); +} + +static struct pool_stats_attribute pool_stats_attr_logical_block_size = { + .attr = { .name = "logical_block_size", .mode = 0444, }, + .print = pool_stats_print_logical_block_size, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_in_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_read = { + .attr = { .name = "bios_in_read", .mode = 0444, }, + .print = pool_stats_print_bios_in_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_in_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_write = { + .attr = { .name = "bios_in_write", .mode = 0444, }, + .print = pool_stats_print_bios_in_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_in_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_empty_flush = { + .attr = { .name = "bios_in_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_in_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_in_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_discard = { + .attr = { .name = "bios_in_discard", .mode = 0444, }, + .print = pool_stats_print_bios_in_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_in_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_flush = { + .attr = { .name = "bios_in_flush", .mode = 0444, }, + .print = pool_stats_print_bios_in_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_in_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_fua = { + .attr = { .name = "bios_in_fua", .mode = 0444, }, + .print = pool_stats_print_bios_in_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_in_partial_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_partial.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_partial_read = { + .attr = { .name = "bios_in_partial_read", .mode = 0444, }, + .print = pool_stats_print_bios_in_partial_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_in_partial_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_partial.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_partial_write = { + .attr = { .name = "bios_in_partial_write", .mode = 0444, }, + .print = pool_stats_print_bios_in_partial_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_in_partial_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_partial.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_partial_empty_flush = { + .attr = { .name = "bios_in_partial_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_in_partial_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_in_partial_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_partial.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_partial_discard = { + .attr = { .name = "bios_in_partial_discard", .mode = 0444, }, + .print = pool_stats_print_bios_in_partial_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_in_partial_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_partial.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_partial_flush = { + .attr = { .name = "bios_in_partial_flush", .mode = 0444, }, + .print = pool_stats_print_bios_in_partial_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_in_partial_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_partial.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_partial_fua = { + .attr = { .name = "bios_in_partial_fua", .mode = 0444, }, + .print = pool_stats_print_bios_in_partial_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_out_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_read = { + .attr = { .name = "bios_out_read", .mode = 0444, }, + .print = pool_stats_print_bios_out_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_out_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_write = { + .attr = { .name = "bios_out_write", .mode = 0444, }, + .print = pool_stats_print_bios_out_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_out_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_empty_flush = { + .attr = { .name = "bios_out_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_out_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_out_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_discard = { + .attr = { .name = "bios_out_discard", .mode = 0444, }, + .print = pool_stats_print_bios_out_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_out_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_flush = { + .attr = { .name = "bios_out_flush", .mode = 0444, }, + .print = pool_stats_print_bios_out_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_out_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_fua = { + .attr = { .name = "bios_out_fua", .mode = 0444, }, + .print = pool_stats_print_bios_out_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_meta_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_read = { + .attr = { .name = "bios_meta_read", .mode = 0444, }, + .print = pool_stats_print_bios_meta_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_meta_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_write = { + .attr = { .name = "bios_meta_write", .mode = 0444, }, + .print = pool_stats_print_bios_meta_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_meta_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_empty_flush = { + .attr = { .name = "bios_meta_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_meta_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_meta_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_discard = { + .attr = { .name = "bios_meta_discard", .mode = 0444, }, + .print = pool_stats_print_bios_meta_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_meta_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_flush = { + .attr = { .name = "bios_meta_flush", .mode = 0444, }, + .print = pool_stats_print_bios_meta_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_meta_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_fua = { + .attr = { .name = "bios_meta_fua", .mode = 0444, }, + .print = pool_stats_print_bios_meta_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_journal_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_read = { + .attr = { .name = "bios_journal_read", .mode = 0444, }, + .print = pool_stats_print_bios_journal_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_journal_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_write = { + .attr = { .name = "bios_journal_write", .mode = 0444, }, + .print = pool_stats_print_bios_journal_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_journal_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_empty_flush = { + .attr = { .name = "bios_journal_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_journal_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_journal_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_discard = { + .attr = { .name = "bios_journal_discard", .mode = 0444, }, + .print = pool_stats_print_bios_journal_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_journal_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_flush = { + .attr = { .name = "bios_journal_flush", .mode = 0444, }, + .print = pool_stats_print_bios_journal_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_journal_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_fua = { + .attr = { .name = "bios_journal_fua", .mode = 0444, }, + .print = pool_stats_print_bios_journal_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_page_cache_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_read = { + .attr = { .name = "bios_page_cache_read", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_page_cache_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_write = { + .attr = { .name = "bios_page_cache_write", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_page_cache_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_empty_flush = { + .attr = { .name = "bios_page_cache_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_page_cache_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_discard = { + .attr = { .name = "bios_page_cache_discard", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_page_cache_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_flush = { + .attr = { .name = "bios_page_cache_flush", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_page_cache_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_fua = { + .attr = { .name = "bios_page_cache_fua", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_out_completed_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out_completed.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_completed_read = { + .attr = { .name = "bios_out_completed_read", .mode = 0444, }, + .print = pool_stats_print_bios_out_completed_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_out_completed_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out_completed.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_completed_write = { + .attr = { .name = "bios_out_completed_write", .mode = 0444, }, + .print = pool_stats_print_bios_out_completed_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_out_completed_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out_completed.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_completed_empty_flush = { + .attr = { .name = "bios_out_completed_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_out_completed_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_out_completed_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out_completed.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_completed_discard = { + .attr = { .name = "bios_out_completed_discard", .mode = 0444, }, + .print = pool_stats_print_bios_out_completed_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_out_completed_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out_completed.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_completed_flush = { + .attr = { .name = "bios_out_completed_flush", .mode = 0444, }, + .print = pool_stats_print_bios_out_completed_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_out_completed_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_out_completed.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_out_completed_fua = { + .attr = { .name = "bios_out_completed_fua", .mode = 0444, }, + .print = pool_stats_print_bios_out_completed_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_meta_completed_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta_completed.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_read = { + .attr = { .name = "bios_meta_completed_read", .mode = 0444, }, + .print = pool_stats_print_bios_meta_completed_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_meta_completed_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta_completed.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_write = { + .attr = { .name = "bios_meta_completed_write", .mode = 0444, }, + .print = pool_stats_print_bios_meta_completed_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_meta_completed_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta_completed.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_empty_flush = { + .attr = { .name = "bios_meta_completed_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_meta_completed_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_meta_completed_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta_completed.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_discard = { + .attr = { .name = "bios_meta_completed_discard", .mode = 0444, }, + .print = pool_stats_print_bios_meta_completed_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_meta_completed_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta_completed.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_flush = { + .attr = { .name = "bios_meta_completed_flush", .mode = 0444, }, + .print = pool_stats_print_bios_meta_completed_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_meta_completed_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_meta_completed.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_fua = { + .attr = { .name = "bios_meta_completed_fua", .mode = 0444, }, + .print = pool_stats_print_bios_meta_completed_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_journal_completed_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal_completed.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_read = { + .attr = { .name = "bios_journal_completed_read", .mode = 0444, }, + .print = pool_stats_print_bios_journal_completed_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_journal_completed_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal_completed.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_write = { + .attr = { .name = "bios_journal_completed_write", .mode = 0444, }, + .print = pool_stats_print_bios_journal_completed_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_journal_completed_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal_completed.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_empty_flush = { + .attr = { .name = "bios_journal_completed_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_journal_completed_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_journal_completed_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal_completed.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_discard = { + .attr = { .name = "bios_journal_completed_discard", .mode = 0444, }, + .print = pool_stats_print_bios_journal_completed_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_journal_completed_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal_completed.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_flush = { + .attr = { .name = "bios_journal_completed_flush", .mode = 0444, }, + .print = pool_stats_print_bios_journal_completed_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_journal_completed_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_journal_completed.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_fua = { + .attr = { .name = "bios_journal_completed_fua", .mode = 0444, }, + .print = pool_stats_print_bios_journal_completed_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_page_cache_completed_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_read = { + .attr = { .name = "bios_page_cache_completed_read", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_completed_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_page_cache_completed_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_write = { + .attr = { .name = "bios_page_cache_completed_write", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_completed_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_page_cache_completed_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_empty_flush = { + .attr = { .name = "bios_page_cache_completed_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_completed_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_page_cache_completed_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_discard = { + .attr = { .name = "bios_page_cache_completed_discard", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_completed_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_page_cache_completed_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_flush = { + .attr = { .name = "bios_page_cache_completed_flush", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_completed_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_page_cache_completed_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_fua = { + .attr = { .name = "bios_page_cache_completed_fua", .mode = 0444, }, + .print = pool_stats_print_bios_page_cache_completed_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_acknowledged_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_read = { + .attr = { .name = "bios_acknowledged_read", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_acknowledged_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_write = { + .attr = { .name = "bios_acknowledged_write", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_acknowledged_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_empty_flush = { + .attr = { .name = "bios_acknowledged_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_acknowledged_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_discard = { + .attr = { .name = "bios_acknowledged_discard", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_acknowledged_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_flush = { + .attr = { .name = "bios_acknowledged_flush", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_acknowledged_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_fua = { + .attr = { .name = "bios_acknowledged_fua", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_acknowledged_partial_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_read = { + .attr = { .name = "bios_acknowledged_partial_read", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_partial_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_acknowledged_partial_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_write = { + .attr = { .name = "bios_acknowledged_partial_write", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_partial_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_acknowledged_partial_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_empty_flush = { + .attr = { .name = "bios_acknowledged_partial_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_partial_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_acknowledged_partial_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_discard = { + .attr = { .name = "bios_acknowledged_partial_discard", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_partial_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_acknowledged_partial_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_flush = { + .attr = { .name = "bios_acknowledged_partial_flush", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_partial_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_acknowledged_partial_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_fua = { + .attr = { .name = "bios_acknowledged_partial_fua", .mode = 0444, }, + .print = pool_stats_print_bios_acknowledged_partial_fua, +}; + +/* Number of REQ_OP_READ bios */ +static ssize_t +pool_stats_print_bios_in_progress_read(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_progress.read); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_progress_read = { + .attr = { .name = "bios_in_progress_read", .mode = 0444, }, + .print = pool_stats_print_bios_in_progress_read, +}; + +/* Number of REQ_OP_WRITE bios with data */ +static ssize_t +pool_stats_print_bios_in_progress_write(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_progress.write); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_progress_write = { + .attr = { .name = "bios_in_progress_write", .mode = 0444, }, + .print = pool_stats_print_bios_in_progress_write, +}; + +/* Number of bios tagged with REQ_PREFLUSH and containing no data */ +static ssize_t +pool_stats_print_bios_in_progress_empty_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_progress.empty_flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_progress_empty_flush = { + .attr = { .name = "bios_in_progress_empty_flush", .mode = 0444, }, + .print = pool_stats_print_bios_in_progress_empty_flush, +}; + +/* Number of REQ_OP_DISCARD bios */ +static ssize_t +pool_stats_print_bios_in_progress_discard(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_progress.discard); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_progress_discard = { + .attr = { .name = "bios_in_progress_discard", .mode = 0444, }, + .print = pool_stats_print_bios_in_progress_discard, +}; + +/* Number of bios tagged with REQ_PREFLUSH */ +static ssize_t +pool_stats_print_bios_in_progress_flush(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_progress.flush); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_progress_flush = { + .attr = { .name = "bios_in_progress_flush", .mode = 0444, }, + .print = pool_stats_print_bios_in_progress_flush, +}; + +/* Number of bios tagged with REQ_FUA */ +static ssize_t +pool_stats_print_bios_in_progress_fua(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->bios_in_progress.fua); +} + +static struct pool_stats_attribute pool_stats_attr_bios_in_progress_fua = { + .attr = { .name = "bios_in_progress_fua", .mode = 0444, }, + .print = pool_stats_print_bios_in_progress_fua, +}; + +/* Tracked bytes currently allocated. */ +static ssize_t +pool_stats_print_memory_usage_bytes_used(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->memory_usage.bytes_used); +} + +static struct pool_stats_attribute pool_stats_attr_memory_usage_bytes_used = { + .attr = { .name = "memory_usage_bytes_used", .mode = 0444, }, + .print = pool_stats_print_memory_usage_bytes_used, +}; + +/* Maximum tracked bytes allocated. */ +static ssize_t +pool_stats_print_memory_usage_peak_bytes_used(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->memory_usage.peak_bytes_used); +} + +static struct pool_stats_attribute pool_stats_attr_memory_usage_peak_bytes_used = { + .attr = { .name = "memory_usage_peak_bytes_used", .mode = 0444, }, + .print = pool_stats_print_memory_usage_peak_bytes_used, +}; + +/* Number of records stored in the index */ +static ssize_t +pool_stats_print_index_entries_indexed(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.entries_indexed); +} + +static struct pool_stats_attribute pool_stats_attr_index_entries_indexed = { + .attr = { .name = "index_entries_indexed", .mode = 0444, }, + .print = pool_stats_print_index_entries_indexed, +}; + +/* Number of post calls that found an existing entry */ +static ssize_t +pool_stats_print_index_posts_found(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.posts_found); +} + +static struct pool_stats_attribute pool_stats_attr_index_posts_found = { + .attr = { .name = "index_posts_found", .mode = 0444, }, + .print = pool_stats_print_index_posts_found, +}; + +/* Number of post calls that added a new entry */ +static ssize_t +pool_stats_print_index_posts_not_found(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.posts_not_found); +} + +static struct pool_stats_attribute pool_stats_attr_index_posts_not_found = { + .attr = { .name = "index_posts_not_found", .mode = 0444, }, + .print = pool_stats_print_index_posts_not_found, +}; + +/* Number of query calls that found an existing entry */ +static ssize_t +pool_stats_print_index_queries_found(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.queries_found); +} + +static struct pool_stats_attribute pool_stats_attr_index_queries_found = { + .attr = { .name = "index_queries_found", .mode = 0444, }, + .print = pool_stats_print_index_queries_found, +}; + +/* Number of query calls that added a new entry */ +static ssize_t +pool_stats_print_index_queries_not_found(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.queries_not_found); +} + +static struct pool_stats_attribute pool_stats_attr_index_queries_not_found = { + .attr = { .name = "index_queries_not_found", .mode = 0444, }, + .print = pool_stats_print_index_queries_not_found, +}; + +/* Number of update calls that found an existing entry */ +static ssize_t +pool_stats_print_index_updates_found(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.updates_found); +} + +static struct pool_stats_attribute pool_stats_attr_index_updates_found = { + .attr = { .name = "index_updates_found", .mode = 0444, }, + .print = pool_stats_print_index_updates_found, +}; + +/* Number of update calls that added a new entry */ +static ssize_t +pool_stats_print_index_updates_not_found(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.updates_not_found); +} + +static struct pool_stats_attribute pool_stats_attr_index_updates_not_found = { + .attr = { .name = "index_updates_not_found", .mode = 0444, }, + .print = pool_stats_print_index_updates_not_found, +}; + +/* Number of entries discarded */ +static ssize_t +pool_stats_print_index_entries_discarded(struct vdo_statistics *stats, char *buf) +{ + return sprintf(buf, "%llu\n", stats->index.entries_discarded); +} + +static struct pool_stats_attribute pool_stats_attr_index_entries_discarded = { + .attr = { .name = "index_entries_discarded", .mode = 0444, }, + .print = pool_stats_print_index_entries_discarded, +}; + +struct attribute *vdo_pool_stats_attrs[] = { + &pool_stats_attr_data_blocks_used.attr, + &pool_stats_attr_overhead_blocks_used.attr, + &pool_stats_attr_logical_blocks_used.attr, + &pool_stats_attr_physical_blocks.attr, + &pool_stats_attr_logical_blocks.attr, + &pool_stats_attr_block_map_cache_size.attr, + &pool_stats_attr_block_size.attr, + &pool_stats_attr_complete_recoveries.attr, + &pool_stats_attr_read_only_recoveries.attr, + &pool_stats_attr_mode.attr, + &pool_stats_attr_in_recovery_mode.attr, + &pool_stats_attr_recovery_percentage.attr, + &pool_stats_attr_packer_compressed_fragments_written.attr, + &pool_stats_attr_packer_compressed_blocks_written.attr, + &pool_stats_attr_packer_compressed_fragments_in_packer.attr, + &pool_stats_attr_allocator_slab_count.attr, + &pool_stats_attr_allocator_slabs_opened.attr, + &pool_stats_attr_allocator_slabs_reopened.attr, + &pool_stats_attr_journal_disk_full.attr, + &pool_stats_attr_journal_slab_journal_commits_requested.attr, + &pool_stats_attr_journal_entries_started.attr, + &pool_stats_attr_journal_entries_written.attr, + &pool_stats_attr_journal_entries_committed.attr, + &pool_stats_attr_journal_blocks_started.attr, + &pool_stats_attr_journal_blocks_written.attr, + &pool_stats_attr_journal_blocks_committed.attr, + &pool_stats_attr_slab_journal_disk_full_count.attr, + &pool_stats_attr_slab_journal_flush_count.attr, + &pool_stats_attr_slab_journal_blocked_count.attr, + &pool_stats_attr_slab_journal_blocks_written.attr, + &pool_stats_attr_slab_journal_tail_busy_count.attr, + &pool_stats_attr_slab_summary_blocks_written.attr, + &pool_stats_attr_ref_counts_blocks_written.attr, + &pool_stats_attr_block_map_dirty_pages.attr, + &pool_stats_attr_block_map_clean_pages.attr, + &pool_stats_attr_block_map_free_pages.attr, + &pool_stats_attr_block_map_failed_pages.attr, + &pool_stats_attr_block_map_incoming_pages.attr, + &pool_stats_attr_block_map_outgoing_pages.attr, + &pool_stats_attr_block_map_cache_pressure.attr, + &pool_stats_attr_block_map_read_count.attr, + &pool_stats_attr_block_map_write_count.attr, + &pool_stats_attr_block_map_failed_reads.attr, + &pool_stats_attr_block_map_failed_writes.attr, + &pool_stats_attr_block_map_reclaimed.attr, + &pool_stats_attr_block_map_read_outgoing.attr, + &pool_stats_attr_block_map_found_in_cache.attr, + &pool_stats_attr_block_map_discard_required.attr, + &pool_stats_attr_block_map_wait_for_page.attr, + &pool_stats_attr_block_map_fetch_required.attr, + &pool_stats_attr_block_map_pages_loaded.attr, + &pool_stats_attr_block_map_pages_saved.attr, + &pool_stats_attr_block_map_flush_count.attr, + &pool_stats_attr_hash_lock_dedupe_advice_valid.attr, + &pool_stats_attr_hash_lock_dedupe_advice_stale.attr, + &pool_stats_attr_hash_lock_concurrent_data_matches.attr, + &pool_stats_attr_hash_lock_concurrent_hash_collisions.attr, + &pool_stats_attr_hash_lock_curr_dedupe_queries.attr, + &pool_stats_attr_errors_invalid_advice_pbn_count.attr, + &pool_stats_attr_errors_no_space_error_count.attr, + &pool_stats_attr_errors_read_only_error_count.attr, + &pool_stats_attr_instance.attr, + &pool_stats_attr_current_vios_in_progress.attr, + &pool_stats_attr_max_vios.attr, + &pool_stats_attr_dedupe_advice_timeouts.attr, + &pool_stats_attr_flush_out.attr, + &pool_stats_attr_logical_block_size.attr, + &pool_stats_attr_bios_in_read.attr, + &pool_stats_attr_bios_in_write.attr, + &pool_stats_attr_bios_in_empty_flush.attr, + &pool_stats_attr_bios_in_discard.attr, + &pool_stats_attr_bios_in_flush.attr, + &pool_stats_attr_bios_in_fua.attr, + &pool_stats_attr_bios_in_partial_read.attr, + &pool_stats_attr_bios_in_partial_write.attr, + &pool_stats_attr_bios_in_partial_empty_flush.attr, + &pool_stats_attr_bios_in_partial_discard.attr, + &pool_stats_attr_bios_in_partial_flush.attr, + &pool_stats_attr_bios_in_partial_fua.attr, + &pool_stats_attr_bios_out_read.attr, + &pool_stats_attr_bios_out_write.attr, + &pool_stats_attr_bios_out_empty_flush.attr, + &pool_stats_attr_bios_out_discard.attr, + &pool_stats_attr_bios_out_flush.attr, + &pool_stats_attr_bios_out_fua.attr, + &pool_stats_attr_bios_meta_read.attr, + &pool_stats_attr_bios_meta_write.attr, + &pool_stats_attr_bios_meta_empty_flush.attr, + &pool_stats_attr_bios_meta_discard.attr, + &pool_stats_attr_bios_meta_flush.attr, + &pool_stats_attr_bios_meta_fua.attr, + &pool_stats_attr_bios_journal_read.attr, + &pool_stats_attr_bios_journal_write.attr, + &pool_stats_attr_bios_journal_empty_flush.attr, + &pool_stats_attr_bios_journal_discard.attr, + &pool_stats_attr_bios_journal_flush.attr, + &pool_stats_attr_bios_journal_fua.attr, + &pool_stats_attr_bios_page_cache_read.attr, + &pool_stats_attr_bios_page_cache_write.attr, + &pool_stats_attr_bios_page_cache_empty_flush.attr, + &pool_stats_attr_bios_page_cache_discard.attr, + &pool_stats_attr_bios_page_cache_flush.attr, + &pool_stats_attr_bios_page_cache_fua.attr, + &pool_stats_attr_bios_out_completed_read.attr, + &pool_stats_attr_bios_out_completed_write.attr, + &pool_stats_attr_bios_out_completed_empty_flush.attr, + &pool_stats_attr_bios_out_completed_discard.attr, + &pool_stats_attr_bios_out_completed_flush.attr, + &pool_stats_attr_bios_out_completed_fua.attr, + &pool_stats_attr_bios_meta_completed_read.attr, + &pool_stats_attr_bios_meta_completed_write.attr, + &pool_stats_attr_bios_meta_completed_empty_flush.attr, + &pool_stats_attr_bios_meta_completed_discard.attr, + &pool_stats_attr_bios_meta_completed_flush.attr, + &pool_stats_attr_bios_meta_completed_fua.attr, + &pool_stats_attr_bios_journal_completed_read.attr, + &pool_stats_attr_bios_journal_completed_write.attr, + &pool_stats_attr_bios_journal_completed_empty_flush.attr, + &pool_stats_attr_bios_journal_completed_discard.attr, + &pool_stats_attr_bios_journal_completed_flush.attr, + &pool_stats_attr_bios_journal_completed_fua.attr, + &pool_stats_attr_bios_page_cache_completed_read.attr, + &pool_stats_attr_bios_page_cache_completed_write.attr, + &pool_stats_attr_bios_page_cache_completed_empty_flush.attr, + &pool_stats_attr_bios_page_cache_completed_discard.attr, + &pool_stats_attr_bios_page_cache_completed_flush.attr, + &pool_stats_attr_bios_page_cache_completed_fua.attr, + &pool_stats_attr_bios_acknowledged_read.attr, + &pool_stats_attr_bios_acknowledged_write.attr, + &pool_stats_attr_bios_acknowledged_empty_flush.attr, + &pool_stats_attr_bios_acknowledged_discard.attr, + &pool_stats_attr_bios_acknowledged_flush.attr, + &pool_stats_attr_bios_acknowledged_fua.attr, + &pool_stats_attr_bios_acknowledged_partial_read.attr, + &pool_stats_attr_bios_acknowledged_partial_write.attr, + &pool_stats_attr_bios_acknowledged_partial_empty_flush.attr, + &pool_stats_attr_bios_acknowledged_partial_discard.attr, + &pool_stats_attr_bios_acknowledged_partial_flush.attr, + &pool_stats_attr_bios_acknowledged_partial_fua.attr, + &pool_stats_attr_bios_in_progress_read.attr, + &pool_stats_attr_bios_in_progress_write.attr, + &pool_stats_attr_bios_in_progress_empty_flush.attr, + &pool_stats_attr_bios_in_progress_discard.attr, + &pool_stats_attr_bios_in_progress_flush.attr, + &pool_stats_attr_bios_in_progress_fua.attr, + &pool_stats_attr_memory_usage_bytes_used.attr, + &pool_stats_attr_memory_usage_peak_bytes_used.attr, + &pool_stats_attr_index_entries_indexed.attr, + &pool_stats_attr_index_posts_found.attr, + &pool_stats_attr_index_posts_not_found.attr, + &pool_stats_attr_index_queries_found.attr, + &pool_stats_attr_index_queries_not_found.attr, + &pool_stats_attr_index_updates_found.attr, + &pool_stats_attr_index_updates_not_found.attr, + &pool_stats_attr_index_entries_discarded.attr, + NULL, +}; diff --git a/drivers/md/dm-vdo/pool-sysfs.c b/drivers/md/dm-vdo/pool-sysfs.c new file mode 100644 index 000000000000..f2be0f2bbd68 --- /dev/null +++ b/drivers/md/dm-vdo/pool-sysfs.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "pool-sysfs.h" + +#include + +#include "memory-alloc.h" +#include "string-utils.h" + +#include "data-vio.h" +#include "dedupe.h" +#include "vdo.h" + +struct pool_attribute { + struct attribute attr; + ssize_t (*show)(struct vdo *vdo, char *buf); + ssize_t (*store)(struct vdo *vdo, const char *value, size_t count); +}; + +static ssize_t vdo_pool_attr_show(struct kobject *directory, struct attribute *attr, + char *buf) +{ + struct pool_attribute *pool_attr = container_of(attr, struct pool_attribute, + attr); + struct vdo *vdo = container_of(directory, struct vdo, vdo_directory); + + if (pool_attr->show == NULL) + return -EINVAL; + return pool_attr->show(vdo, buf); +} + +static ssize_t vdo_pool_attr_store(struct kobject *directory, struct attribute *attr, + const char *buf, size_t length) +{ + struct pool_attribute *pool_attr = container_of(attr, struct pool_attribute, + attr); + struct vdo *vdo = container_of(directory, struct vdo, vdo_directory); + + if (pool_attr->store == NULL) + return -EINVAL; + return pool_attr->store(vdo, buf, length); +} + +static const struct sysfs_ops vdo_pool_sysfs_ops = { + .show = vdo_pool_attr_show, + .store = vdo_pool_attr_store, +}; + +static ssize_t pool_compressing_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%s\n", (vdo_get_compressing(vdo) ? "1" : "0")); +} + +static ssize_t pool_discards_active_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", + get_data_vio_pool_active_discards(vdo->data_vio_pool)); +} + +static ssize_t pool_discards_limit_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", get_data_vio_pool_discard_limit(vdo->data_vio_pool)); +} + +static ssize_t pool_discards_limit_store(struct vdo *vdo, const char *buf, size_t length) +{ + unsigned int value; + int result; + + if ((length > 12) || (kstrtouint(buf, 10, &value) < 0) || (value < 1)) + return -EINVAL; + + result = set_data_vio_pool_discard_limit(vdo->data_vio_pool, value); + if (result != VDO_SUCCESS) + return -EINVAL; + + return length; +} + +static ssize_t pool_discards_maximum_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", + get_data_vio_pool_maximum_discards(vdo->data_vio_pool)); +} + +static ssize_t pool_instance_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", vdo->instance); +} + +static ssize_t pool_requests_active_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", + get_data_vio_pool_active_requests(vdo->data_vio_pool)); +} + +static ssize_t pool_requests_limit_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", get_data_vio_pool_request_limit(vdo->data_vio_pool)); +} + +static ssize_t pool_requests_maximum_show(struct vdo *vdo, char *buf) +{ + return sprintf(buf, "%u\n", + get_data_vio_pool_maximum_requests(vdo->data_vio_pool)); +} + +static void vdo_pool_release(struct kobject *directory) +{ + uds_free(container_of(directory, struct vdo, vdo_directory)); +} + +static struct pool_attribute vdo_pool_compressing_attr = { + .attr = { + .name = "compressing", + .mode = 0444, + }, + .show = pool_compressing_show, +}; + +static struct pool_attribute vdo_pool_discards_active_attr = { + .attr = { + .name = "discards_active", + .mode = 0444, + }, + .show = pool_discards_active_show, +}; + +static struct pool_attribute vdo_pool_discards_limit_attr = { + .attr = { + .name = "discards_limit", + .mode = 0644, + }, + .show = pool_discards_limit_show, + .store = pool_discards_limit_store, +}; + +static struct pool_attribute vdo_pool_discards_maximum_attr = { + .attr = { + .name = "discards_maximum", + .mode = 0444, + }, + .show = pool_discards_maximum_show, +}; + +static struct pool_attribute vdo_pool_instance_attr = { + .attr = { + .name = "instance", + .mode = 0444, + }, + .show = pool_instance_show, +}; + +static struct pool_attribute vdo_pool_requests_active_attr = { + .attr = { + .name = "requests_active", + .mode = 0444, + }, + .show = pool_requests_active_show, +}; + +static struct pool_attribute vdo_pool_requests_limit_attr = { + .attr = { + .name = "requests_limit", + .mode = 0444, + }, + .show = pool_requests_limit_show, +}; + +static struct pool_attribute vdo_pool_requests_maximum_attr = { + .attr = { + .name = "requests_maximum", + .mode = 0444, + }, + .show = pool_requests_maximum_show, +}; + +static struct attribute *pool_attrs[] = { + &vdo_pool_compressing_attr.attr, + &vdo_pool_discards_active_attr.attr, + &vdo_pool_discards_limit_attr.attr, + &vdo_pool_discards_maximum_attr.attr, + &vdo_pool_instance_attr.attr, + &vdo_pool_requests_active_attr.attr, + &vdo_pool_requests_limit_attr.attr, + &vdo_pool_requests_maximum_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(pool); + +const struct kobj_type vdo_directory_type = { + .release = vdo_pool_release, + .sysfs_ops = &vdo_pool_sysfs_ops, + .default_groups = pool_groups, +}; diff --git a/drivers/md/dm-vdo/pool-sysfs.h b/drivers/md/dm-vdo/pool-sysfs.h new file mode 100644 index 000000000000..00e680924dc1 --- /dev/null +++ b/drivers/md/dm-vdo/pool-sysfs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_POOL_SYSFS_H +#define VDO_POOL_SYSFS_H + +#include + +/* The kobj_type used for setting up the kernel layer kobject. */ +extern const struct kobj_type vdo_directory_type; + +/* The sysfs_ops used for the "statistics" subdirectory. */ +extern const struct sysfs_ops vdo_pool_stats_sysfs_ops; +/* The attribute used for the "statistics" subdirectory. */ +extern struct attribute *vdo_pool_stats_attrs[]; + +#endif /* VDO_POOL_SYSFS_H */ diff --git a/drivers/md/dm-vdo/sysfs.c b/drivers/md/dm-vdo/sysfs.c new file mode 100644 index 000000000000..70feffe9d4c4 --- /dev/null +++ b/drivers/md/dm-vdo/sysfs.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include + +#include "logger.h" + +#include "constants.h" +#include "dedupe.h" +#include "vdo.h" + +static int vdo_log_level_show(char *buf, const struct kernel_param *kp) +{ + return sprintf(buf, "%s\n", uds_log_priority_to_string(uds_get_log_level())); +} + +static int vdo_log_level_store(const char *buf, const struct kernel_param *kp) +{ + static char internal_buf[11]; + + int n = strlen(buf); + + if (n > 10) + return -EINVAL; + + memset(internal_buf, '\000', sizeof(internal_buf)); + memcpy(internal_buf, buf, n); + if (internal_buf[n - 1] == '\n') + internal_buf[n - 1] = '\000'; + uds_set_log_level(uds_log_string_to_priority(internal_buf)); + return 0; +} + + +static int vdo_dedupe_timeout_interval_store(const char *buf, + const struct kernel_param *kp) +{ + int result = param_set_uint(buf, kp); + + if (result != 0) + return result; + vdo_set_dedupe_index_timeout_interval(*(uint *)kp->arg); + return 0; +} + +static int vdo_min_dedupe_timer_interval_store(const char *buf, + const struct kernel_param *kp) +{ + int result = param_set_uint(buf, kp); + + if (result != 0) + return result; + vdo_set_dedupe_index_min_timer_interval(*(uint *)kp->arg); + return 0; +} + +static const struct kernel_param_ops log_level_ops = { + .set = vdo_log_level_store, + .get = vdo_log_level_show, +}; + + +static const struct kernel_param_ops dedupe_timeout_ops = { + .set = vdo_dedupe_timeout_interval_store, + .get = param_get_uint, +}; + +static const struct kernel_param_ops dedupe_timer_ops = { + .set = vdo_min_dedupe_timer_interval_store, + .get = param_get_uint, +}; + +module_param_cb(log_level, &log_level_ops, NULL, 0644); + + +module_param_cb(deduplication_timeout_interval, &dedupe_timeout_ops, + &vdo_dedupe_index_timeout_interval, 0644); + +module_param_cb(min_deduplication_timer_interval, &dedupe_timer_ops, + &vdo_dedupe_index_min_timer_interval, 0644); From 29a811959c72b60601842b5c61a66d677ecbcc92 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:12:14 -0500 Subject: [PATCH 0585/1038] dm vdo: add debugging support Add support for dumping detailed vdo state to the kernel log via a dmsetup message. The dump code is not thread-safe and is generally intended for use only when the vdo is hung. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dump.c | 278 +++++++++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/dump.h | 17 +++ 2 files changed, 295 insertions(+) create mode 100644 drivers/md/dm-vdo/dump.c create mode 100644 drivers/md/dm-vdo/dump.h diff --git a/drivers/md/dm-vdo/dump.c b/drivers/md/dm-vdo/dump.c new file mode 100644 index 000000000000..99266a946ed7 --- /dev/null +++ b/drivers/md/dm-vdo/dump.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "dump.h" + +#include + +#include "memory-alloc.h" +#include "string-utils.h" + +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "funnel-workqueue.h" +#include "io-submitter.h" +#include "logger.h" +#include "types.h" +#include "vdo.h" + +enum dump_options { + /* Work queues */ + SHOW_QUEUES, + /* Memory pools */ + SHOW_VIO_POOL, + /* Others */ + SHOW_VDO_STATUS, + /* This one means an option overrides the "default" choices, instead of altering them. */ + SKIP_DEFAULT +}; + +enum dump_option_flags { + /* Work queues */ + FLAG_SHOW_QUEUES = (1 << SHOW_QUEUES), + /* Memory pools */ + FLAG_SHOW_VIO_POOL = (1 << SHOW_VIO_POOL), + /* Others */ + FLAG_SHOW_VDO_STATUS = (1 << SHOW_VDO_STATUS), + /* Special */ + FLAG_SKIP_DEFAULT = (1 << SKIP_DEFAULT) +}; + +enum { + FLAGS_ALL_POOLS = (FLAG_SHOW_VIO_POOL), + DEFAULT_DUMP_FLAGS = (FLAG_SHOW_QUEUES | FLAG_SHOW_VDO_STATUS) +}; + +static inline bool is_arg_string(const char *arg, const char *this_option) +{ + /* convention seems to be case-independent options */ + return strncasecmp(arg, this_option, strlen(this_option)) == 0; +} + +static void do_dump(struct vdo *vdo, unsigned int dump_options_requested, + const char *why) +{ + u32 active, maximum; + s64 outstanding; + + uds_log_info("%s dump triggered via %s", UDS_LOGGING_MODULE_NAME, why); + active = get_data_vio_pool_active_requests(vdo->data_vio_pool); + maximum = get_data_vio_pool_maximum_requests(vdo->data_vio_pool); + outstanding = (atomic64_read(&vdo->stats.bios_submitted) - + atomic64_read(&vdo->stats.bios_completed)); + uds_log_info("%u device requests outstanding (max %u), %lld bio requests outstanding, device '%s'", + active, maximum, outstanding, + vdo_get_device_name(vdo->device_config->owning_target)); + if (((dump_options_requested & FLAG_SHOW_QUEUES) != 0) && (vdo->threads != NULL)) { + thread_id_t id; + + for (id = 0; id < vdo->thread_config.thread_count; id++) + vdo_dump_work_queue(vdo->threads[id].queue); + } + + vdo_dump_hash_zones(vdo->hash_zones); + dump_data_vio_pool(vdo->data_vio_pool, + (dump_options_requested & FLAG_SHOW_VIO_POOL) != 0); + if ((dump_options_requested & FLAG_SHOW_VDO_STATUS) != 0) + vdo_dump_status(vdo); + + uds_report_memory_usage(); + uds_log_info("end of %s dump", UDS_LOGGING_MODULE_NAME); +} + +static int parse_dump_options(unsigned int argc, char *const *argv, + unsigned int *dump_options_requested_ptr) +{ + unsigned int dump_options_requested = 0; + + static const struct { + const char *name; + unsigned int flags; + } option_names[] = { + { "viopool", FLAG_SKIP_DEFAULT | FLAG_SHOW_VIO_POOL }, + { "vdo", FLAG_SKIP_DEFAULT | FLAG_SHOW_VDO_STATUS }, + { "pools", FLAG_SKIP_DEFAULT | FLAGS_ALL_POOLS }, + { "queues", FLAG_SKIP_DEFAULT | FLAG_SHOW_QUEUES }, + { "threads", FLAG_SKIP_DEFAULT | FLAG_SHOW_QUEUES }, + { "default", FLAG_SKIP_DEFAULT | DEFAULT_DUMP_FLAGS }, + { "all", ~0 }, + }; + + bool options_okay = true; + unsigned int i; + + for (i = 1; i < argc; i++) { + unsigned int j; + + for (j = 0; j < ARRAY_SIZE(option_names); j++) { + if (is_arg_string(argv[i], option_names[j].name)) { + dump_options_requested |= option_names[j].flags; + break; + } + } + if (j == ARRAY_SIZE(option_names)) { + uds_log_warning("dump option name '%s' unknown", argv[i]); + options_okay = false; + } + } + if (!options_okay) + return -EINVAL; + if ((dump_options_requested & FLAG_SKIP_DEFAULT) == 0) + dump_options_requested |= DEFAULT_DUMP_FLAGS; + *dump_options_requested_ptr = dump_options_requested; + return 0; +} + +/* Dump as specified by zero or more string arguments. */ +int vdo_dump(struct vdo *vdo, unsigned int argc, char *const *argv, const char *why) +{ + unsigned int dump_options_requested = 0; + int result = parse_dump_options(argc, argv, &dump_options_requested); + + if (result != 0) + return result; + + do_dump(vdo, dump_options_requested, why); + return 0; +} + +/* Dump everything we know how to dump */ +void vdo_dump_all(struct vdo *vdo, const char *why) +{ + do_dump(vdo, ~0, why); +} + +/* + * Dump out the data_vio waiters on a wait queue. + * wait_on should be the label to print for queue (e.g. logical or physical) + */ +static void dump_vio_waiters(struct wait_queue *queue, char *wait_on) +{ + struct waiter *waiter, *first = vdo_get_first_waiter(queue); + struct data_vio *data_vio; + + if (first == NULL) + return; + + data_vio = waiter_as_data_vio(first); + + uds_log_info(" %s is locked. Waited on by: vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", + wait_on, data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, + data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio)); + + for (waiter = first->next_waiter; waiter != first; waiter = waiter->next_waiter) { + data_vio = waiter_as_data_vio(waiter); + uds_log_info(" ... and : vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", + data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, + data_vio->duplicate.pbn, + get_data_vio_operation_name(data_vio)); + } +} + +/* + * Encode various attributes of a data_vio as a string of one-character flags. This encoding is for + * logging brevity: + * + * R => vio completion result not VDO_SUCCESS + * W => vio is on a wait queue + * D => vio is a duplicate + * p => vio is a partial block operation + * z => vio is a zero block + * d => vio is a discard + * + * The common case of no flags set will result in an empty, null-terminated buffer. If any flags + * are encoded, the first character in the string will be a space character. + */ +static void encode_vio_dump_flags(struct data_vio *data_vio, char buffer[8]) +{ + char *p_flag = buffer; + *p_flag++ = ' '; + if (data_vio->vio.completion.result != VDO_SUCCESS) + *p_flag++ = 'R'; + if (data_vio->waiter.next_waiter != NULL) + *p_flag++ = 'W'; + if (data_vio->is_duplicate) + *p_flag++ = 'D'; + if (data_vio->is_partial) + *p_flag++ = 'p'; + if (data_vio->is_zero) + *p_flag++ = 'z'; + if (data_vio->remaining_discard > 0) + *p_flag++ = 'd'; + if (p_flag == &buffer[1]) { + /* No flags, so remove the blank space. */ + p_flag = buffer; + } + *p_flag = '\0'; +} + +/* Implements buffer_dump_function. */ +void dump_data_vio(void *data) +{ + struct data_vio *data_vio = data; + + /* + * This just needs to be big enough to hold a queue (thread) name and a function name (plus + * a separator character and NUL). The latter is limited only by taste. + * + * In making this static, we're assuming only one "dump" will run at a time. If more than + * one does run, the log output will be garbled anyway. + */ + static char vio_completion_dump_buffer[100 + MAX_VDO_WORK_QUEUE_NAME_LEN]; + /* Another static buffer... log10(256) = 2.408+, round up: */ + enum { DIGITS_PER_U64 = 1 + sizeof(u64) * 2409 / 1000 }; + + static char vio_block_number_dump_buffer[sizeof("P L D") + 3 * DIGITS_PER_U64]; + static char vio_flush_generation_buffer[sizeof(" FG") + DIGITS_PER_U64]; + static char flags_dump_buffer[8]; + + /* + * We're likely to be logging a couple thousand of these lines, and in some circumstances + * syslogd may have trouble keeping up, so keep it BRIEF rather than user-friendly. + */ + vdo_dump_completion_to_buffer(&data_vio->vio.completion, + vio_completion_dump_buffer, + sizeof(vio_completion_dump_buffer)); + if (data_vio->is_duplicate) { + snprintf(vio_block_number_dump_buffer, + sizeof(vio_block_number_dump_buffer), "P%llu L%llu D%llu", + data_vio->allocation.pbn, data_vio->logical.lbn, + data_vio->duplicate.pbn); + } else if (data_vio_has_allocation(data_vio)) { + snprintf(vio_block_number_dump_buffer, + sizeof(vio_block_number_dump_buffer), "P%llu L%llu", + data_vio->allocation.pbn, data_vio->logical.lbn); + } else { + snprintf(vio_block_number_dump_buffer, + sizeof(vio_block_number_dump_buffer), "L%llu", + data_vio->logical.lbn); + } + + if (data_vio->flush_generation != 0) { + snprintf(vio_flush_generation_buffer, + sizeof(vio_flush_generation_buffer), " FG%llu", + data_vio->flush_generation); + } else { + vio_flush_generation_buffer[0] = 0; + } + + encode_vio_dump_flags(data_vio, flags_dump_buffer); + + uds_log_info(" vio %px %s%s %s %s%s", data_vio, + vio_block_number_dump_buffer, + vio_flush_generation_buffer, + get_data_vio_operation_name(data_vio), + vio_completion_dump_buffer, + flags_dump_buffer); + /* + * might want info on: wantUDSAnswer / operation / status + * might want info on: bio / bios_merged + */ + + dump_vio_waiters(&data_vio->logical.waiters, "lbn"); + + /* might want to dump more info from vio here */ +} diff --git a/drivers/md/dm-vdo/dump.h b/drivers/md/dm-vdo/dump.h new file mode 100644 index 000000000000..ad47c70cca78 --- /dev/null +++ b/drivers/md/dm-vdo/dump.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_DUMP_H +#define VDO_DUMP_H + +#include "types.h" + +int vdo_dump(struct vdo *vdo, unsigned int argc, char *const *argv, const char *why); + +void vdo_dump_all(struct vdo *vdo, const char *why); + +void dump_data_vio(void *data); + +#endif /* VDO_DUMP_H */ From 03d1e20fa16e0aaa753c09f2ae72faec27a01273 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:24:06 -0500 Subject: [PATCH 0586/1038] dm vdo: add the top-level DM target This adds the dm-vdo target. The dm-vdo target provides inline deduplication, compression, and zero-block elimination, allowing applications to consume less actual storage than a normal target. By layering it with other device mapper targets, it can add these features to any storage stack. It can also provide a common deduplication pool for groups of targets. The vdo target does not protect against data corruption, relying instead on integrity protection of the storage below it. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: Michael Sclafani Signed-off-by: Michael Sclafani Co-developed-by: Sweet Tea Dorminy Signed-off-by: Sweet Tea Dorminy Co-developed-by: Bruce Johnston Signed-off-by: Bruce Johnston Co-developed-by: Ken Raeburn Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo-target.c | 2952 ++++++++++++++++++++++++++++++++++++ 1 file changed, 2952 insertions(+) create mode 100644 drivers/md/dm-vdo-target.c diff --git a/drivers/md/dm-vdo-target.c b/drivers/md/dm-vdo-target.c new file mode 100644 index 000000000000..72d2291dd93f --- /dev/null +++ b/drivers/md/dm-vdo-target.c @@ -0,0 +1,2952 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dm-vdo/admin-state.h" +#include "dm-vdo/block-map.h" +#include "dm-vdo/completion.h" +#include "dm-vdo/constants.h" +#include "dm-vdo/data-vio.h" +#include "dm-vdo/dedupe.h" +#include "dm-vdo/dump.h" +#include "dm-vdo/encodings.h" +#include "dm-vdo/errors.h" +#include "dm-vdo/flush.h" +#include "dm-vdo/io-submitter.h" +#include "dm-vdo/logger.h" +#include "dm-vdo/memory-alloc.h" +#include "dm-vdo/message-stats.h" +#include "dm-vdo/pool-sysfs.h" +#include "dm-vdo/recovery-journal.h" +#include "dm-vdo/repair.h" +#include "dm-vdo/slab-depot.h" +#include "dm-vdo/status-codes.h" +#include "dm-vdo/string-utils.h" +#include "dm-vdo/thread-device.h" +#include "dm-vdo/thread-registry.h" +#include "dm-vdo/types.h" +#include "dm-vdo/uds-sysfs.h" +#include "dm-vdo/vdo.h" +#include "dm-vdo/vio.h" + +#define CURRENT_VERSION "8.3.0.65" + +enum { + GROW_LOGICAL_PHASE_START, + GROW_LOGICAL_PHASE_GROW_BLOCK_MAP, + GROW_LOGICAL_PHASE_END, + GROW_LOGICAL_PHASE_ERROR, + GROW_PHYSICAL_PHASE_START, + GROW_PHYSICAL_PHASE_COPY_SUMMARY, + GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS, + GROW_PHYSICAL_PHASE_USE_NEW_SLABS, + GROW_PHYSICAL_PHASE_END, + GROW_PHYSICAL_PHASE_ERROR, + LOAD_PHASE_START, + LOAD_PHASE_STATS, + LOAD_PHASE_LOAD_DEPOT, + LOAD_PHASE_MAKE_DIRTY, + LOAD_PHASE_PREPARE_TO_ALLOCATE, + LOAD_PHASE_SCRUB_SLABS, + LOAD_PHASE_DATA_REDUCTION, + LOAD_PHASE_FINISHED, + LOAD_PHASE_DRAIN_JOURNAL, + LOAD_PHASE_WAIT_FOR_READ_ONLY, + PRE_LOAD_PHASE_START, + PRE_LOAD_PHASE_LOAD_COMPONENTS, + PRE_LOAD_PHASE_END, + PREPARE_GROW_PHYSICAL_PHASE_START, + RESUME_PHASE_START, + RESUME_PHASE_ALLOW_READ_ONLY_MODE, + RESUME_PHASE_DEDUPE, + RESUME_PHASE_DEPOT, + RESUME_PHASE_JOURNAL, + RESUME_PHASE_BLOCK_MAP, + RESUME_PHASE_LOGICAL_ZONES, + RESUME_PHASE_PACKER, + RESUME_PHASE_FLUSHER, + RESUME_PHASE_DATA_VIOS, + RESUME_PHASE_END, + SUSPEND_PHASE_START, + SUSPEND_PHASE_PACKER, + SUSPEND_PHASE_DATA_VIOS, + SUSPEND_PHASE_DEDUPE, + SUSPEND_PHASE_FLUSHES, + SUSPEND_PHASE_LOGICAL_ZONES, + SUSPEND_PHASE_BLOCK_MAP, + SUSPEND_PHASE_JOURNAL, + SUSPEND_PHASE_DEPOT, + SUSPEND_PHASE_READ_ONLY_WAIT, + SUSPEND_PHASE_WRITE_SUPER_BLOCK, + SUSPEND_PHASE_END, +}; + +static const char * const ADMIN_PHASE_NAMES[] = { + "GROW_LOGICAL_PHASE_START", + "GROW_LOGICAL_PHASE_GROW_BLOCK_MAP", + "GROW_LOGICAL_PHASE_END", + "GROW_LOGICAL_PHASE_ERROR", + "GROW_PHYSICAL_PHASE_START", + "GROW_PHYSICAL_PHASE_COPY_SUMMARY", + "GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS", + "GROW_PHYSICAL_PHASE_USE_NEW_SLABS", + "GROW_PHYSICAL_PHASE_END", + "GROW_PHYSICAL_PHASE_ERROR", + "LOAD_PHASE_START", + "LOAD_PHASE_STATS", + "LOAD_PHASE_LOAD_DEPOT", + "LOAD_PHASE_MAKE_DIRTY", + "LOAD_PHASE_PREPARE_TO_ALLOCATE", + "LOAD_PHASE_SCRUB_SLABS", + "LOAD_PHASE_DATA_REDUCTION", + "LOAD_PHASE_FINISHED", + "LOAD_PHASE_DRAIN_JOURNAL", + "LOAD_PHASE_WAIT_FOR_READ_ONLY", + "PRE_LOAD_PHASE_START", + "PRE_LOAD_PHASE_LOAD_COMPONENTS", + "PRE_LOAD_PHASE_END", + "PREPARE_GROW_PHYSICAL_PHASE_START", + "RESUME_PHASE_START", + "RESUME_PHASE_ALLOW_READ_ONLY_MODE", + "RESUME_PHASE_DEDUPE", + "RESUME_PHASE_DEPOT", + "RESUME_PHASE_JOURNAL", + "RESUME_PHASE_BLOCK_MAP", + "RESUME_PHASE_LOGICAL_ZONES", + "RESUME_PHASE_PACKER", + "RESUME_PHASE_FLUSHER", + "RESUME_PHASE_DATA_VIOS", + "RESUME_PHASE_END", + "SUSPEND_PHASE_START", + "SUSPEND_PHASE_PACKER", + "SUSPEND_PHASE_DATA_VIOS", + "SUSPEND_PHASE_DEDUPE", + "SUSPEND_PHASE_FLUSHES", + "SUSPEND_PHASE_LOGICAL_ZONES", + "SUSPEND_PHASE_BLOCK_MAP", + "SUSPEND_PHASE_JOURNAL", + "SUSPEND_PHASE_DEPOT", + "SUSPEND_PHASE_READ_ONLY_WAIT", + "SUSPEND_PHASE_WRITE_SUPER_BLOCK", + "SUSPEND_PHASE_END", +}; + +enum { + /* If we bump this, update the arrays below */ + TABLE_VERSION = 4, +}; + +/* arrays for handling different table versions */ +static const u8 REQUIRED_ARGC[] = { 10, 12, 9, 7, 6 }; +/* pool name no longer used. only here for verification of older versions */ +static const u8 POOL_NAME_ARG_INDEX[] = { 8, 10, 8 }; + +/* + * Track in-use instance numbers using a flat bit array. + * + * O(n) run time isn't ideal, but if we have 1000 VDO devices in use simultaneously we still only + * need to scan 16 words, so it's not likely to be a big deal compared to other resource usage. + */ + +enum { + /* + * This minimum size for the bit array creates a numbering space of 0-999, which allows + * successive starts of the same volume to have different instance numbers in any + * reasonably-sized test. Changing instances on restart allows vdoMonReport to detect that + * the ephemeral stats have reset to zero. + */ + BIT_COUNT_MINIMUM = 1000, + /** Grow the bit array by this many bits when needed */ + BIT_COUNT_INCREMENT = 100, +}; + +struct instance_tracker { + unsigned int bit_count; + unsigned long *words; + unsigned int count; + unsigned int next; +}; + +static DEFINE_MUTEX(instances_lock); +static struct instance_tracker instances; + +/** + * free_device_config() - Free a device config created by parse_device_config(). + * @config: The config to free. + */ +static void free_device_config(struct device_config *config) +{ + if (config == NULL) + return; + + if (config->owned_device != NULL) + dm_put_device(config->owning_target, config->owned_device); + + uds_free(config->parent_device_name); + uds_free(config->original_string); + + /* Reduce the chance a use-after-free (as in BZ 1669960) happens to work. */ + memset(config, 0, sizeof(*config)); + uds_free(config); +} + +/** + * get_version_number() - Decide the version number from argv. + * + * @argc: The number of table values. + * @argv: The array of table values. + * @error_ptr: A pointer to return a error string in. + * @version_ptr: A pointer to return the version. + * + * Return: VDO_SUCCESS or an error code. + */ +static int get_version_number(int argc, char **argv, char **error_ptr, + unsigned int *version_ptr) +{ + /* version, if it exists, is in a form of V */ + if (sscanf(argv[0], "V%u", version_ptr) == 1) { + if (*version_ptr < 1 || *version_ptr > TABLE_VERSION) { + *error_ptr = "Unknown version number detected"; + return VDO_BAD_CONFIGURATION; + } + } else { + /* V0 actually has no version number in the table string */ + *version_ptr = 0; + } + + /* + * V0 and V1 have no optional parameters. There will always be a parameter for thread + * config, even if it's a "." to show it's an empty list. + */ + if (*version_ptr <= 1) { + if (argc != REQUIRED_ARGC[*version_ptr]) { + *error_ptr = "Incorrect number of arguments for version"; + return VDO_BAD_CONFIGURATION; + } + } else if (argc < REQUIRED_ARGC[*version_ptr]) { + *error_ptr = "Incorrect number of arguments for version"; + return VDO_BAD_CONFIGURATION; + } + + if (*version_ptr != TABLE_VERSION) { + uds_log_warning("Detected version mismatch between kernel module and tools kernel: %d, tool: %d", + TABLE_VERSION, *version_ptr); + uds_log_warning("Please consider upgrading management tools to match kernel."); + } + return VDO_SUCCESS; +} + +/* Free a list of non-NULL string pointers, and then the list itself. */ +static void free_string_array(char **string_array) +{ + unsigned int offset; + + for (offset = 0; string_array[offset] != NULL; offset++) + uds_free(string_array[offset]); + uds_free(string_array); +} + +/* + * Split the input string into substrings, separated at occurrences of the indicated character, + * returning a null-terminated list of string pointers. + * + * The string pointers and the pointer array itself should both be freed with uds_free() when no + * longer needed. This can be done with vdo_free_string_array (below) if the pointers in the array + * are not changed. Since the array and copied strings are allocated by this function, it may only + * be used in contexts where allocation is permitted. + * + * Empty substrings are not ignored; that is, returned substrings may be empty strings if the + * separator occurs twice in a row. + */ +static int split_string(const char *string, char separator, char ***substring_array_ptr) +{ + unsigned int current_substring = 0, substring_count = 1; + const char *s; + char **substrings; + int result; + ptrdiff_t length; + + for (s = string; *s != 0; s++) { + if (*s == separator) + substring_count++; + } + + result = uds_allocate(substring_count + 1, char *, "string-splitting array", + &substrings); + if (result != UDS_SUCCESS) + return result; + + for (s = string; *s != 0; s++) { + if (*s == separator) { + ptrdiff_t length = s - string; + + result = uds_allocate(length + 1, char, "split string", + &substrings[current_substring]); + if (result != UDS_SUCCESS) { + free_string_array(substrings); + return result; + } + /* + * Trailing NUL is already in place after allocation; deal with the zero or + * more non-NUL bytes in the string. + */ + if (length > 0) + memcpy(substrings[current_substring], string, length); + string = s + 1; + current_substring++; + BUG_ON(current_substring >= substring_count); + } + } + /* Process final string, with no trailing separator. */ + BUG_ON(current_substring != (substring_count - 1)); + length = strlen(string); + + result = uds_allocate(length + 1, char, "split string", + &substrings[current_substring]); + if (result != UDS_SUCCESS) { + free_string_array(substrings); + return result; + } + memcpy(substrings[current_substring], string, length); + current_substring++; + /* substrings[current_substring] is NULL already */ + *substring_array_ptr = substrings; + return UDS_SUCCESS; +} + +/* + * Join the input substrings into one string, joined with the indicated character, returning a + * string. array_length is a bound on the number of valid elements in substring_array, in case it + * is not NULL-terminated. + */ +static int join_strings(char **substring_array, size_t array_length, char separator, + char **string_ptr) +{ + size_t string_length = 0; + size_t i; + int result; + char *output, *current_position; + + for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++) + string_length += strlen(substring_array[i]) + 1; + + result = uds_allocate(string_length, char, __func__, &output); + if (result != VDO_SUCCESS) + return result; + + current_position = &output[0]; + + for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++) { + current_position = uds_append_to_buffer(current_position, + output + string_length, "%s", + substring_array[i]); + *current_position = separator; + current_position++; + } + + /* We output one too many separators; replace the last with a zero byte. */ + if (current_position != output) + *(current_position - 1) = '\0'; + + *string_ptr = output; + return UDS_SUCCESS; +} + +/** + * parse_bool() - Parse a two-valued option into a bool. + * @bool_str: The string value to convert to a bool. + * @true_str: The string value which should be converted to true. + * @false_str: The string value which should be converted to false. + * @bool_ptr: A pointer to return the bool value in. + * + * Return: VDO_SUCCESS or an error if bool_str is neither true_str nor false_str. + */ +static inline int __must_check parse_bool(const char *bool_str, const char *true_str, + const char *false_str, bool *bool_ptr) +{ + bool value = false; + + if (strcmp(bool_str, true_str) == 0) + value = true; + else if (strcmp(bool_str, false_str) == 0) + value = false; + else + return VDO_BAD_CONFIGURATION; + + *bool_ptr = value; + return VDO_SUCCESS; +} + +/** + * process_one_thread_config_spec() - Process one component of a thread parameter configuration + * string and update the configuration data structure. + * @thread_param_type: The type of thread specified. + * @count: The thread count requested. + * @config: The configuration data structure to update. + * + * If the thread count requested is invalid, a message is logged and -EINVAL returned. If the + * thread name is unknown, a message is logged but no error is returned. + * + * Return: VDO_SUCCESS or -EINVAL + */ +static int process_one_thread_config_spec(const char *thread_param_type, + unsigned int count, + struct thread_count_config *config) +{ + /* Handle limited thread parameters */ + if (strcmp(thread_param_type, "bioRotationInterval") == 0) { + if (count == 0) { + uds_log_error("thread config string error: 'bioRotationInterval' of at least 1 is required"); + return -EINVAL; + } else if (count > VDO_BIO_ROTATION_INTERVAL_LIMIT) { + uds_log_error("thread config string error: 'bioRotationInterval' cannot be higher than %d", + VDO_BIO_ROTATION_INTERVAL_LIMIT); + return -EINVAL; + } + config->bio_rotation_interval = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "logical") == 0) { + if (count > MAX_VDO_LOGICAL_ZONES) { + uds_log_error("thread config string error: at most %d 'logical' threads are allowed", + MAX_VDO_LOGICAL_ZONES); + return -EINVAL; + } + config->logical_zones = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "physical") == 0) { + if (count > MAX_VDO_PHYSICAL_ZONES) { + uds_log_error("thread config string error: at most %d 'physical' threads are allowed", + MAX_VDO_PHYSICAL_ZONES); + return -EINVAL; + } + config->physical_zones = count; + return VDO_SUCCESS; + } + /* Handle other thread count parameters */ + if (count > MAXIMUM_VDO_THREADS) { + uds_log_error("thread config string error: at most %d '%s' threads are allowed", + MAXIMUM_VDO_THREADS, thread_param_type); + return -EINVAL; + } + if (strcmp(thread_param_type, "hash") == 0) { + config->hash_zones = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "cpu") == 0) { + if (count == 0) { + uds_log_error("thread config string error: at least one 'cpu' thread required"); + return -EINVAL; + } + config->cpu_threads = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "ack") == 0) { + config->bio_ack_threads = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "bio") == 0) { + if (count == 0) { + uds_log_error("thread config string error: at least one 'bio' thread required"); + return -EINVAL; + } + config->bio_threads = count; + return VDO_SUCCESS; + } + + /* + * Don't fail, just log. This will handle version mismatches between user mode tools and + * kernel. + */ + uds_log_info("unknown thread parameter type \"%s\"", thread_param_type); + return VDO_SUCCESS; +} + +/** + * parse_one_thread_config_spec() - Parse one component of a thread parameter configuration string + * and update the configuration data structure. + * @spec: The thread parameter specification string. + * @config: The configuration data to be updated. + */ +static int parse_one_thread_config_spec(const char *spec, + struct thread_count_config *config) +{ + unsigned int count; + char **fields; + int result; + + result = split_string(spec, '=', &fields); + if (result != UDS_SUCCESS) + return result; + + if ((fields[0] == NULL) || (fields[1] == NULL) || (fields[2] != NULL)) { + uds_log_error("thread config string error: expected thread parameter assignment, saw \"%s\"", + spec); + free_string_array(fields); + return -EINVAL; + } + + result = kstrtouint(fields[1], 10, &count); + if (result != UDS_SUCCESS) { + uds_log_error("thread config string error: integer value needed, found \"%s\"", + fields[1]); + free_string_array(fields); + return result; + } + + result = process_one_thread_config_spec(fields[0], count, config); + free_string_array(fields); + return result; +} + +/** + * parse_thread_config_string() - Parse the configuration string passed and update the specified + * counts and other parameters of various types of threads to be + * created. + * @string: Thread parameter configuration string. + * @config: The thread configuration data to update. + * + * The configuration string should contain one or more comma-separated specs of the form + * "typename=number"; the supported type names are "cpu", "ack", "bio", "bioRotationInterval", + * "logical", "physical", and "hash". + * + * If an error occurs during parsing of a single key/value pair, we deem it serious enough to stop + * further parsing. + * + * This function can't set the "reason" value the caller wants to pass back, because we'd want to + * format it to say which field was invalid, and we can't allocate the "reason" strings + * dynamically. So if an error occurs, we'll log the details and pass back an error. + * + * Return: VDO_SUCCESS or -EINVAL or -ENOMEM + */ +static int parse_thread_config_string(const char *string, + struct thread_count_config *config) +{ + int result = VDO_SUCCESS; + char **specs; + + if (strcmp(".", string) != 0) { + unsigned int i; + + result = split_string(string, ',', &specs); + if (result != UDS_SUCCESS) + return result; + + for (i = 0; specs[i] != NULL; i++) { + result = parse_one_thread_config_spec(specs[i], config); + if (result != VDO_SUCCESS) + break; + } + free_string_array(specs); + } + return result; +} + +/** + * process_one_key_value_pair() - Process one component of an optional parameter string and update + * the configuration data structure. + * @key: The optional parameter key name. + * @value: The optional parameter value. + * @config: The configuration data structure to update. + * + * If the value requested is invalid, a message is logged and -EINVAL returned. If the key is + * unknown, a message is logged but no error is returned. + * + * Return: VDO_SUCCESS or -EINVAL + */ +static int process_one_key_value_pair(const char *key, unsigned int value, + struct device_config *config) +{ + /* Non thread optional parameters */ + if (strcmp(key, "maxDiscard") == 0) { + if (value == 0) { + uds_log_error("optional parameter error: at least one max discard block required"); + return -EINVAL; + } + /* Max discard sectors in blkdev_issue_discard is UINT_MAX >> 9 */ + if (value > (UINT_MAX / VDO_BLOCK_SIZE)) { + uds_log_error("optional parameter error: at most %d max discard blocks are allowed", + UINT_MAX / VDO_BLOCK_SIZE); + return -EINVAL; + } + config->max_discard_blocks = value; + return VDO_SUCCESS; + } + /* Handles unknown key names */ + return process_one_thread_config_spec(key, value, &config->thread_counts); +} + +/** + * parse_one_key_value_pair() - Parse one key/value pair and update the configuration data + * structure. + * @key: The optional key name. + * @value: The optional value. + * @config: The configuration data to be updated. + * + * Return: VDO_SUCCESS or error. + */ +static int parse_one_key_value_pair(const char *key, const char *value, + struct device_config *config) +{ + unsigned int count; + int result; + + if (strcmp(key, "deduplication") == 0) + return parse_bool(value, "on", "off", &config->deduplication); + + if (strcmp(key, "compression") == 0) + return parse_bool(value, "on", "off", &config->compression); + + /* The remaining arguments must have integral values. */ + result = kstrtouint(value, 10, &count); + if (result != UDS_SUCCESS) { + uds_log_error("optional config string error: integer value needed, found \"%s\"", + value); + return result; + } + return process_one_key_value_pair(key, count, config); +} + +/** + * parse_key_value_pairs() - Parse all key/value pairs from a list of arguments. + * @argc: The total number of arguments in list. + * @argv: The list of key/value pairs. + * @config: The device configuration data to update. + * + * If an error occurs during parsing of a single key/value pair, we deem it serious enough to stop + * further parsing. + * + * This function can't set the "reason" value the caller wants to pass back, because we'd want to + * format it to say which field was invalid, and we can't allocate the "reason" strings + * dynamically. So if an error occurs, we'll log the details and return the error. + * + * Return: VDO_SUCCESS or error + */ +static int parse_key_value_pairs(int argc, char **argv, struct device_config *config) +{ + int result = VDO_SUCCESS; + + while (argc) { + result = parse_one_key_value_pair(argv[0], argv[1], config); + if (result != VDO_SUCCESS) + break; + + argc -= 2; + argv += 2; + } + + return result; +} + +/** + * parse_optional_arguments() - Parse the configuration string passed in for optional arguments. + * @arg_set: The structure holding the arguments to parse. + * @error_ptr: Pointer to a buffer to hold the error string. + * @config: Pointer to device configuration data to update. + * + * For V0/V1 configurations, there will only be one optional parameter; the thread configuration. + * The configuration string should contain one or more comma-separated specs of the form + * "typename=number"; the supported type names are "cpu", "ack", "bio", "bioRotationInterval", + * "logical", "physical", and "hash". + * + * For V2 configurations and beyond, there could be any number of arguments. They should contain + * one or more key/value pairs separated by a space. + * + * Return: VDO_SUCCESS or error + */ +static int parse_optional_arguments(struct dm_arg_set *arg_set, char **error_ptr, + struct device_config *config) +{ + int result = VDO_SUCCESS; + + if (config->version == 0 || config->version == 1) { + result = parse_thread_config_string(arg_set->argv[0], + &config->thread_counts); + if (result != VDO_SUCCESS) { + *error_ptr = "Invalid thread-count configuration"; + return VDO_BAD_CONFIGURATION; + } + } else { + if ((arg_set->argc % 2) != 0) { + *error_ptr = "Odd number of optional arguments given but they should be pairs"; + return VDO_BAD_CONFIGURATION; + } + result = parse_key_value_pairs(arg_set->argc, arg_set->argv, config); + if (result != VDO_SUCCESS) { + *error_ptr = "Invalid optional argument configuration"; + return VDO_BAD_CONFIGURATION; + } + } + return result; +} + +/** + * handle_parse_error() - Handle a parsing error. + * @config: The config to free. + * @error_ptr: A place to store a constant string about the error. + * @error_str: A constant string to store in error_ptr. + */ +static void handle_parse_error(struct device_config *config, char **error_ptr, + char *error_str) +{ + free_device_config(config); + *error_ptr = error_str; +} + +/** + * parse_device_config() - Convert the dmsetup table into a struct device_config. + * @argc: The number of table values. + * @argv: The array of table values. + * @ti: The target structure for this table. + * @config_ptr: A pointer to return the allocated config. + * + * Return: VDO_SUCCESS or an error code. + */ +static int parse_device_config(int argc, char **argv, struct dm_target *ti, + struct device_config **config_ptr) +{ + bool enable_512e; + size_t logical_bytes = to_bytes(ti->len); + struct dm_arg_set arg_set; + char **error_ptr = &ti->error; + struct device_config *config = NULL; + int result; + + if ((logical_bytes % VDO_BLOCK_SIZE) != 0) { + handle_parse_error(config, error_ptr, + "Logical size must be a multiple of 4096"); + return VDO_BAD_CONFIGURATION; + } + + if (argc == 0) { + handle_parse_error(config, error_ptr, "Incorrect number of arguments"); + return VDO_BAD_CONFIGURATION; + } + + result = uds_allocate(1, struct device_config, "device_config", &config); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Could not allocate config structure"); + return VDO_BAD_CONFIGURATION; + } + + config->owning_target = ti; + config->logical_blocks = logical_bytes / VDO_BLOCK_SIZE; + INIT_LIST_HEAD(&config->config_list); + + /* Save the original string. */ + result = join_strings(argv, argc, ' ', &config->original_string); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, "Could not populate string"); + return VDO_BAD_CONFIGURATION; + } + + uds_log_info("table line: %s", config->original_string); + + config->thread_counts = (struct thread_count_config) { + .bio_ack_threads = 1, + .bio_threads = DEFAULT_VDO_BIO_SUBMIT_QUEUE_COUNT, + .bio_rotation_interval = DEFAULT_VDO_BIO_SUBMIT_QUEUE_ROTATE_INTERVAL, + .cpu_threads = 1, + .logical_zones = 0, + .physical_zones = 0, + .hash_zones = 0, + }; + config->max_discard_blocks = 1; + config->deduplication = true; + config->compression = false; + + arg_set.argc = argc; + arg_set.argv = argv; + + result = get_version_number(argc, argv, error_ptr, &config->version); + if (result != VDO_SUCCESS) { + /* get_version_number sets error_ptr itself. */ + handle_parse_error(config, error_ptr, *error_ptr); + return result; + } + /* Move the arg pointer forward only if the argument was there. */ + if (config->version >= 1) + dm_shift_arg(&arg_set); + + result = uds_duplicate_string(dm_shift_arg(&arg_set), "parent device name", + &config->parent_device_name); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Could not copy parent device name"); + return VDO_BAD_CONFIGURATION; + } + + /* Get the physical blocks, if known. */ + if (config->version >= 1) { + result = kstrtoull(dm_shift_arg(&arg_set), 10, &config->physical_blocks); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Invalid physical block count"); + return VDO_BAD_CONFIGURATION; + } + } + + /* Get the logical block size and validate */ + result = parse_bool(dm_shift_arg(&arg_set), "512", "4096", &enable_512e); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, "Invalid logical block size"); + return VDO_BAD_CONFIGURATION; + } + config->logical_block_size = (enable_512e ? 512 : 4096); + + /* Skip past the two no longer used read cache options. */ + if (config->version <= 1) + dm_consume_args(&arg_set, 2); + + /* Get the page cache size. */ + result = kstrtouint(dm_shift_arg(&arg_set), 10, &config->cache_size); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Invalid block map page cache size"); + return VDO_BAD_CONFIGURATION; + } + + /* Get the block map era length. */ + result = kstrtouint(dm_shift_arg(&arg_set), 10, &config->block_map_maximum_age); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, "Invalid block map maximum age"); + return VDO_BAD_CONFIGURATION; + } + + /* Skip past the no longer used MD RAID5 optimization mode */ + if (config->version <= 2) + dm_consume_args(&arg_set, 1); + + /* Skip past the no longer used write policy setting */ + if (config->version <= 3) + dm_consume_args(&arg_set, 1); + + /* Skip past the no longer used pool name for older table lines */ + if (config->version <= 2) { + /* + * Make sure the enum to get the pool name from argv directly is still in sync with + * the parsing of the table line. + */ + if (&arg_set.argv[0] != &argv[POOL_NAME_ARG_INDEX[config->version]]) { + handle_parse_error(config, error_ptr, + "Pool name not in expected location"); + return VDO_BAD_CONFIGURATION; + } + dm_shift_arg(&arg_set); + } + + /* Get the optional arguments and validate. */ + result = parse_optional_arguments(&arg_set, error_ptr, config); + if (result != VDO_SUCCESS) { + /* parse_optional_arguments sets error_ptr itself. */ + handle_parse_error(config, error_ptr, *error_ptr); + return result; + } + + /* + * Logical, physical, and hash zone counts can all be zero; then we get one thread doing + * everything, our older configuration. If any zone count is non-zero, the others must be + * as well. + */ + if (((config->thread_counts.logical_zones == 0) != + (config->thread_counts.physical_zones == 0)) || + ((config->thread_counts.physical_zones == 0) != + (config->thread_counts.hash_zones == 0))) { + handle_parse_error(config, error_ptr, + "Logical, physical, and hash zones counts must all be zero or all non-zero"); + return VDO_BAD_CONFIGURATION; + } + + if (config->cache_size < + (2 * MAXIMUM_VDO_USER_VIOS * config->thread_counts.logical_zones)) { + handle_parse_error(config, error_ptr, + "Insufficient block map cache for logical zones"); + return VDO_BAD_CONFIGURATION; + } + + result = dm_get_device(ti, config->parent_device_name, + dm_table_get_mode(ti->table), &config->owned_device); + if (result != 0) { + uds_log_error("couldn't open device \"%s\": error %d", + config->parent_device_name, result); + handle_parse_error(config, error_ptr, "Unable to open storage device"); + return VDO_BAD_CONFIGURATION; + } + + if (config->version == 0) { + u64 device_size = i_size_read(config->owned_device->bdev->bd_inode); + + config->physical_blocks = device_size / VDO_BLOCK_SIZE; + } + + *config_ptr = config; + return result; +} + +static struct vdo *get_vdo_for_target(struct dm_target *ti) +{ + return ((struct device_config *) ti->private)->vdo; +} + + +static int vdo_map_bio(struct dm_target *ti, struct bio *bio) +{ + struct vdo *vdo = get_vdo_for_target(ti); + struct vdo_work_queue *current_work_queue; + const struct admin_state_code *code = vdo_get_admin_state_code(&vdo->admin.state); + + ASSERT_LOG_ONLY(code->normal, "vdo should not receive bios while in state %s", + code->name); + + /* Count all incoming bios. */ + vdo_count_bios(&vdo->stats.bios_in, bio); + + + /* Handle empty bios. Empty flush bios are not associated with a vio. */ + if ((bio_op(bio) == REQ_OP_FLUSH) || ((bio->bi_opf & REQ_PREFLUSH) != 0)) { + vdo_launch_flush(vdo, bio); + return DM_MAPIO_SUBMITTED; + } + + /* This could deadlock, */ + current_work_queue = vdo_get_current_work_queue(); + BUG_ON((current_work_queue != NULL) && + (vdo == vdo_get_work_queue_owner(current_work_queue)->vdo)); + vdo_launch_bio(vdo->data_vio_pool, bio); + return DM_MAPIO_SUBMITTED; +} + +static void vdo_io_hints(struct dm_target *ti, struct queue_limits *limits) +{ + struct vdo *vdo = get_vdo_for_target(ti); + + limits->logical_block_size = vdo->device_config->logical_block_size; + limits->physical_block_size = VDO_BLOCK_SIZE; + + /* The minimum io size for random io */ + blk_limits_io_min(limits, VDO_BLOCK_SIZE); + /* The optimal io size for streamed/sequential io */ + blk_limits_io_opt(limits, VDO_BLOCK_SIZE); + + /* + * Sets the maximum discard size that will be passed into VDO. This value comes from a + * table line value passed in during dmsetup create. + * + * The value 1024 is the largest usable value on HD systems. A 2048 sector discard on a + * busy HD system takes 31 seconds. We should use a value no higher than 1024, which takes + * 15 to 16 seconds on a busy HD system. + * + * But using large values results in 120 second blocked task warnings in /var/log/kern.log. + * In order to avoid these warnings, we choose to use the smallest reasonable value. See + * VDO-3062 and VDO-3087. + * + * The value is displayed in sysfs, and also used by dm-thin to determine whether to pass + * down discards. The block layer splits large discards on this boundary when this is set. + */ + limits->max_discard_sectors = + (vdo->device_config->max_discard_blocks * VDO_SECTORS_PER_BLOCK); + + /* + * Force discards to not begin or end with a partial block by stating the granularity is + * 4k. + */ + limits->discard_granularity = VDO_BLOCK_SIZE; +} + +static int vdo_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, + void *data) +{ + struct device_config *config = get_vdo_for_target(ti)->device_config; + + return fn(ti, config->owned_device, 0, + config->physical_blocks * VDO_SECTORS_PER_BLOCK, data); +} + +/* + * Status line is: + * + * + */ + +static void vdo_status(struct dm_target *ti, status_type_t status_type, + unsigned int status_flags, char *result, unsigned int maxlen) +{ + struct vdo *vdo = get_vdo_for_target(ti); + struct vdo_statistics *stats; + struct device_config *device_config; + /* N.B.: The DMEMIT macro uses the variables named "sz", "result", "maxlen". */ + int sz = 0; + + switch (status_type) { + case STATUSTYPE_INFO: + /* Report info for dmsetup status */ + mutex_lock(&vdo->stats_mutex); + vdo_fetch_statistics(vdo, &vdo->stats_buffer); + stats = &vdo->stats_buffer; + + DMEMIT("/dev/%pg %s %s %s %s %llu %llu", + vdo_get_backing_device(vdo), stats->mode, + stats->in_recovery_mode ? "recovering" : "-", + vdo_get_dedupe_index_state_name(vdo->hash_zones), + vdo_get_compressing(vdo) ? "online" : "offline", + stats->data_blocks_used + stats->overhead_blocks_used, + stats->physical_blocks); + mutex_unlock(&vdo->stats_mutex); + break; + + case STATUSTYPE_TABLE: + /* Report the string actually specified in the beginning. */ + device_config = (struct device_config *) ti->private; + DMEMIT("%s", device_config->original_string); + break; + + case STATUSTYPE_IMA: + /* FIXME: We ought to be more detailed here, but this is what thin does. */ + *result = '\0'; + break; + } +} + +static block_count_t __must_check get_underlying_device_block_count(const struct vdo *vdo) +{ + return i_size_read(vdo_get_backing_device(vdo)->bd_inode) / VDO_BLOCK_SIZE; +} + +static int __must_check process_vdo_message_locked(struct vdo *vdo, unsigned int argc, + char **argv) +{ + if ((argc == 2) && (strcasecmp(argv[0], "compression") == 0)) { + if (strcasecmp(argv[1], "on") == 0) { + vdo_set_compressing(vdo, true); + return 0; + } + + if (strcasecmp(argv[1], "off") == 0) { + vdo_set_compressing(vdo, false); + return 0; + } + + uds_log_warning("invalid argument '%s' to dmsetup compression message", + argv[1]); + return -EINVAL; + } + + uds_log_warning("unrecognized dmsetup message '%s' received", argv[0]); + return -EINVAL; +} + +/* + * If the message is a dump, just do it. Otherwise, check that no other message is being processed, + * and only proceed if so. + * Returns -EBUSY if another message is being processed + */ +static int __must_check process_vdo_message(struct vdo *vdo, unsigned int argc, + char **argv) +{ + int result; + + /* + * All messages which may be processed in parallel with other messages should be handled + * here before the atomic check below. Messages which should be exclusive should be + * processed in process_vdo_message_locked(). + */ + + /* Dump messages should always be processed */ + if (strcasecmp(argv[0], "dump") == 0) + return vdo_dump(vdo, argc, argv, "dmsetup message"); + + if (argc == 1) { + if (strcasecmp(argv[0], "dump-on-shutdown") == 0) { + vdo->dump_on_shutdown = true; + return 0; + } + + /* Index messages should always be processed */ + if ((strcasecmp(argv[0], "index-close") == 0) || + (strcasecmp(argv[0], "index-create") == 0) || + (strcasecmp(argv[0], "index-disable") == 0) || + (strcasecmp(argv[0], "index-enable") == 0)) + return vdo_message_dedupe_index(vdo->hash_zones, argv[0]); + } + + if (atomic_cmpxchg(&vdo->processing_message, 0, 1) != 0) + return -EBUSY; + + result = process_vdo_message_locked(vdo, argc, argv); + + /* Pairs with the implicit barrier in cmpxchg just above */ + smp_wmb(); + atomic_set(&vdo->processing_message, 0); + return result; +} + +static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, + char *result_buffer, unsigned int maxlen) +{ + struct registered_thread allocating_thread, instance_thread; + struct vdo *vdo; + int result; + + if (argc == 0) { + uds_log_warning("unspecified dmsetup message"); + return -EINVAL; + } + + vdo = get_vdo_for_target(ti); + uds_register_allocating_thread(&allocating_thread, NULL); + uds_register_thread_device_id(&instance_thread, &vdo->instance); + + /* + * Must be done here so we don't map return codes. The code in dm-ioctl expects a 1 for a + * return code to look at the buffer and see if it is full or not. + */ + if ((argc == 1) && (strcasecmp(argv[0], "stats") == 0)) { + vdo_write_stats(vdo, result_buffer, maxlen); + result = 1; + } else { + result = vdo_map_to_system_error(process_vdo_message(vdo, argc, argv)); + } + + uds_unregister_thread_device_id(); + uds_unregister_allocating_thread(); + return result; +} + +static void configure_target_capabilities(struct dm_target *ti) +{ + ti->discards_supported = 1; + ti->flush_supported = true; + ti->num_discard_bios = 1; + ti->num_flush_bios = 1; + + /* + * If this value changes, please make sure to update the value for max_discard_sectors + * accordingly. + */ + BUG_ON(dm_set_target_max_io_len(ti, VDO_SECTORS_PER_BLOCK) != 0); +} + +/* + * Implements vdo_filter_fn. + */ +static bool vdo_uses_device(struct vdo *vdo, const void *context) +{ + const struct device_config *config = context; + + return vdo_get_backing_device(vdo)->bd_dev == config->owned_device->bdev->bd_dev; +} + +/** + * get_thread_id_for_phase() - Get the thread id for the current phase of the admin operation in + * progress. + */ +static thread_id_t __must_check get_thread_id_for_phase(struct vdo *vdo) +{ + switch (vdo->admin.phase) { + case RESUME_PHASE_PACKER: + case RESUME_PHASE_FLUSHER: + case SUSPEND_PHASE_PACKER: + case SUSPEND_PHASE_FLUSHES: + return vdo->thread_config.packer_thread; + + case RESUME_PHASE_DATA_VIOS: + case SUSPEND_PHASE_DATA_VIOS: + return vdo->thread_config.cpu_thread; + + case LOAD_PHASE_DRAIN_JOURNAL: + case RESUME_PHASE_JOURNAL: + case SUSPEND_PHASE_JOURNAL: + return vdo->thread_config.journal_thread; + + default: + return vdo->thread_config.admin_thread; + } +} + +static struct vdo_completion *prepare_admin_completion(struct vdo *vdo, + vdo_action_fn callback, + vdo_action_fn error_handler) +{ + struct vdo_completion *completion = &vdo->admin.completion; + + /* + * We can't use vdo_prepare_completion_for_requeue() here because we don't want to reset + * any error in the completion. + */ + completion->callback = callback; + completion->error_handler = error_handler; + completion->callback_thread_id = get_thread_id_for_phase(vdo); + completion->requeue = true; + return completion; +} + +/** + * advance_phase() - Increment the phase of the current admin operation and prepare the admin + * completion to run on the thread for the next phase. + * @vdo: The on which an admin operation is being performed + * + * Return: The current phase + */ +static u32 advance_phase(struct vdo *vdo) +{ + u32 phase = vdo->admin.phase++; + + vdo->admin.completion.callback_thread_id = get_thread_id_for_phase(vdo); + vdo->admin.completion.requeue = true; + return phase; +} + +/* + * Perform an administrative operation (load, suspend, grow logical, or grow physical). This method + * should not be called from vdo threads. + */ +static int perform_admin_operation(struct vdo *vdo, u32 starting_phase, + vdo_action_fn callback, vdo_action_fn error_handler, + const char *type) +{ + int result; + struct vdo_administrator *admin = &vdo->admin; + + if (atomic_cmpxchg(&admin->busy, 0, 1) != 0) { + return uds_log_error_strerror(VDO_COMPONENT_BUSY, + "Can't start %s operation, another operation is already in progress", + type); + } + + admin->phase = starting_phase; + reinit_completion(&admin->callback_sync); + vdo_reset_completion(&admin->completion); + vdo_launch_completion(prepare_admin_completion(vdo, callback, error_handler)); + + /* + * Using the "interruptible" interface means that Linux will not log a message when we wait + * for more than 120 seconds. + */ + while (wait_for_completion_interruptible(&admin->callback_sync) != 0) + /* * However, if we get a signal in a user-mode process, we could spin... */ + fsleep(1000); + + result = admin->completion.result; + /* pairs with implicit barrier in cmpxchg above */ + smp_wmb(); + atomic_set(&admin->busy, 0); + return result; +} + +/* Assert that we are operating on the correct thread for the current phase. */ +static void assert_admin_phase_thread(struct vdo *vdo, const char *what) +{ + ASSERT_LOG_ONLY(vdo_get_callback_thread_id() == get_thread_id_for_phase(vdo), + "%s on correct thread for %s", what, + ADMIN_PHASE_NAMES[vdo->admin.phase]); +} + +/** + * finish_operation_callback() - Callback to finish an admin operation. + * @completion: The admin_completion. + */ +static void finish_operation_callback(struct vdo_completion *completion) +{ + struct vdo_administrator *admin = &completion->vdo->admin; + + vdo_finish_operation(&admin->state, completion->result); + complete(&admin->callback_sync); +} + +/** + * decode_from_super_block() - Decode the VDO state from the super block and validate that it is + * correct. + * @vdo: The vdo being loaded. + * + * On error from this method, the component states must be destroyed explicitly. If this method + * returns successfully, the component states must not be destroyed. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check decode_from_super_block(struct vdo *vdo) +{ + const struct device_config *config = vdo->device_config; + int result; + + result = vdo_decode_component_states(vdo->super_block.buffer, &vdo->geometry, + &vdo->states); + if (result != VDO_SUCCESS) + return result; + + vdo_set_state(vdo, vdo->states.vdo.state); + vdo->load_state = vdo->states.vdo.state; + + /* + * If the device config specifies a larger logical size than was recorded in the super + * block, just accept it. + */ + if (vdo->states.vdo.config.logical_blocks < config->logical_blocks) { + uds_log_warning("Growing logical size: a logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", + (unsigned long long) config->logical_blocks, + (unsigned long long) vdo->states.vdo.config.logical_blocks); + vdo->states.vdo.config.logical_blocks = config->logical_blocks; + } + + result = vdo_validate_component_states(&vdo->states, vdo->geometry.nonce, + config->physical_blocks, + config->logical_blocks); + if (result != VDO_SUCCESS) + return result; + + vdo->layout = vdo->states.layout; + return VDO_SUCCESS; +} + +/** + * decode_vdo() - Decode the component data portion of a super block and fill in the corresponding + * portions of the vdo being loaded. + * @vdo: The vdo being loaded. + * + * This will also allocate the recovery journal and slab depot. If this method is called with an + * asynchronous layer (i.e. a thread config which specifies at least one base thread), the block + * map and packer will be constructed as well. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check decode_vdo(struct vdo *vdo) +{ + block_count_t maximum_age, journal_length; + struct partition *partition; + int result; + + result = decode_from_super_block(vdo); + if (result != VDO_SUCCESS) { + vdo_destroy_component_states(&vdo->states); + return result; + } + + maximum_age = vdo_convert_maximum_age(vdo->device_config->block_map_maximum_age); + journal_length = + vdo_get_recovery_journal_length(vdo->states.vdo.config.recovery_journal_size); + if (maximum_age > (journal_length / 2)) { + return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + "maximum age: %llu exceeds limit %llu", + (unsigned long long) maximum_age, + (unsigned long long) (journal_length / 2)); + } + + if (maximum_age == 0) { + return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + "maximum age must be greater than 0"); + } + + result = vdo_enable_read_only_entry(vdo); + if (result != VDO_SUCCESS) + return result; + + partition = vdo_get_known_partition(&vdo->layout, + VDO_RECOVERY_JOURNAL_PARTITION); + result = vdo_decode_recovery_journal(vdo->states.recovery_journal, + vdo->states.vdo.nonce, vdo, partition, + vdo->states.vdo.complete_recoveries, + vdo->states.vdo.config.recovery_journal_size, + &vdo->recovery_journal); + if (result != VDO_SUCCESS) + return result; + + partition = vdo_get_known_partition(&vdo->layout, VDO_SLAB_SUMMARY_PARTITION); + result = vdo_decode_slab_depot(vdo->states.slab_depot, vdo, partition, + &vdo->depot); + if (result != VDO_SUCCESS) + return result; + + result = vdo_decode_block_map(vdo->states.block_map, + vdo->states.vdo.config.logical_blocks, vdo, + vdo->recovery_journal, vdo->states.vdo.nonce, + vdo->device_config->cache_size, maximum_age, + &vdo->block_map); + if (result != VDO_SUCCESS) + return result; + + result = vdo_make_physical_zones(vdo, &vdo->physical_zones); + if (result != VDO_SUCCESS) + return result; + + /* The logical zones depend on the physical zones already existing. */ + result = vdo_make_logical_zones(vdo, &vdo->logical_zones); + if (result != VDO_SUCCESS) + return result; + + return vdo_make_hash_zones(vdo, &vdo->hash_zones); +} + +/** + * pre_load_callback() - Callback to initiate a pre-load, registered in vdo_initialize(). + * @completion: The admin completion. + */ +static void pre_load_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case PRE_LOAD_PHASE_START: + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_PRE_LOADING); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + vdo_load_super_block(vdo, completion); + return; + + case PRE_LOAD_PHASE_LOAD_COMPONENTS: + vdo_continue_completion(completion, decode_vdo(vdo)); + return; + + case PRE_LOAD_PHASE_END: + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +static void release_instance(unsigned int instance) +{ + mutex_lock(&instances_lock); + if (instance >= instances.bit_count) { + ASSERT_LOG_ONLY(false, + "instance number %u must be less than bit count %u", + instance, instances.bit_count); + } else if (test_bit(instance, instances.words) == 0) { + ASSERT_LOG_ONLY(false, "instance number %u must be allocated", instance); + } else { + __clear_bit(instance, instances.words); + instances.count -= 1; + } + mutex_unlock(&instances_lock); +} + +static void set_device_config(struct dm_target *ti, struct vdo *vdo, + struct device_config *config) +{ + list_del_init(&config->config_list); + list_add_tail(&config->config_list, &vdo->device_config_list); + config->vdo = vdo; + ti->private = config; + configure_target_capabilities(ti); +} + +static int vdo_initialize(struct dm_target *ti, unsigned int instance, + struct device_config *config) +{ + struct vdo *vdo; + int result; + u64 block_size = VDO_BLOCK_SIZE; + u64 logical_size = to_bytes(ti->len); + block_count_t logical_blocks = logical_size / block_size; + + uds_log_info("loading device '%s'", vdo_get_device_name(ti)); + uds_log_debug("Logical block size = %llu", (u64) config->logical_block_size); + uds_log_debug("Logical blocks = %llu", logical_blocks); + uds_log_debug("Physical block size = %llu", (u64) block_size); + uds_log_debug("Physical blocks = %llu", config->physical_blocks); + uds_log_debug("Block map cache blocks = %u", config->cache_size); + uds_log_debug("Block map maximum age = %u", config->block_map_maximum_age); + uds_log_debug("Deduplication = %s", (config->deduplication ? "on" : "off")); + uds_log_debug("Compression = %s", (config->compression ? "on" : "off")); + + vdo = vdo_find_matching(vdo_uses_device, config); + if (vdo != NULL) { + uds_log_error("Existing vdo already uses device %s", + vdo->device_config->parent_device_name); + ti->error = "Cannot share storage device with already-running VDO"; + return VDO_BAD_CONFIGURATION; + } + + result = vdo_make(instance, config, &ti->error, &vdo); + if (result != VDO_SUCCESS) { + uds_log_error("Could not create VDO device. (VDO error %d, message %s)", + result, ti->error); + vdo_destroy(vdo); + return result; + } + + result = perform_admin_operation(vdo, PRE_LOAD_PHASE_START, pre_load_callback, + finish_operation_callback, "pre-load"); + if (result != VDO_SUCCESS) { + ti->error = ((result == VDO_INVALID_ADMIN_STATE) ? + "Pre-load is only valid immediately after initialization" : + "Cannot load metadata from device"); + uds_log_error("Could not start VDO device. (VDO error %d, message %s)", + result, ti->error); + vdo_destroy(vdo); + return result; + } + + set_device_config(ti, vdo, config); + vdo->device_config = config; + return VDO_SUCCESS; +} + +/* Implements vdo_filter_fn. */ +static bool __must_check vdo_is_named(struct vdo *vdo, const void *context) +{ + struct dm_target *ti = vdo->device_config->owning_target; + const char *device_name = vdo_get_device_name(ti); + + return strcmp(device_name, context) == 0; +} + +/** + * get_bit_array_size() - Return the number of bytes needed to store a bit array of the specified + * capacity in an array of unsigned longs. + * @bit_count: The number of bits the array must hold. + * + * Return: the number of bytes needed for the array representation. + */ +static size_t get_bit_array_size(unsigned int bit_count) +{ + /* Round up to a multiple of the word size and convert to a byte count. */ + return (BITS_TO_LONGS(bit_count) * sizeof(unsigned long)); +} + +/** + * grow_bit_array() - Re-allocate the bitmap word array so there will more instance numbers that + * can be allocated. + * + * Since the array is initially NULL, this also initializes the array the first time we allocate an + * instance number. + * + * Return: UDS_SUCCESS or an error code from the allocation + */ +static int grow_bit_array(void) +{ + unsigned int new_count = max(instances.bit_count + BIT_COUNT_INCREMENT, + (unsigned int) BIT_COUNT_MINIMUM); + unsigned long *new_words; + int result; + + result = uds_reallocate_memory(instances.words, + get_bit_array_size(instances.bit_count), + get_bit_array_size(new_count), + "instance number bit array", &new_words); + if (result != UDS_SUCCESS) + return result; + + instances.bit_count = new_count; + instances.words = new_words; + return UDS_SUCCESS; +} + +/** + * allocate_instance() - Allocate an instance number. + * @instance_ptr: A point to hold the instance number + * + * Return: UDS_SUCCESS or an error code + * + * This function must be called while holding the instances lock. + */ +static int allocate_instance(unsigned int *instance_ptr) +{ + unsigned int instance; + int result; + + /* If there are no unallocated instances, grow the bit array. */ + if (instances.count >= instances.bit_count) { + result = grow_bit_array(); + if (result != UDS_SUCCESS) + return result; + } + + /* + * There must be a zero bit somewhere now. Find it, starting just after the last instance + * allocated. + */ + instance = find_next_zero_bit(instances.words, instances.bit_count, + instances.next); + if (instance >= instances.bit_count) { + /* Nothing free after next, so wrap around to instance zero. */ + instance = find_first_zero_bit(instances.words, instances.bit_count); + result = ASSERT(instance < instances.bit_count, + "impossibly, no zero bit found"); + if (result != UDS_SUCCESS) + return result; + } + + __set_bit(instance, instances.words); + instances.count++; + instances.next = instance + 1; + *instance_ptr = instance; + return UDS_SUCCESS; +} + +static int construct_new_vdo_registered(struct dm_target *ti, unsigned int argc, + char **argv, unsigned int instance) +{ + int result; + struct device_config *config; + + result = parse_device_config(argc, argv, ti, &config); + if (result != VDO_SUCCESS) { + uds_log_error_strerror(result, "parsing failed: %s", ti->error); + release_instance(instance); + return -EINVAL; + } + + /* Beyond this point, the instance number will be cleaned up for us if needed */ + result = vdo_initialize(ti, instance, config); + if (result != VDO_SUCCESS) { + release_instance(instance); + free_device_config(config); + return vdo_map_to_system_error(result); + } + + return VDO_SUCCESS; +} + +static int construct_new_vdo(struct dm_target *ti, unsigned int argc, char **argv) +{ + int result; + unsigned int instance; + struct registered_thread instance_thread; + + mutex_lock(&instances_lock); + result = allocate_instance(&instance); + mutex_unlock(&instances_lock); + if (result != VDO_SUCCESS) + return -ENOMEM; + + uds_register_thread_device_id(&instance_thread, &instance); + result = construct_new_vdo_registered(ti, argc, argv, instance); + uds_unregister_thread_device_id(); + return result; +} + +/** + * check_may_grow_physical() - Callback to check that we're not in recovery mode, used in + * vdo_prepare_to_grow_physical(). + * @completion: The admin completion. + */ +static void check_may_grow_physical(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + assert_admin_phase_thread(vdo, __func__); + + /* These checks can only be done from a vdo thread. */ + if (vdo_is_read_only(vdo)) + vdo_set_completion_result(completion, VDO_READ_ONLY); + + if (vdo_in_recovery_mode(vdo)) + vdo_set_completion_result(completion, VDO_RETRY_AFTER_REBUILD); + + finish_operation_callback(completion); +} + +static block_count_t get_partition_size(struct layout *layout, enum partition_id id) +{ + return vdo_get_known_partition(layout, id)->count; +} + +/** + * grow_layout() - Make the layout for growing a vdo. + * @vdo: The vdo preparing to grow. + * @old_size: The current size of the vdo. + * @new_size: The size to which the vdo will be grown. + * + * Return: VDO_SUCCESS or an error code. + */ +static int grow_layout(struct vdo *vdo, block_count_t old_size, block_count_t new_size) +{ + int result; + block_count_t min_new_size; + + if (vdo->next_layout.size == new_size) { + /* We are already prepared to grow to the new size, so we're done. */ + return VDO_SUCCESS; + } + + /* Make a copy completion if there isn't one */ + if (vdo->partition_copier == NULL) { + vdo->partition_copier = dm_kcopyd_client_create(NULL); + if (vdo->partition_copier == NULL) + return -ENOMEM; + } + + /* Free any unused preparation. */ + vdo_uninitialize_layout(&vdo->next_layout); + + /* + * Make a new layout with the existing partition sizes for everything but the slab depot + * partition. + */ + result = vdo_initialize_layout(new_size, vdo->layout.start, + get_partition_size(&vdo->layout, + VDO_BLOCK_MAP_PARTITION), + get_partition_size(&vdo->layout, + VDO_RECOVERY_JOURNAL_PARTITION), + get_partition_size(&vdo->layout, + VDO_SLAB_SUMMARY_PARTITION), + &vdo->next_layout); + if (result != VDO_SUCCESS) { + dm_kcopyd_client_destroy(uds_forget(vdo->partition_copier)); + return result; + } + + /* Ensure the new journal and summary are entirely within the added blocks. */ + min_new_size = (old_size + + get_partition_size(&vdo->next_layout, + VDO_SLAB_SUMMARY_PARTITION) + + get_partition_size(&vdo->next_layout, + VDO_RECOVERY_JOURNAL_PARTITION)); + if (min_new_size > new_size) { + /* Copying the journal and summary would destroy some old metadata. */ + vdo_uninitialize_layout(&vdo->next_layout); + dm_kcopyd_client_destroy(uds_forget(vdo->partition_copier)); + return VDO_INCREMENT_TOO_SMALL; + } + + return VDO_SUCCESS; +} + +static int prepare_to_grow_physical(struct vdo *vdo, block_count_t new_physical_blocks) +{ + int result; + block_count_t current_physical_blocks = vdo->states.vdo.config.physical_blocks; + + uds_log_info("Preparing to resize physical to %llu", + (unsigned long long) new_physical_blocks); + ASSERT_LOG_ONLY((new_physical_blocks > current_physical_blocks), + "New physical size is larger than current physical size"); + result = perform_admin_operation(vdo, PREPARE_GROW_PHYSICAL_PHASE_START, + check_may_grow_physical, + finish_operation_callback, + "prepare grow-physical"); + if (result != VDO_SUCCESS) + return result; + + result = grow_layout(vdo, current_physical_blocks, new_physical_blocks); + if (result != VDO_SUCCESS) + return result; + + result = vdo_prepare_to_grow_slab_depot(vdo->depot, + vdo_get_known_partition(&vdo->next_layout, + VDO_SLAB_DEPOT_PARTITION)); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(&vdo->next_layout); + return result; + } + + uds_log_info("Done preparing to resize physical"); + return VDO_SUCCESS; +} + +/** + * validate_new_device_config() - Check whether a new device config represents a valid modification + * to an existing config. + * @to_validate: The new config to validate. + * @config: The existing config. + * @may_grow: Set to true if growing the logical and physical size of the vdo is currently + * permitted. + * @error_ptr: A pointer to hold the reason for any error. + * + * Return: VDO_SUCCESS or an error. + */ +static int validate_new_device_config(struct device_config *to_validate, + struct device_config *config, bool may_grow, + char **error_ptr) +{ + if (to_validate->owning_target->begin != config->owning_target->begin) { + *error_ptr = "Starting sector cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->logical_block_size != config->logical_block_size) { + *error_ptr = "Logical block size cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->logical_blocks < config->logical_blocks) { + *error_ptr = "Can't shrink VDO logical size"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->cache_size != config->cache_size) { + *error_ptr = "Block map cache size cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->block_map_maximum_age != config->block_map_maximum_age) { + *error_ptr = "Block map maximum age cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (memcmp(&to_validate->thread_counts, &config->thread_counts, + sizeof(struct thread_count_config)) != 0) { + *error_ptr = "Thread configuration cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->physical_blocks < config->physical_blocks) { + *error_ptr = "Removing physical storage from a VDO is not supported"; + return VDO_NOT_IMPLEMENTED; + } + + if (!may_grow && (to_validate->physical_blocks > config->physical_blocks)) { + *error_ptr = "VDO physical size may not grow in current state"; + return VDO_NOT_IMPLEMENTED; + } + + return VDO_SUCCESS; +} + +static int prepare_to_modify(struct dm_target *ti, struct device_config *config, + struct vdo *vdo) +{ + int result; + bool may_grow = (vdo_get_admin_state(vdo) != VDO_ADMIN_STATE_PRE_LOADED); + + result = validate_new_device_config(config, vdo->device_config, may_grow, + &ti->error); + if (result != VDO_SUCCESS) + return -EINVAL; + + if (config->logical_blocks > vdo->device_config->logical_blocks) { + block_count_t logical_blocks = vdo->states.vdo.config.logical_blocks; + + uds_log_info("Preparing to resize logical to %llu", + (unsigned long long) config->logical_blocks); + ASSERT_LOG_ONLY((config->logical_blocks > logical_blocks), + "New logical size is larger than current size"); + + result = vdo_prepare_to_grow_block_map(vdo->block_map, + config->logical_blocks); + if (result != VDO_SUCCESS) { + ti->error = "Device vdo_prepare_to_grow_logical failed"; + return result; + } + + uds_log_info("Done preparing to resize logical"); + } + + if (config->physical_blocks > vdo->device_config->physical_blocks) { + result = prepare_to_grow_physical(vdo, config->physical_blocks); + if (result != VDO_SUCCESS) { + if (result == VDO_PARAMETER_MISMATCH) { + /* + * If we don't trap this case, vdo_map_to_system_error() will remap + * it to -EIO, which is misleading and ahistorical. + */ + result = -EINVAL; + } + + if (result == VDO_TOO_MANY_SLABS) + ti->error = "Device vdo_prepare_to_grow_physical failed (specified physical size too big based on formatted slab size)"; + else + ti->error = "Device vdo_prepare_to_grow_physical failed"; + + return result; + } + } + + if (strcmp(config->parent_device_name, vdo->device_config->parent_device_name) != 0) { + const char *device_name = vdo_get_device_name(config->owning_target); + + uds_log_info("Updating backing device of %s from %s to %s", device_name, + vdo->device_config->parent_device_name, + config->parent_device_name); + } + + return VDO_SUCCESS; +} + +static int update_existing_vdo(const char *device_name, struct dm_target *ti, + unsigned int argc, char **argv, struct vdo *vdo) +{ + int result; + struct device_config *config; + + result = parse_device_config(argc, argv, ti, &config); + if (result != VDO_SUCCESS) + return -EINVAL; + + uds_log_info("preparing to modify device '%s'", device_name); + result = prepare_to_modify(ti, config, vdo); + if (result != VDO_SUCCESS) { + free_device_config(config); + return vdo_map_to_system_error(result); + } + + set_device_config(ti, vdo, config); + return VDO_SUCCESS; +} + +static int vdo_ctr(struct dm_target *ti, unsigned int argc, char **argv) +{ + int result; + struct registered_thread allocating_thread, instance_thread; + const char *device_name; + struct vdo *vdo; + + uds_register_allocating_thread(&allocating_thread, NULL); + device_name = vdo_get_device_name(ti); + vdo = vdo_find_matching(vdo_is_named, device_name); + if (vdo == NULL) { + result = construct_new_vdo(ti, argc, argv); + } else { + uds_register_thread_device_id(&instance_thread, &vdo->instance); + result = update_existing_vdo(device_name, ti, argc, argv, vdo); + uds_unregister_thread_device_id(); + } + + uds_unregister_allocating_thread(); + return result; +} + +static void vdo_dtr(struct dm_target *ti) +{ + struct device_config *config = ti->private; + struct vdo *vdo = uds_forget(config->vdo); + + list_del_init(&config->config_list); + if (list_empty(&vdo->device_config_list)) { + const char *device_name; + + /* This was the last config referencing the VDO. Free it. */ + unsigned int instance = vdo->instance; + struct registered_thread allocating_thread, instance_thread; + + uds_register_thread_device_id(&instance_thread, &instance); + uds_register_allocating_thread(&allocating_thread, NULL); + + device_name = vdo_get_device_name(ti); + uds_log_info("stopping device '%s'", device_name); + if (vdo->dump_on_shutdown) + vdo_dump_all(vdo, "device shutdown"); + + vdo_destroy(uds_forget(vdo)); + uds_log_info("device '%s' stopped", device_name); + uds_unregister_thread_device_id(); + uds_unregister_allocating_thread(); + release_instance(instance); + } else if (config == vdo->device_config) { + /* + * The VDO still references this config. Give it a reference to a config that isn't + * being destroyed. + */ + vdo->device_config = list_first_entry(&vdo->device_config_list, + struct device_config, config_list); + } + + free_device_config(config); + ti->private = NULL; +} + +static void vdo_presuspend(struct dm_target *ti) +{ + get_vdo_for_target(ti)->suspend_type = + (dm_noflush_suspending(ti) ? VDO_ADMIN_STATE_SUSPENDING : VDO_ADMIN_STATE_SAVING); +} + +/** + * write_super_block_for_suspend() - Update the VDO state and save the super block. + * @completion: The admin completion + */ +static void write_super_block_for_suspend(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + switch (vdo_get_state(vdo)) { + case VDO_DIRTY: + case VDO_NEW: + vdo_set_state(vdo, VDO_CLEAN); + break; + + case VDO_CLEAN: + case VDO_READ_ONLY_MODE: + case VDO_FORCE_REBUILD: + case VDO_RECOVERING: + case VDO_REBUILD_FOR_UPGRADE: + break; + + case VDO_REPLAYING: + default: + vdo_continue_completion(completion, UDS_BAD_STATE); + return; + } + + vdo_save_components(vdo, completion); +} + +/** + * suspend_callback() - Callback to initiate a suspend, registered in vdo_postsuspend(). + * @completion: The sub-task completion. + */ +static void suspend_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + struct admin_state *state = &vdo->admin.state; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case SUSPEND_PHASE_START: + if (vdo_get_admin_state_code(state)->quiescent) { + /* Already suspended */ + break; + } + + vdo_continue_completion(completion, + vdo_start_operation(state, vdo->suspend_type)); + return; + + case SUSPEND_PHASE_PACKER: + /* + * If the VDO was already resumed from a prior suspend while read-only, some of the + * components may not have been resumed. By setting a read-only error here, we + * guarantee that the result of this suspend will be VDO_READ_ONLY and not + * VDO_INVALID_ADMIN_STATE in that case. + */ + if (vdo_in_read_only_mode(vdo)) + vdo_set_completion_result(completion, VDO_READ_ONLY); + + vdo_drain_packer(vdo->packer, completion); + return; + + case SUSPEND_PHASE_DATA_VIOS: + drain_data_vio_pool(vdo->data_vio_pool, completion); + return; + + case SUSPEND_PHASE_DEDUPE: + vdo_drain_hash_zones(vdo->hash_zones, completion); + return; + + case SUSPEND_PHASE_FLUSHES: + vdo_drain_flusher(vdo->flusher, completion); + return; + + case SUSPEND_PHASE_LOGICAL_ZONES: + /* + * Attempt to flush all I/O before completing post suspend work. We believe a + * suspended device is expected to have persisted all data written before the + * suspend, even if it hasn't been flushed yet. + */ + result = vdo_synchronous_flush(vdo); + if (result != VDO_SUCCESS) + vdo_enter_read_only_mode(vdo, result); + + vdo_drain_logical_zones(vdo->logical_zones, + vdo_get_admin_state_code(state), completion); + return; + + case SUSPEND_PHASE_BLOCK_MAP: + vdo_drain_block_map(vdo->block_map, vdo_get_admin_state_code(state), + completion); + return; + + case SUSPEND_PHASE_JOURNAL: + vdo_drain_recovery_journal(vdo->recovery_journal, + vdo_get_admin_state_code(state), completion); + return; + + case SUSPEND_PHASE_DEPOT: + vdo_drain_slab_depot(vdo->depot, vdo_get_admin_state_code(state), + completion); + return; + + case SUSPEND_PHASE_READ_ONLY_WAIT: + vdo_wait_until_not_entering_read_only_mode(completion); + return; + + case SUSPEND_PHASE_WRITE_SUPER_BLOCK: + if (vdo_is_state_suspending(state) || (completion->result != VDO_SUCCESS)) { + /* If we didn't save the VDO or there was an error, we're done. */ + break; + } + + write_super_block_for_suspend(completion); + return; + + case SUSPEND_PHASE_END: + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +static void vdo_postsuspend(struct dm_target *ti) +{ + struct vdo *vdo = get_vdo_for_target(ti); + struct registered_thread instance_thread; + const char *device_name; + int result; + + uds_register_thread_device_id(&instance_thread, &vdo->instance); + device_name = vdo_get_device_name(vdo->device_config->owning_target); + uds_log_info("suspending device '%s'", device_name); + + /* + * It's important to note any error here does not actually stop device-mapper from + * suspending the device. All this work is done post suspend. + */ + result = perform_admin_operation(vdo, SUSPEND_PHASE_START, suspend_callback, + suspend_callback, "suspend"); + + if ((result == VDO_SUCCESS) || (result == VDO_READ_ONLY)) { + /* + * Treat VDO_READ_ONLY as a success since a read-only suspension still leaves the + * VDO suspended. + */ + uds_log_info("device '%s' suspended", device_name); + } else if (result == VDO_INVALID_ADMIN_STATE) { + uds_log_error("Suspend invoked while in unexpected state: %s", + vdo_get_admin_state(vdo)->name); + } else { + uds_log_error_strerror(result, "Suspend of device '%s' failed", + device_name); + } + + uds_unregister_thread_device_id(); +} + +/** + * was_new() - Check whether the vdo was new when it was loaded. + * @vdo: The vdo to query. + * + * Return: true if the vdo was new. + */ +static bool was_new(const struct vdo *vdo) +{ + return (vdo->load_state == VDO_NEW); +} + +/** + * requires_repair() - Check whether a vdo requires recovery or rebuild. + * @vdo: The vdo to query. + * + * Return: true if the vdo must be repaired. + */ +static bool __must_check requires_repair(const struct vdo *vdo) +{ + switch (vdo_get_state(vdo)) { + case VDO_DIRTY: + case VDO_FORCE_REBUILD: + case VDO_REPLAYING: + case VDO_REBUILD_FOR_UPGRADE: + return true; + + default: + return false; + } +} + +/** + * get_load_type() - Determine how the slab depot was loaded. + * @vdo: The vdo. + * + * Return: How the depot was loaded. + */ +static enum slab_depot_load_type get_load_type(struct vdo *vdo) +{ + if (vdo_state_requires_read_only_rebuild(vdo->load_state)) + return VDO_SLAB_DEPOT_REBUILD_LOAD; + + if (vdo_state_requires_recovery(vdo->load_state)) + return VDO_SLAB_DEPOT_RECOVERY_LOAD; + + return VDO_SLAB_DEPOT_NORMAL_LOAD; +} + +/** + * vdo_initialize_kobjects() - Initialize the vdo sysfs directory. + * @vdo: The vdo being initialized. + * + * Return: VDO_SUCCESS or an error code. + */ +static int vdo_initialize_kobjects(struct vdo *vdo) +{ + int result; + struct dm_target *target = vdo->device_config->owning_target; + struct mapped_device *md = dm_table_get_md(target->table); + + kobject_init(&vdo->vdo_directory, &vdo_directory_type); + vdo->sysfs_added = true; + result = kobject_add(&vdo->vdo_directory, &disk_to_dev(dm_disk(md))->kobj, + "vdo"); + if (result != 0) + return VDO_CANT_ADD_SYSFS_NODE; + + result = vdo_add_dedupe_index_sysfs(vdo->hash_zones); + if (result != 0) + return VDO_CANT_ADD_SYSFS_NODE; + + return vdo_add_sysfs_stats_dir(vdo); +} + +/** + * load_callback() - Callback to do the destructive parts of loading a VDO. + * @completion: The sub-task completion. + */ +static void load_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case LOAD_PHASE_START: + result = vdo_start_operation(&vdo->admin.state, VDO_ADMIN_STATE_LOADING); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + /* Prepare the recovery journal for new entries. */ + vdo_open_recovery_journal(vdo->recovery_journal, vdo->depot, + vdo->block_map); + vdo_allow_read_only_mode_entry(completion); + return; + + case LOAD_PHASE_STATS: + vdo_continue_completion(completion, vdo_initialize_kobjects(vdo)); + return; + + case LOAD_PHASE_LOAD_DEPOT: + if (vdo_is_read_only(vdo)) { + /* + * In read-only mode we don't use the allocator and it may not even be + * readable, so don't bother trying to load it. + */ + vdo_set_completion_result(completion, VDO_READ_ONLY); + break; + } + + if (requires_repair(vdo)) { + vdo_repair(completion); + return; + } + + vdo_load_slab_depot(vdo->depot, + (was_new(vdo) ? VDO_ADMIN_STATE_FORMATTING : + VDO_ADMIN_STATE_LOADING), + completion, NULL); + return; + + case LOAD_PHASE_MAKE_DIRTY: + vdo_set_state(vdo, VDO_DIRTY); + vdo_save_components(vdo, completion); + return; + + case LOAD_PHASE_PREPARE_TO_ALLOCATE: + vdo_initialize_block_map_from_journal(vdo->block_map, + vdo->recovery_journal); + vdo_prepare_slab_depot_to_allocate(vdo->depot, get_load_type(vdo), + completion); + return; + + case LOAD_PHASE_SCRUB_SLABS: + if (vdo_state_requires_recovery(vdo->load_state)) + vdo_enter_recovery_mode(vdo); + + vdo_scrub_all_unrecovered_slabs(vdo->depot, completion); + return; + + case LOAD_PHASE_DATA_REDUCTION: + WRITE_ONCE(vdo->compressing, vdo->device_config->compression); + if (vdo->device_config->deduplication) { + /* + * Don't try to load or rebuild the index first (and log scary error + * messages) if this is known to be a newly-formatted volume. + */ + vdo_start_dedupe_index(vdo->hash_zones, was_new(vdo)); + } + + vdo->allocations_allowed = false; + fallthrough; + + case LOAD_PHASE_FINISHED: + break; + + case LOAD_PHASE_DRAIN_JOURNAL: + vdo_drain_recovery_journal(vdo->recovery_journal, VDO_ADMIN_STATE_SAVING, + completion); + return; + + case LOAD_PHASE_WAIT_FOR_READ_ONLY: + /* Avoid an infinite loop */ + completion->error_handler = NULL; + vdo->admin.phase = LOAD_PHASE_FINISHED; + vdo_wait_until_not_entering_read_only_mode(completion); + return; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +/** + * handle_load_error() - Handle an error during the load operation. + * @completion: The admin completion. + * + * If at all possible, brings the vdo online in read-only mode. This handler is registered in + * vdo_preresume_registered(). + */ +static void handle_load_error(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + if (vdo_requeue_completion_if_needed(completion, + vdo->thread_config.admin_thread)) + return; + + if (vdo_state_requires_read_only_rebuild(vdo->load_state) && + (vdo->admin.phase == LOAD_PHASE_MAKE_DIRTY)) { + uds_log_error_strerror(completion->result, "aborting load"); + vdo->admin.phase = LOAD_PHASE_DRAIN_JOURNAL; + load_callback(uds_forget(completion)); + return; + } + + uds_log_error_strerror(completion->result, + "Entering read-only mode due to load error"); + vdo->admin.phase = LOAD_PHASE_WAIT_FOR_READ_ONLY; + vdo_enter_read_only_mode(vdo, completion->result); + completion->result = VDO_READ_ONLY; + load_callback(completion); +} + +/** + * write_super_block_for_resume() - Update the VDO state and save the super block. + * @completion: The admin completion + */ +static void write_super_block_for_resume(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + switch (vdo_get_state(vdo)) { + case VDO_CLEAN: + case VDO_NEW: + vdo_set_state(vdo, VDO_DIRTY); + vdo_save_components(vdo, completion); + return; + + case VDO_DIRTY: + case VDO_READ_ONLY_MODE: + case VDO_FORCE_REBUILD: + case VDO_RECOVERING: + case VDO_REBUILD_FOR_UPGRADE: + /* No need to write the super block in these cases */ + vdo_launch_completion(completion); + return; + + case VDO_REPLAYING: + default: + vdo_continue_completion(completion, UDS_BAD_STATE); + } +} + +/** + * resume_callback() - Callback to resume a VDO. + * @completion: The admin completion. + */ +static void resume_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case RESUME_PHASE_START: + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_RESUMING); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + write_super_block_for_resume(completion); + return; + + case RESUME_PHASE_ALLOW_READ_ONLY_MODE: + vdo_allow_read_only_mode_entry(completion); + return; + + case RESUME_PHASE_DEDUPE: + vdo_resume_hash_zones(vdo->hash_zones, completion); + return; + + case RESUME_PHASE_DEPOT: + vdo_resume_slab_depot(vdo->depot, completion); + return; + + case RESUME_PHASE_JOURNAL: + vdo_resume_recovery_journal(vdo->recovery_journal, completion); + return; + + case RESUME_PHASE_BLOCK_MAP: + vdo_resume_block_map(vdo->block_map, completion); + return; + + case RESUME_PHASE_LOGICAL_ZONES: + vdo_resume_logical_zones(vdo->logical_zones, completion); + return; + + case RESUME_PHASE_PACKER: + { + bool was_enabled = vdo_get_compressing(vdo); + bool enable = vdo->device_config->compression; + + if (enable != was_enabled) + WRITE_ONCE(vdo->compressing, enable); + uds_log_info("compression is %s", (enable ? "enabled" : "disabled")); + + vdo_resume_packer(vdo->packer, completion); + return; + } + + case RESUME_PHASE_FLUSHER: + vdo_resume_flusher(vdo->flusher, completion); + return; + + case RESUME_PHASE_DATA_VIOS: + resume_data_vio_pool(vdo->data_vio_pool, completion); + return; + + case RESUME_PHASE_END: + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +/** + * grow_logical_callback() - Callback to initiate a grow logical. + * @completion: The admin completion. + * + * Registered in perform_grow_logical(). + */ +static void grow_logical_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case GROW_LOGICAL_PHASE_START: + if (vdo_is_read_only(vdo)) { + uds_log_error_strerror(VDO_READ_ONLY, + "Can't grow logical size of a read-only VDO"); + vdo_set_completion_result(completion, VDO_READ_ONLY); + break; + } + + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_SUSPENDED_OPERATION); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + vdo->states.vdo.config.logical_blocks = vdo->block_map->next_entry_count; + vdo_save_components(vdo, completion); + return; + + case GROW_LOGICAL_PHASE_GROW_BLOCK_MAP: + vdo_grow_block_map(vdo->block_map, completion); + return; + + case GROW_LOGICAL_PHASE_END: + break; + + case GROW_LOGICAL_PHASE_ERROR: + vdo_enter_read_only_mode(vdo, completion->result); + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +/** + * handle_logical_growth_error() - Handle an error during the grow physical process. + * @completion: The admin completion. + */ +static void handle_logical_growth_error(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + if (vdo->admin.phase == GROW_LOGICAL_PHASE_GROW_BLOCK_MAP) { + /* + * We've failed to write the new size in the super block, so set our in memory + * config back to the old size. + */ + vdo->states.vdo.config.logical_blocks = vdo->block_map->entry_count; + vdo_abandon_block_map_growth(vdo->block_map); + } + + vdo->admin.phase = GROW_LOGICAL_PHASE_ERROR; + grow_logical_callback(completion); +} + +/** + * perform_grow_logical() - Grow the logical size of the vdo. + * @vdo: The vdo to grow. + * @new_logical_blocks: The size to which the vdo should be grown. + * + * Context: This method may only be called when the vdo has been suspended and must not be called + * from a base thread. + * + * Return: VDO_SUCCESS or an error. + */ +static int perform_grow_logical(struct vdo *vdo, block_count_t new_logical_blocks) +{ + int result; + + if (vdo->device_config->logical_blocks == new_logical_blocks) { + /* + * A table was loaded for which we prepared to grow, but a table without that + * growth was what we are resuming with. + */ + vdo_abandon_block_map_growth(vdo->block_map); + return VDO_SUCCESS; + } + + uds_log_info("Resizing logical to %llu", + (unsigned long long) new_logical_blocks); + if (vdo->block_map->next_entry_count != new_logical_blocks) + return VDO_PARAMETER_MISMATCH; + + result = perform_admin_operation(vdo, GROW_LOGICAL_PHASE_START, + grow_logical_callback, + handle_logical_growth_error, "grow logical"); + if (result != VDO_SUCCESS) + return result; + + uds_log_info("Logical blocks now %llu", (unsigned long long) new_logical_blocks); + return VDO_SUCCESS; +} + +static void copy_callback(int read_err, unsigned long write_err, void *context) +{ + struct vdo_completion *completion = context; + int result = (((read_err == 0) && (write_err == 0)) ? VDO_SUCCESS : -EIO); + + vdo_continue_completion(completion, result); +} + +static void partition_to_region(struct partition *partition, struct vdo *vdo, + struct dm_io_region *region) +{ + physical_block_number_t pbn = partition->offset - vdo->geometry.bio_offset; + + *region = (struct dm_io_region) { + .bdev = vdo_get_backing_device(vdo), + .sector = pbn * VDO_SECTORS_PER_BLOCK, + .count = partition->count * VDO_SECTORS_PER_BLOCK, + }; +} + +/** + * copy_partition() - Copy a partition from the location specified in the current layout to that in + * the next layout. + * @vdo: The vdo preparing to grow. + * @id: The ID of the partition to copy. + * @parent: The completion to notify when the copy is complete. + */ +static void copy_partition(struct vdo *vdo, enum partition_id id, + struct vdo_completion *parent) +{ + struct dm_io_region read_region, write_regions[1]; + struct partition *from = vdo_get_known_partition(&vdo->layout, id); + struct partition *to = vdo_get_known_partition(&vdo->next_layout, id); + + partition_to_region(from, vdo, &read_region); + partition_to_region(to, vdo, &write_regions[0]); + dm_kcopyd_copy(vdo->partition_copier, &read_region, 1, write_regions, 0, + copy_callback, parent); +} + +/** + * grow_physical_callback() - Callback to initiate a grow physical. + * @completion: The admin completion. + * + * Registered in perform_grow_physical(). + */ +static void grow_physical_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case GROW_PHYSICAL_PHASE_START: + if (vdo_is_read_only(vdo)) { + uds_log_error_strerror(VDO_READ_ONLY, + "Can't grow physical size of a read-only VDO"); + vdo_set_completion_result(completion, VDO_READ_ONLY); + break; + } + + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_SUSPENDED_OPERATION); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + /* Copy the journal into the new layout. */ + copy_partition(vdo, VDO_RECOVERY_JOURNAL_PARTITION, completion); + return; + + case GROW_PHYSICAL_PHASE_COPY_SUMMARY: + copy_partition(vdo, VDO_SLAB_SUMMARY_PARTITION, completion); + return; + + case GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS: + vdo_uninitialize_layout(&vdo->layout); + vdo->layout = vdo->next_layout; + uds_forget(vdo->next_layout.head); + vdo->states.vdo.config.physical_blocks = vdo->layout.size; + vdo_update_slab_depot_size(vdo->depot); + vdo_save_components(vdo, completion); + return; + + case GROW_PHYSICAL_PHASE_USE_NEW_SLABS: + vdo_use_new_slabs(vdo->depot, completion); + return; + + case GROW_PHYSICAL_PHASE_END: + vdo->depot->summary_origin = + vdo_get_known_partition(&vdo->layout, + VDO_SLAB_SUMMARY_PARTITION)->offset; + vdo->recovery_journal->origin = + vdo_get_known_partition(&vdo->layout, + VDO_RECOVERY_JOURNAL_PARTITION)->offset; + break; + + case GROW_PHYSICAL_PHASE_ERROR: + vdo_enter_read_only_mode(vdo, completion->result); + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + vdo_uninitialize_layout(&vdo->next_layout); + finish_operation_callback(completion); +} + +/** + * handle_physical_growth_error() - Handle an error during the grow physical process. + * @completion: The sub-task completion. + */ +static void handle_physical_growth_error(struct vdo_completion *completion) +{ + completion->vdo->admin.phase = GROW_PHYSICAL_PHASE_ERROR; + grow_physical_callback(completion); +} + +/** + * perform_grow_physical() - Grow the physical size of the vdo. + * @vdo: The vdo to resize. + * @new_physical_blocks: The new physical size in blocks. + * + * Context: This method may only be called when the vdo has been suspended and must not be called + * from a base thread. + * + * Return: VDO_SUCCESS or an error. + */ +static int perform_grow_physical(struct vdo *vdo, block_count_t new_physical_blocks) +{ + int result; + block_count_t new_depot_size, prepared_depot_size; + block_count_t old_physical_blocks = vdo->states.vdo.config.physical_blocks; + + /* Skip any noop grows. */ + if (old_physical_blocks == new_physical_blocks) + return VDO_SUCCESS; + + if (new_physical_blocks != vdo->next_layout.size) { + /* + * Either the VDO isn't prepared to grow, or it was prepared to grow to a different + * size. Doing this check here relies on the fact that the call to this method is + * done under the dmsetup message lock. + */ + vdo_uninitialize_layout(&vdo->next_layout); + vdo_abandon_new_slabs(vdo->depot); + return VDO_PARAMETER_MISMATCH; + } + + /* Validate that we are prepared to grow appropriately. */ + new_depot_size = + vdo_get_known_partition(&vdo->next_layout, VDO_SLAB_DEPOT_PARTITION)->count; + prepared_depot_size = (vdo->depot->new_slabs == NULL) ? 0 : vdo->depot->new_size; + if (prepared_depot_size != new_depot_size) + return VDO_PARAMETER_MISMATCH; + + result = perform_admin_operation(vdo, GROW_PHYSICAL_PHASE_START, + grow_physical_callback, + handle_physical_growth_error, "grow physical"); + if (result != VDO_SUCCESS) + return result; + + uds_log_info("Physical block count was %llu, now %llu", + (unsigned long long) old_physical_blocks, + (unsigned long long) new_physical_blocks); + return VDO_SUCCESS; +} + +/** + * apply_new_vdo_configuration() - Attempt to make any configuration changes from the table being + * resumed. + * @vdo: The vdo being resumed. + * @config: The new device configuration derived from the table with which the vdo is being + * resumed. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check apply_new_vdo_configuration(struct vdo *vdo, + struct device_config *config) +{ + int result; + + result = perform_grow_logical(vdo, config->logical_blocks); + if (result != VDO_SUCCESS) { + uds_log_error("grow logical operation failed, result = %d", result); + return result; + } + + result = perform_grow_physical(vdo, config->physical_blocks); + if (result != VDO_SUCCESS) + uds_log_error("resize operation failed, result = %d", result); + + return result; +} + +static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) +{ + struct device_config *config = ti->private; + const char *device_name = vdo_get_device_name(ti); + block_count_t backing_blocks; + int result; + + backing_blocks = get_underlying_device_block_count(vdo); + if (backing_blocks < config->physical_blocks) { + /* FIXME: can this still happen? */ + uds_log_error("resume of device '%s' failed: backing device has %llu blocks but VDO physical size is %llu blocks", + device_name, (unsigned long long) backing_blocks, + (unsigned long long) config->physical_blocks); + return -EINVAL; + } + + if (vdo_get_admin_state(vdo) == VDO_ADMIN_STATE_PRE_LOADED) { + uds_log_info("starting device '%s'", device_name); + result = perform_admin_operation(vdo, LOAD_PHASE_START, load_callback, + handle_load_error, "load"); + if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { + /* + * Something has gone very wrong. Make sure everything has drained and + * leave the device in an unresumable state. + */ + uds_log_error_strerror(result, + "Start failed, could not load VDO metadata"); + vdo->suspend_type = VDO_ADMIN_STATE_STOPPING; + perform_admin_operation(vdo, SUSPEND_PHASE_START, + suspend_callback, suspend_callback, + "suspend"); + return result; + } + + /* Even if the VDO is read-only, it is now able to handle read requests. */ + uds_log_info("device '%s' started", device_name); + } + + uds_log_info("resuming device '%s'", device_name); + + /* If this fails, the VDO was not in a state to be resumed. This should never happen. */ + result = apply_new_vdo_configuration(vdo, config); + BUG_ON(result == VDO_INVALID_ADMIN_STATE); + + /* + * Now that we've tried to modify the vdo, the new config *is* the config, whether the + * modifications worked or not. + */ + vdo->device_config = config; + + /* + * Any error here is highly unexpected and the state of the vdo is questionable, so we mark + * it read-only in memory. Because we are suspended, the read-only state will not be + * written to disk. + */ + if (result != VDO_SUCCESS) { + uds_log_error_strerror(result, + "Commit of modifications to device '%s' failed", + device_name); + vdo_enter_read_only_mode(vdo, result); + return result; + } + + if (vdo_get_admin_state(vdo)->normal) { + /* The VDO was just started, so we don't need to resume it. */ + return VDO_SUCCESS; + } + + result = perform_admin_operation(vdo, RESUME_PHASE_START, resume_callback, + resume_callback, "resume"); + BUG_ON(result == VDO_INVALID_ADMIN_STATE); + if (result == VDO_READ_ONLY) { + /* Even if the vdo is read-only, it has still resumed. */ + result = VDO_SUCCESS; + } + + if (result != VDO_SUCCESS) + uds_log_error("resume of device '%s' failed with error: %d", device_name, + result); + + return result; +} + +static int vdo_preresume(struct dm_target *ti) +{ + struct registered_thread instance_thread; + struct vdo *vdo = get_vdo_for_target(ti); + int result; + + uds_register_thread_device_id(&instance_thread, &vdo->instance); + result = vdo_preresume_registered(ti, vdo); + if ((result == VDO_PARAMETER_MISMATCH) || (result == VDO_INVALID_ADMIN_STATE)) + result = -EINVAL; + uds_unregister_thread_device_id(); + return vdo_map_to_system_error(result); +} + +static void vdo_resume(struct dm_target *ti) +{ + struct registered_thread instance_thread; + + uds_register_thread_device_id(&instance_thread, + &get_vdo_for_target(ti)->instance); + uds_log_info("device '%s' resumed", vdo_get_device_name(ti)); + uds_unregister_thread_device_id(); +} + +/* + * If anything changes that affects how user tools will interact with vdo, update the version + * number and make sure documentation about the change is complete so tools can properly update + * their management code. + */ +static struct target_type vdo_target_bio = { + .features = DM_TARGET_SINGLETON, + .name = "vdo", + .version = { 8, 2, 0 }, + .module = THIS_MODULE, + .ctr = vdo_ctr, + .dtr = vdo_dtr, + .io_hints = vdo_io_hints, + .iterate_devices = vdo_iterate_devices, + .map = vdo_map_bio, + .message = vdo_message, + .status = vdo_status, + .presuspend = vdo_presuspend, + .postsuspend = vdo_postsuspend, + .preresume = vdo_preresume, + .resume = vdo_resume, +}; + +static bool dm_registered; + +static void vdo_module_destroy(void) +{ + uds_log_debug("unloading"); + + if (dm_registered) + dm_unregister_target(&vdo_target_bio); + + ASSERT_LOG_ONLY(instances.count == 0, + "should have no instance numbers still in use, but have %u", + instances.count); + uds_free(instances.words); + memset(&instances, 0, sizeof(struct instance_tracker)); + + uds_log_info("unloaded version %s", CURRENT_VERSION); +} + +static int __init vdo_init(void) +{ + int result = 0; + + /* + * UDS module level initialization must be done first, as VDO initialization depends on it + */ + uds_initialize_thread_device_registry(); + uds_memory_init(); + uds_init_sysfs(); + + vdo_initialize_device_registry_once(); + uds_log_info("loaded version %s", CURRENT_VERSION); + + /* Add VDO errors to the already existing set of errors in UDS. */ + result = vdo_register_status_codes(); + if (result != UDS_SUCCESS) { + uds_log_error("vdo_register_status_codes failed %d", result); + vdo_module_destroy(); + return result; + } + + result = dm_register_target(&vdo_target_bio); + if (result < 0) { + uds_log_error("dm_register_target failed %d", result); + vdo_module_destroy(); + return result; + } + dm_registered = true; + + return result; +} + +static void __exit vdo_exit(void) +{ + vdo_module_destroy(); + /* + * UDS module level exit processing must be done after all VDO module exit processing is + * complete. + */ + uds_put_sysfs(); + uds_memory_exit(); +} + +module_init(vdo_init); +module_exit(vdo_exit); + +MODULE_DESCRIPTION(DM_NAME " target for transparent deduplication"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); From f11aca85b0346fe7d04eca6eb8db2011398782e7 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:28:35 -0500 Subject: [PATCH 0587/1038] dm vdo: enable configuration and building of dm-vdo dm-vdo targets are not supported for 32-bit configurations. A vdo target typically requires 1 to 1.5 GB of memory at any given time, which is likely a large fraction of the addressable memory of a 32-bit system. At the same time, the amount of addressable storage attached to a 32-bit system may not be large enough for deduplication to provide much benefit. Because of these concerns, 32-bit platforms are deemed unlikely to benefit from using a vdo target, so dm-vdo is targeted only at 64-bit platforms. Co-developed-by: J. corwin Coburn Signed-off-by: J. corwin Coburn Co-developed-by: John Wiele Signed-off-by: John Wiele Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/Kconfig | 16 ++++++++++++++++ drivers/md/Makefile | 2 ++ 2 files changed, 18 insertions(+) diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index a743e2c572fc..cef25cf4800c 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -498,6 +498,22 @@ config DM_FLAKEY help A target that intermittently fails I/O for debugging purposes. +config DM_VDO + tristate "VDO: deduplication and compression target" + depends on 64BIT + depends on BLK_DEV_DM + select DM_BUFIO + select LZ4_COMPRESS + select LZ4_DECOMPRESS + help + This device mapper target presents a block device with + deduplication, compression and thin-provisioning. + + To compile this code as a module, choose M here: the module will + be called dm-vdo. + + If unsure, say N. + config DM_VERITY tristate "Verity target support" depends on BLK_DEV_DM diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 027d7cfeca3f..2ac0e6331104 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -25,6 +25,7 @@ dm-ebs-y += dm-ebs-target.o dm-era-y += dm-era-target.o dm-clone-y += dm-clone-target.o dm-clone-metadata.o dm-verity-y += dm-verity-target.o +dm-vdo-y += dm-vdo-target.o $(patsubst drivers/md/dm-vdo/%.c,dm-vdo/%.o,$(wildcard $(src)/dm-vdo/*.c)) dm-zoned-y += dm-zoned-target.o dm-zoned-metadata.o dm-zoned-reclaim.o md-mod-y += md.o md-bitmap.o @@ -68,6 +69,7 @@ obj-$(CONFIG_DM_ZERO) += dm-zero.o obj-$(CONFIG_DM_RAID) += dm-raid.o obj-$(CONFIG_DM_THIN_PROVISIONING) += dm-thin-pool.o obj-$(CONFIG_DM_VERITY) += dm-verity.o +obj-$(CONFIG_DM_VDO) += dm-vdo.o obj-$(CONFIG_DM_CACHE) += dm-cache.o obj-$(CONFIG_DM_CACHE_SMQ) += dm-cache-smq.o obj-$(CONFIG_DM_EBS) += dm-ebs.o From 77bc133191c132da55c1533c16100191f863a211 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 16 Nov 2023 21:32:39 -0500 Subject: [PATCH 0588/1038] dm vdo: add MAINTAINERS file entry Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- MAINTAINERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 960512bec428..74b82cc8db48 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6118,6 +6118,15 @@ F: include/linux/device-mapper.h F: include/linux/dm-*.h F: include/uapi/linux/dm-*.h +DEVICE-MAPPER VDO TARGET +M: Matthew Sakai +M: dm-devel@lists.linux.dev +L: dm-devel@lists.linux.dev +S: Maintained +F: Documentation/admin-guide/device-mapper/vdo*.rst +F: drivers/md/dm-vdo-target.c +F: drivers/md/dm-vdo/ + DEVLINK M: Jiri Pirko L: netdev@vger.kernel.org From 0dc2009d9791b158746c6ed48486e992e0901342 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 25 Aug 2023 12:42:25 -0400 Subject: [PATCH 0589/1038] dm vdo io-submitter: remove get_bio_sector Just open-code access to bio's sector. Reviewed-by: Susan LeGendre-McGhee Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/io-submitter.c | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index ea27043f06e1..6952ee572a7b 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -115,11 +115,6 @@ static void send_bio_to_device(struct vio *vio, struct bio *bio) submit_bio_noacct(bio); } -static sector_t get_bio_sector(struct bio *bio) -{ - return bio->bi_iter.bi_sector; -} - /** * process_vio_io() - Submits a vio's bio to the underlying block device. May block if the device * is busy. This callback should be used by vios which did not attempt to merge. @@ -149,8 +144,10 @@ static struct bio *get_bio_list(struct vio *vio) assert_in_bio_zone(vio); mutex_lock(&bio_queue_data->lock); - vdo_int_map_remove(bio_queue_data->map, get_bio_sector(vio->bios_merged.head)); - vdo_int_map_remove(bio_queue_data->map, get_bio_sector(vio->bios_merged.tail)); + vdo_int_map_remove(bio_queue_data->map, + vio->bios_merged.head->bi_iter.bi_sector); + vdo_int_map_remove(bio_queue_data->map, + vio->bios_merged.tail->bi_iter.bi_sector); bio = vio->bios_merged.head; bio_list_init(&vio->bios_merged); mutex_unlock(&bio_queue_data->lock); @@ -193,7 +190,7 @@ static struct vio *get_mergeable_locked(struct int_map *map, struct vio *vio, bool back_merge) { struct bio *bio = vio->bio; - sector_t merge_sector = get_bio_sector(bio); + sector_t merge_sector = bio->bi_iter.bi_sector; struct vio *vio_merge; if (back_merge) @@ -216,31 +213,32 @@ static struct vio *get_mergeable_locked(struct int_map *map, struct vio *vio, return NULL; if (back_merge) { - return (get_bio_sector(vio_merge->bios_merged.tail) == merge_sector ? + return (vio_merge->bios_merged.tail->bi_iter.bi_sector == merge_sector ? vio_merge : NULL); } - return (get_bio_sector(vio_merge->bios_merged.head) == merge_sector ? + return (vio_merge->bios_merged.head->bi_iter.bi_sector == merge_sector ? vio_merge : NULL); } static int map_merged_vio(struct int_map *bio_map, struct vio *vio) { int result; + sector_t bio_sector; - result = vdo_int_map_put(bio_map, get_bio_sector(vio->bios_merged.head), vio, - true, NULL); + bio_sector = vio->bios_merged.head->bi_iter.bi_sector; + result = vdo_int_map_put(bio_map, bio_sector, vio, true, NULL); if (result != VDO_SUCCESS) return result; - return vdo_int_map_put(bio_map, get_bio_sector(vio->bios_merged.tail), vio, true, - NULL); + bio_sector = vio->bios_merged.tail->bi_iter.bi_sector; + return vdo_int_map_put(bio_map, bio_sector, vio, true, NULL); } static int merge_to_prev_tail(struct int_map *bio_map, struct vio *vio, struct vio *prev_vio) { - vdo_int_map_remove(bio_map, get_bio_sector(prev_vio->bios_merged.tail)); + vdo_int_map_remove(bio_map, prev_vio->bios_merged.tail->bi_iter.bi_sector); bio_list_merge(&prev_vio->bios_merged, &vio->bios_merged); return map_merged_vio(bio_map, prev_vio); } @@ -253,7 +251,7 @@ static int merge_to_next_head(struct int_map *bio_map, struct vio *vio, * that's compatible with using funnel queues in work queues. This avoids removing an * existing completion. */ - vdo_int_map_remove(bio_map, get_bio_sector(next_vio->bios_merged.head)); + vdo_int_map_remove(bio_map, next_vio->bios_merged.head->bi_iter.bi_sector); bio_list_merge_head(&next_vio->bios_merged, &vio->bios_merged); return map_merged_vio(bio_map, next_vio); } @@ -290,7 +288,7 @@ static bool try_bio_map_merge(struct vio *vio) /* no merge. just add to bio_queue */ merged = false; result = vdo_int_map_put(bio_queue_data->map, - get_bio_sector(bio), + bio->bi_iter.bi_sector, vio, true, NULL); } else if (next_vio == NULL) { /* Only prev. merge to prev's tail */ @@ -299,7 +297,6 @@ static bool try_bio_map_merge(struct vio *vio) /* Only next. merge to next's head */ result = merge_to_next_head(bio_queue_data->map, vio, next_vio); } - mutex_unlock(&bio_queue_data->lock); /* We don't care about failure of int_map_put in this case. */ From f7f46761ccd9b46392ff14e22b8c2ed9f5ecc06d Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 25 Aug 2023 14:36:46 -0400 Subject: [PATCH 0590/1038] dm vdo io-submitter: rename to vdo_submit_metadata_vio Rename submit_metadata_vio() to vdo_submit_metadata_vio(). Reviewed-by: Susan LeGendre-McGhee Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 38 +++++++++++++------------- drivers/md/dm-vdo/io-submitter.c | 10 +++---- drivers/md/dm-vdo/io-submitter.h | 20 +++++++------- drivers/md/dm-vdo/recovery-journal.c | 4 +-- drivers/md/dm-vdo/repair.c | 7 +++-- drivers/md/dm-vdo/slab-depot.c | 40 ++++++++++++++-------------- drivers/md/dm-vdo/vdo.c | 18 ++++++------- 7 files changed, 68 insertions(+), 69 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index e703cb9eafc6..0df1517294e1 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -786,8 +786,8 @@ static int __must_check launch_page_load(struct page_info *info, cache->outstanding_reads++; ADD_ONCE(cache->stats.pages_loaded, 1); callback = (cache->rebuilding ? handle_rebuild_read_error : handle_load_error); - submit_metadata_vio(info->vio, pbn, load_cache_page_endio, - callback, REQ_OP_READ | REQ_PRIO); + vdo_submit_metadata_vio(info->vio, pbn, load_cache_page_endio, + callback, REQ_OP_READ | REQ_PRIO); return VDO_SUCCESS; } @@ -1055,10 +1055,10 @@ static void page_is_written_out(struct vdo_completion *completion) if (!page->header.initialized) { page->header.initialized = true; - submit_metadata_vio(info->vio, info->pbn, - write_cache_page_endio, - handle_page_write_error, - (REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH)); + vdo_submit_metadata_vio(info->vio, info->pbn, + write_cache_page_endio, + handle_page_write_error, + REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH); return; } @@ -1123,8 +1123,8 @@ static void write_pages(struct vdo_completion *flush_completion) continue; } ADD_ONCE(info->cache->stats.pages_saved, 1); - submit_metadata_vio(info->vio, info->pbn, write_cache_page_endio, - handle_page_write_error, REQ_OP_WRITE | REQ_PRIO); + vdo_submit_metadata_vio(info->vio, info->pbn, write_cache_page_endio, + handle_page_write_error, REQ_OP_WRITE | REQ_PRIO); } if (has_unflushed_pages) { @@ -1632,9 +1632,9 @@ static void write_initialized_page(struct vdo_completion *completion) if (zone->flusher == tree_page) operation |= REQ_PREFLUSH; - submit_metadata_vio(vio, vdo_get_block_map_page_pbn(page), - write_page_endio, handle_write_error, - operation); + vdo_submit_metadata_vio(vio, vdo_get_block_map_page_pbn(page), + write_page_endio, handle_write_error, + operation); } static void write_page_endio(struct bio *bio) @@ -1689,9 +1689,9 @@ static void write_page(struct tree_page *tree_page, struct pooled_vio *vio) } page->header.initialized = true; - submit_metadata_vio(&vio->vio, vdo_get_block_map_page_pbn(page), - write_page_endio, handle_write_error, - REQ_OP_WRITE | REQ_PRIO); + vdo_submit_metadata_vio(&vio->vio, vdo_get_block_map_page_pbn(page), + write_page_endio, handle_write_error, + REQ_OP_WRITE | REQ_PRIO); } /* Release a lock on a page which was being loaded or allocated. */ @@ -1879,8 +1879,8 @@ static void load_page(struct waiter *waiter, void *context) physical_block_number_t pbn = lock->tree_slots[lock->height - 1].block_map_slot.pbn; pooled->vio.completion.parent = data_vio; - submit_metadata_vio(&pooled->vio, pbn, load_page_endio, - handle_io_error, REQ_OP_READ | REQ_PRIO); + vdo_submit_metadata_vio(&pooled->vio, pbn, load_page_endio, + handle_io_error, REQ_OP_READ | REQ_PRIO); } /* @@ -2613,9 +2613,9 @@ static void traverse(struct cursor *cursor) next_level->page_index = entry_index; next_level->slot = 0; level->slot++; - submit_metadata_vio(&cursor->vio->vio, location.pbn, - traversal_endio, continue_traversal, - REQ_OP_READ | REQ_PRIO); + vdo_submit_metadata_vio(&cursor->vio->vio, location.pbn, + traversal_endio, continue_traversal, + REQ_OP_READ | REQ_PRIO); return; } } diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 6952ee572a7b..8ca9825357e6 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -320,7 +320,7 @@ void submit_data_vio_io(struct data_vio *data_vio) } /** - * vdo_submit_metadata_io() - Submit I/O for a metadata vio. + * __submit_metadata_vio() - Submit I/O for a metadata vio. * @vio: the vio for which to issue I/O * @physical: the physical block number to read or write * @callback: the bio endio function which will be called after the I/O completes @@ -336,12 +336,12 @@ void submit_data_vio_io(struct data_vio *data_vio) * no error can occur on the bio queue. Currently this is true for all callers, but additional care * will be needed if this ever changes. */ -void vdo_submit_metadata_io(struct vio *vio, physical_block_number_t physical, - bio_end_io_t callback, vdo_action_fn error_handler, - unsigned int operation, char *data) +void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + unsigned int operation, char *data) { - struct vdo_completion *completion = &vio->completion; int result; + struct vdo_completion *completion = &vio->completion; const struct admin_state_code *code = vdo_get_admin_state(completion->vdo); diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h index 96befb3739e9..9e97cfc91bee 100644 --- a/drivers/md/dm-vdo/io-submitter.h +++ b/drivers/md/dm-vdo/io-submitter.h @@ -24,24 +24,24 @@ void process_vio_io(struct vdo_completion *completion); void submit_data_vio_io(struct data_vio *data_vio); -void vdo_submit_metadata_io(struct vio *vio, physical_block_number_t physical, - bio_end_io_t callback, vdo_action_fn error_handler, - unsigned int operation, char *data); +void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + unsigned int operation, char *data); -static inline void submit_metadata_vio(struct vio *vio, physical_block_number_t physical, - bio_end_io_t callback, vdo_action_fn error_handler, - unsigned int operation) +static inline void vdo_submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + unsigned int operation) { - vdo_submit_metadata_io(vio, physical, callback, error_handler, - operation, vio->data); + __submit_metadata_vio(vio, physical, callback, error_handler, + operation, vio->data); } static inline void submit_flush_vio(struct vio *vio, bio_end_io_t callback, vdo_action_fn error_handler) { /* FIXME: Can we just use REQ_OP_FLUSH? */ - vdo_submit_metadata_io(vio, 0, callback, error_handler, - REQ_OP_WRITE | REQ_PREFLUSH, NULL); + __submit_metadata_vio(vio, 0, callback, error_handler, + REQ_OP_WRITE | REQ_PREFLUSH, NULL); } #endif /* VDO_IO_SUBMITTER_H */ diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 463f35ae1430..55411eff5bd7 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -1394,8 +1394,8 @@ static void write_block(struct waiter *waiter, void *context __always_unused) * the data being referenced is stable. The FUA is necessary to ensure that the journal * block itself is stable before allowing overwrites of the lbn's previous data. */ - submit_metadata_vio(&block->vio, journal->origin + block->block_number, - complete_write_endio, handle_write_error, WRITE_FLAGS); + vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number, + complete_write_endio, handle_write_error, WRITE_FLAGS); } diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index 2cf99a7ce958..a75278eb8aa4 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -1748,10 +1748,9 @@ void vdo_repair(struct vdo_completion *parent) remaining -= blocks; } - for (vio_count = 0; - vio_count < repair->vio_count; + for (vio_count = 0; vio_count < repair->vio_count; vio_count++, pbn += MAX_BLOCKS_PER_VIO) { - submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio, - handle_journal_load_error, REQ_OP_READ); + vdo_submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio, + handle_journal_load_error, REQ_OP_READ); } } diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 1ad97d2bf80b..5fc4a1cdfafc 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -338,8 +338,8 @@ static void launch_write(struct slab_summary_block *block) pbn = (depot->summary_origin + (VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * allocator->zone_number) + block->index); - submit_metadata_vio(&block->vio, pbn, write_slab_summary_endio, - handle_write_error, REQ_OP_WRITE | REQ_PREFLUSH); + vdo_submit_metadata_vio(&block->vio, pbn, write_slab_summary_endio, + handle_write_error, REQ_OP_WRITE | REQ_PREFLUSH); } /** @@ -771,8 +771,8 @@ static void write_slab_journal_block(struct waiter *waiter, void *context) * This block won't be read in recovery until the slab summary is updated to refer to it. * The slab summary update does a flush which is sufficient to protect us from VDO-2331. */ - submit_metadata_vio(uds_forget(vio), block_number, write_slab_journal_endio, - complete_write, REQ_OP_WRITE); + vdo_submit_metadata_vio(uds_forget(vio), block_number, write_slab_journal_endio, + complete_write, REQ_OP_WRITE); /* Since the write is submitted, the tail block structure can be reused. */ journal->tail++; @@ -1205,8 +1205,8 @@ static void write_reference_block(struct waiter *waiter, void *context) block->slab->allocator->ref_counts_statistics.blocks_written + 1); completion->callback_thread_id = ((struct block_allocator *) pooled->context)->thread_id; - submit_metadata_vio(&pooled->vio, pbn, write_reference_block_endio, - handle_io_error, REQ_OP_WRITE | REQ_PREFLUSH); + vdo_submit_metadata_vio(&pooled->vio, pbn, write_reference_block_endio, + handle_io_error, REQ_OP_WRITE | REQ_PREFLUSH); } static void reclaim_journal_space(struct slab_journal *journal) @@ -2268,9 +2268,9 @@ static void load_reference_block(struct waiter *waiter, void *context) size_t block_offset = (block - block->slab->reference_blocks); vio->completion.parent = block; - submit_metadata_vio(vio, block->slab->ref_counts_origin + block_offset, - load_reference_block_endio, handle_io_error, - REQ_OP_READ); + vdo_submit_metadata_vio(vio, block->slab->ref_counts_origin + block_offset, + load_reference_block_endio, handle_io_error, + REQ_OP_READ); } /** @@ -2475,9 +2475,9 @@ static void read_slab_journal_tail(struct waiter *waiter, void *context) vio->completion.parent = journal; vio->completion.callback_thread_id = slab->allocator->thread_id; - submit_metadata_vio(vio, slab->journal_origin + tail_block, - read_slab_journal_tail_endio, handle_load_error, - REQ_OP_READ); + vdo_submit_metadata_vio(vio, slab->journal_origin + tail_block, + read_slab_journal_tail_endio, handle_load_error, + REQ_OP_READ); } /** @@ -2915,9 +2915,9 @@ static void start_scrubbing(struct vdo_completion *completion) return; } - submit_metadata_vio(&scrubber->vio, slab->journal_origin, - read_slab_journal_endio, handle_scrubber_error, - REQ_OP_READ); + vdo_submit_metadata_vio(&scrubber->vio, slab->journal_origin, + read_slab_journal_endio, handle_scrubber_error, + REQ_OP_READ); } /** @@ -4513,9 +4513,9 @@ static void finish_loading_summary(struct vdo_completion *completion) combine_summaries(depot); /* Write the combined summary back out. */ - submit_metadata_vio(as_vio(completion), depot->summary_origin, - write_summary_endio, handle_combining_error, - REQ_OP_WRITE); + vdo_submit_metadata_vio(as_vio(completion), depot->summary_origin, + write_summary_endio, handle_combining_error, + REQ_OP_WRITE); } static void load_summary_endio(struct bio *bio) @@ -4555,8 +4555,8 @@ static void load_slab_summary(void *context, struct vdo_completion *parent) return; } - submit_metadata_vio(vio, depot->summary_origin, load_summary_endio, - handle_combining_error, REQ_OP_READ); + vdo_submit_metadata_vio(vio, depot->summary_origin, load_summary_endio, + handle_combining_error, REQ_OP_READ); } /* Implements vdo_zone_action_fn. */ diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index b494c11e1d64..1ead769f0150 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -813,11 +813,11 @@ void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent) } vdo->super_block.vio.completion.parent = parent; - submit_metadata_vio(&vdo->super_block.vio, - vdo_get_data_region_start(vdo->geometry), - read_super_block_endio, - handle_super_block_read_error, - REQ_OP_READ); + vdo_submit_metadata_vio(&vdo->super_block.vio, + vdo_get_data_region_start(vdo->geometry), + read_super_block_endio, + handle_super_block_read_error, + REQ_OP_READ); } /** @@ -1028,10 +1028,10 @@ void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent) vdo_encode_super_block(super_block->buffer, &vdo->states); super_block->vio.completion.parent = parent; super_block->vio.completion.callback_thread_id = parent->callback_thread_id; - submit_metadata_vio(&super_block->vio, - vdo_get_data_region_start(vdo->geometry), - super_block_write_endio, handle_save_error, - REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA); + vdo_submit_metadata_vio(&super_block->vio, + vdo_get_data_region_start(vdo->geometry), + super_block_write_endio, handle_save_error, + REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA); } /** From ebe16015c389bcc1721dfdcba74c13100b849a5b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 25 Aug 2023 14:43:30 -0400 Subject: [PATCH 0591/1038] dm vdo io-submitter: rename to vdo_submit_flush_vio Rename submit_flush_vio() to vdo_submit_flush_vio(). Reviewed-by: Susan LeGendre-McGhee Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 2 +- drivers/md/dm-vdo/io-submitter.h | 4 ++-- drivers/md/dm-vdo/recovery-journal.c | 2 +- drivers/md/dm-vdo/slab-depot.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 0df1517294e1..1edb3b2a80eb 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -835,7 +835,7 @@ static void save_pages(struct vdo_page_cache *cache) * successfully persisted, and thus must issue a flush before each batch of pages is * written to ensure this. */ - submit_flush_vio(vio, flush_endio, handle_flush_error); + vdo_submit_flush_vio(vio, flush_endio, handle_flush_error); } /** diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h index 9e97cfc91bee..77b8df6d559e 100644 --- a/drivers/md/dm-vdo/io-submitter.h +++ b/drivers/md/dm-vdo/io-submitter.h @@ -36,8 +36,8 @@ static inline void vdo_submit_metadata_vio(struct vio *vio, physical_block_numbe operation, vio->data); } -static inline void submit_flush_vio(struct vio *vio, bio_end_io_t callback, - vdo_action_fn error_handler) +static inline void vdo_submit_flush_vio(struct vio *vio, bio_end_io_t callback, + vdo_action_fn error_handler) { /* FIXME: Can we just use REQ_OP_FLUSH? */ __submit_metadata_vio(vio, 0, callback, error_handler, diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 55411eff5bd7..2dfc39deef94 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -1541,7 +1541,7 @@ static void reap_recovery_journal(struct recovery_journal *journal) * summary update covering the slab journal that just released some lock. */ journal->reaping = true; - submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error); + vdo_submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error); } /** diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 5fc4a1cdfafc..670a464ddbb0 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -449,7 +449,7 @@ static void flush_for_reaping(struct waiter *waiter, void *context) struct vio *vio = &pooled->vio; vio->completion.parent = journal; - submit_flush_vio(vio, flush_endio, handle_flush_error); + vdo_submit_flush_vio(vio, flush_endio, handle_flush_error); } /** From d58d3c86c3af96de44171a3748f3e534c5d1631b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 25 Aug 2023 14:52:05 -0400 Subject: [PATCH 0592/1038] dm vdo io-submitter: rename to vdo_submit_data_vio Rename submit_data_vio_io() to vdo_submit_data_vio(). Reviewed-by: Susan LeGendre-McGhee Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/data-vio.c | 4 ++-- drivers/md/dm-vdo/io-submitter.c | 4 ++-- drivers/md/dm-vdo/io-submitter.h | 2 +- drivers/md/dm-vdo/packer.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 09a99a28da4f..54c06e86d321 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -1592,7 +1592,7 @@ static void read_block(struct vdo_completion *completion) return; } - submit_data_vio_io(data_vio); + vdo_submit_data_vio(data_vio); } static inline struct data_vio * @@ -1910,7 +1910,7 @@ void write_data_vio(struct data_vio *data_vio) } data_vio->last_async_operation = VIO_ASYNC_OP_WRITE_DATA_VIO; - submit_data_vio_io(data_vio); + vdo_submit_data_vio(data_vio); } /** diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 8ca9825357e6..1dd670bb0a74 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -305,13 +305,13 @@ static bool try_bio_map_merge(struct vio *vio) } /** - * submit_data_vio_io() - Submit I/O for a data_vio. + * vdo_submit_data_vio() - Submit I/O for a data_vio. * @data_vio: the data_vio for which to issue I/O. * * If possible, this I/O will be merged other pending I/Os. Otherwise, the data_vio will be sent to * the appropriate bio zone directly. */ -void submit_data_vio_io(struct data_vio *data_vio) +void vdo_submit_data_vio(struct data_vio *data_vio) { if (try_bio_map_merge(&data_vio->vio)) return; diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h index 77b8df6d559e..57e1074a65a4 100644 --- a/drivers/md/dm-vdo/io-submitter.h +++ b/drivers/md/dm-vdo/io-submitter.h @@ -22,7 +22,7 @@ void vdo_free_io_submitter(struct io_submitter *io_submitter); void process_vio_io(struct vdo_completion *completion); -void submit_data_vio_io(struct data_vio *data_vio); +void vdo_submit_data_vio(struct data_vio *data_vio); void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, bio_end_io_t callback, vdo_action_fn error_handler, diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c index f09eb1ca4902..e391cac6c92d 100644 --- a/drivers/md/dm-vdo/packer.c +++ b/drivers/md/dm-vdo/packer.c @@ -486,7 +486,7 @@ static void write_bin(struct packer *packer, struct packer_bin *bin) WRITE_ONCE(stats->compressed_blocks_written, stats->compressed_blocks_written + 1); - submit_data_vio_io(agent); + vdo_submit_data_vio(agent); } /** From 46a707cce078303e4114ae5547eb48162bae7323 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 25 Aug 2023 15:16:17 -0400 Subject: [PATCH 0593/1038] dm vdo io-submitter: rename to vdo_submit_vio and submit_data_vio Rename process_vio_io() to vdo_submit_vio(), and process_data_vio_io() to submit_data_vio(). Reviewed-by: Susan LeGendre-McGhee Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dedupe.c | 2 +- drivers/md/dm-vdo/io-submitter.c | 16 ++++++++-------- drivers/md/dm-vdo/io-submitter.h | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 4c4d697a4888..8cc31110f5a8 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -1205,7 +1205,7 @@ static void start_verifying(struct hash_lock *lock, struct data_vio *agent) return; } - set_data_vio_bio_zone_callback(agent, process_vio_io); + set_data_vio_bio_zone_callback(agent, vdo_submit_vio); vdo_launch_completion_with_priority(&vio->completion, BIO_Q_VERIFY_PRIORITY); } diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 1dd670bb0a74..39f5f202d69d 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -116,10 +116,10 @@ static void send_bio_to_device(struct vio *vio, struct bio *bio) } /** - * process_vio_io() - Submits a vio's bio to the underlying block device. May block if the device - * is busy. This callback should be used by vios which did not attempt to merge. + * vdo_submit_vio() - Submits a vio's bio to the underlying block device. May block if the device + * is busy. This callback should be used by vios which did not attempt to merge. */ -void process_vio_io(struct vdo_completion *completion) +void vdo_submit_vio(struct vdo_completion *completion) { struct vio *vio = as_vio(completion); @@ -156,12 +156,12 @@ static struct bio *get_bio_list(struct vio *vio) } /** - * process_data_vio_io() - Submit a data_vio's bio to the storage below along with any bios that - * have been merged with it. + * submit_data_vio() - Submit a data_vio's bio to the storage below along with + * any bios that have been merged with it. * * Context: This call may block and so should only be called from a bio thread. */ -static void process_data_vio_io(struct vdo_completion *completion) +static void submit_data_vio(struct vdo_completion *completion) { struct bio *bio, *next; struct vio *vio = as_vio(completion); @@ -316,7 +316,7 @@ void vdo_submit_data_vio(struct data_vio *data_vio) if (try_bio_map_merge(&data_vio->vio)) return; - launch_data_vio_bio_zone_callback(data_vio, process_data_vio_io); + launch_data_vio_bio_zone_callback(data_vio, submit_data_vio); } /** @@ -356,7 +356,7 @@ void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, return; } - vdo_set_completion_callback(completion, process_vio_io, + vdo_set_completion_callback(completion, vdo_submit_vio, get_vio_bio_zone_thread_id(vio)); vdo_launch_completion_with_priority(completion, get_metadata_priority(vio)); } diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h index 57e1074a65a4..ddd5f4b391b6 100644 --- a/drivers/md/dm-vdo/io-submitter.h +++ b/drivers/md/dm-vdo/io-submitter.h @@ -20,7 +20,7 @@ void vdo_cleanup_io_submitter(struct io_submitter *io_submitter); void vdo_free_io_submitter(struct io_submitter *io_submitter); -void process_vio_io(struct vdo_completion *completion); +void vdo_submit_vio(struct vdo_completion *completion); void vdo_submit_data_vio(struct data_vio *data_vio); From d6e260cc426164820a496528fda809add15be1ea Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 20 Nov 2023 17:29:16 -0500 Subject: [PATCH 0594/1038] dm vdo wait-queue: add proper namespace to interface Rename various interfaces and structs associated with vdo's wait-queue, e.g.: s/wait_queue/vdo_wait_queue/, s/waiter/vdo_waiter/, etc. Now all function names start with "vdo_waitq_" or "vdo_waiter_". Reviewed-by: Ken Raeburn Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 134 ++++++++++--------- drivers/md/dm-vdo/block-map.h | 10 +- drivers/md/dm-vdo/data-vio.c | 14 +- drivers/md/dm-vdo/data-vio.h | 12 +- drivers/md/dm-vdo/dedupe.c | 48 +++---- drivers/md/dm-vdo/dump.c | 12 +- drivers/md/dm-vdo/flush.c | 32 ++--- drivers/md/dm-vdo/flush.h | 2 +- drivers/md/dm-vdo/physical-zone.c | 4 +- drivers/md/dm-vdo/recovery-journal.c | 69 +++++----- drivers/md/dm-vdo/recovery-journal.h | 10 +- drivers/md/dm-vdo/slab-depot.c | 99 +++++++------- drivers/md/dm-vdo/slab-depot.h | 22 ++-- drivers/md/dm-vdo/vio.c | 12 +- drivers/md/dm-vdo/vio.h | 2 +- drivers/md/dm-vdo/wait-queue.c | 190 ++++++++++++++------------- drivers/md/dm-vdo/wait-queue.h | 130 +++++++++--------- 17 files changed, 413 insertions(+), 389 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 1edb3b2a80eb..a1f2c9d38192 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -85,7 +85,7 @@ struct cursor_level { struct cursors; struct cursor { - struct waiter waiter; + struct vdo_waiter waiter; struct block_map_tree *tree; height_t height; struct cursors *parent; @@ -162,7 +162,7 @@ static char *get_page_buffer(struct page_info *info) return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE]; } -static inline struct vdo_page_completion *page_completion_from_waiter(struct waiter *waiter) +static inline struct vdo_page_completion *page_completion_from_waiter(struct vdo_waiter *waiter) { struct vdo_page_completion *completion; @@ -407,7 +407,7 @@ static int reset_page_info(struct page_info *info) if (result != UDS_SUCCESS) return result; - result = ASSERT(!vdo_has_waiters(&info->waiting), + result = ASSERT(!vdo_waitq_has_waiters(&info->waiting), "VDO Page must not have waiters"); if (result != UDS_SUCCESS) return result; @@ -506,7 +506,7 @@ static void complete_with_page(struct page_info *info, * * Implements waiter_callback_fn. */ -static void complete_waiter_with_error(struct waiter *waiter, void *result_ptr) +static void complete_waiter_with_error(struct vdo_waiter *waiter, void *result_ptr) { int *result = result_ptr; @@ -520,25 +520,25 @@ static void complete_waiter_with_error(struct waiter *waiter, void *result_ptr) * * Implements waiter_callback_fn. */ -static void complete_waiter_with_page(struct waiter *waiter, void *page_info) +static void complete_waiter_with_page(struct vdo_waiter *waiter, void *page_info) { complete_with_page(page_info, page_completion_from_waiter(waiter)); } /** - * distribute_page_over_queue() - Complete a queue of VDO page completions with a page result. + * distribute_page_over_waitq() - Complete a waitq of VDO page completions with a page result. * - * Upon completion the queue will be empty. + * Upon completion the waitq will be empty. * * Return: The number of pages distributed. */ -static unsigned int distribute_page_over_queue(struct page_info *info, - struct wait_queue *queue) +static unsigned int distribute_page_over_waitq(struct page_info *info, + struct vdo_wait_queue *waitq) { size_t pages; update_lru(info); - pages = vdo_count_waiters(queue); + pages = vdo_waitq_num_waiters(waitq); /* * Increment the busy count once for each pending completion so that this page does not @@ -546,7 +546,7 @@ static unsigned int distribute_page_over_queue(struct page_info *info, */ info->busy += pages; - vdo_notify_all_waiters(queue, complete_waiter_with_page, info); + vdo_waitq_notify_all_waiters(waitq, complete_waiter_with_page, info); return pages; } @@ -572,13 +572,14 @@ static void set_persistent_error(struct vdo_page_cache *cache, const char *conte assert_on_cache_thread(cache, __func__); - vdo_notify_all_waiters(&cache->free_waiters, complete_waiter_with_error, - &result); + vdo_waitq_notify_all_waiters(&cache->free_waiters, + complete_waiter_with_error, &result); cache->waiter_count = 0; - for (info = cache->infos; info < cache->infos + cache->page_count; info++) - vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, - &result); + for (info = cache->infos; info < cache->infos + cache->page_count; info++) { + vdo_waitq_notify_all_waiters(&info->waiting, + complete_waiter_with_error, &result); + } } /** @@ -625,7 +626,7 @@ static void check_for_drain_complete(struct block_map_zone *zone) { if (vdo_is_state_draining(&zone->state) && (zone->active_lookups == 0) && - !vdo_has_waiters(&zone->flush_waiters) && + !vdo_waitq_has_waiters(&zone->flush_waiters) && !is_vio_pool_busy(zone->vio_pool) && (zone->page_cache.outstanding_reads == 0) && (zone->page_cache.outstanding_writes == 0)) { @@ -643,8 +644,8 @@ static void enter_zone_read_only_mode(struct block_map_zone *zone, int result) * We are in read-only mode, so we won't ever write any page out. Just take all waiters off * the queue so the zone can drain. */ - while (vdo_has_waiters(&zone->flush_waiters)) - vdo_dequeue_next_waiter(&zone->flush_waiters); + while (vdo_waitq_has_waiters(&zone->flush_waiters)) + vdo_waitq_dequeue_next_waiter(&zone->flush_waiters); check_for_drain_complete(zone); } @@ -677,7 +678,7 @@ static void handle_load_error(struct vdo_completion *completion) vdo_enter_read_only_mode(cache->zone->block_map->vdo, result); ADD_ONCE(cache->stats.failed_reads, 1); set_info_state(info, PS_FAILED); - vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); + vdo_waitq_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); reset_page_info(info); /* @@ -720,7 +721,7 @@ static void page_is_loaded(struct vdo_completion *completion) info->recovery_lock = 0; set_info_state(info, PS_RESIDENT); - distribute_page_over_queue(info, &info->waiting); + distribute_page_over_waitq(info, &info->waiting); /* * Don't decrement until right before calling check_for_drain_complete() to @@ -874,7 +875,7 @@ static void launch_page_save(struct page_info *info) * * Return: true if the page completion is for the desired page number. */ -static bool completion_needs_page(struct waiter *waiter, void *context) +static bool completion_needs_page(struct vdo_waiter *waiter, void *context) { physical_block_number_t *pbn = context; @@ -888,13 +889,13 @@ static bool completion_needs_page(struct waiter *waiter, void *context) static void allocate_free_page(struct page_info *info) { int result; - struct waiter *oldest_waiter; + struct vdo_waiter *oldest_waiter; physical_block_number_t pbn; struct vdo_page_cache *cache = info->cache; assert_on_cache_thread(cache, __func__); - if (!vdo_has_waiters(&cache->free_waiters)) { + if (!vdo_waitq_has_waiters(&cache->free_waiters)) { if (cache->stats.cache_pressure > 0) { uds_log_info("page cache pressure relieved"); WRITE_ONCE(cache->stats.cache_pressure, 0); @@ -909,20 +910,22 @@ static void allocate_free_page(struct page_info *info) return; } - oldest_waiter = vdo_get_first_waiter(&cache->free_waiters); + oldest_waiter = vdo_waitq_get_first_waiter(&cache->free_waiters); pbn = page_completion_from_waiter(oldest_waiter)->pbn; /* * Remove all entries which match the page number in question and push them onto the page * info's wait queue. */ - vdo_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page, - &pbn, &info->waiting); - cache->waiter_count -= vdo_count_waiters(&info->waiting); + vdo_waitq_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page, + &pbn, &info->waiting); + cache->waiter_count -= vdo_waitq_num_waiters(&info->waiting); result = launch_page_load(info, pbn); - if (result != VDO_SUCCESS) - vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); + if (result != VDO_SUCCESS) { + vdo_waitq_notify_all_waiters(&info->waiting, + complete_waiter_with_error, &result); + } } /** @@ -966,7 +969,7 @@ static void discard_page_for_completion(struct vdo_page_completion *vdo_page_com struct vdo_page_cache *cache = vdo_page_comp->cache; cache->waiter_count++; - vdo_enqueue_waiter(&cache->free_waiters, &vdo_page_comp->waiter); + vdo_waitq_enqueue_waiter(&cache->free_waiters, &vdo_page_comp->waiter); discard_a_page(cache); } @@ -1069,11 +1072,11 @@ static void page_is_written_out(struct vdo_completion *completion) cache->zone->zone_number); info->recovery_lock = 0; was_discard = write_has_finished(info); - reclaimed = (!was_discard || (info->busy > 0) || vdo_has_waiters(&info->waiting)); + reclaimed = (!was_discard || (info->busy > 0) || vdo_waitq_has_waiters(&info->waiting)); set_info_state(info, PS_RESIDENT); - reclamations = distribute_page_over_queue(info, &info->waiting); + reclamations = distribute_page_over_waitq(info, &info->waiting); ADD_ONCE(cache->stats.reclaimed, reclamations); if (was_discard) @@ -1187,10 +1190,12 @@ static void load_page_for_completion(struct page_info *info, { int result; - vdo_enqueue_waiter(&info->waiting, &vdo_page_comp->waiter); + vdo_waitq_enqueue_waiter(&info->waiting, &vdo_page_comp->waiter); result = launch_page_load(info, vdo_page_comp->pbn); - if (result != VDO_SUCCESS) - vdo_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); + if (result != VDO_SUCCESS) { + vdo_waitq_notify_all_waiters(&info->waiting, + complete_waiter_with_error, &result); + } } /** @@ -1251,7 +1256,7 @@ void vdo_get_page(struct vdo_page_completion *page_completion, (is_outgoing(info) && page_completion->writable)) { /* The page is unusable until it has finished I/O. */ ADD_ONCE(cache->stats.wait_for_page, 1); - vdo_enqueue_waiter(&info->waiting, &page_completion->waiter); + vdo_waitq_enqueue_waiter(&info->waiting, &page_completion->waiter); return; } @@ -1476,7 +1481,7 @@ static void set_generation(struct block_map_zone *zone, struct tree_page *page, { u32 new_count; int result; - bool decrement_old = vdo_is_waiting(&page->waiter); + bool decrement_old = vdo_waiter_is_waiting(&page->waiter); u8 old_generation = page->generation; if (decrement_old && (old_generation == new_generation)) @@ -1498,12 +1503,12 @@ static void set_generation(struct block_map_zone *zone, struct tree_page *page, static void write_page(struct tree_page *tree_page, struct pooled_vio *vio); /* Implements waiter_callback_fn */ -static void write_page_callback(struct waiter *waiter, void *context) +static void write_page_callback(struct vdo_waiter *waiter, void *context) { write_page(container_of(waiter, struct tree_page, waiter), context); } -static void acquire_vio(struct waiter *waiter, struct block_map_zone *zone) +static void acquire_vio(struct vdo_waiter *waiter, struct block_map_zone *zone) { waiter->callback = write_page_callback; acquire_vio_from_pool(zone->vio_pool, waiter); @@ -1530,10 +1535,10 @@ static void enqueue_page(struct tree_page *page, struct block_map_zone *zone) return; } - vdo_enqueue_waiter(&zone->flush_waiters, &page->waiter); + vdo_waitq_enqueue_waiter(&zone->flush_waiters, &page->waiter); } -static void write_page_if_not_dirtied(struct waiter *waiter, void *context) +static void write_page_if_not_dirtied(struct vdo_waiter *waiter, void *context) { struct tree_page *page = container_of(waiter, struct tree_page, waiter); struct write_if_not_dirtied_context *write_context = context; @@ -1576,8 +1581,8 @@ static void finish_page_write(struct vdo_completion *completion) .generation = page->writing_generation, }; - vdo_notify_all_waiters(&zone->flush_waiters, - write_page_if_not_dirtied, &context); + vdo_waitq_notify_all_waiters(&zone->flush_waiters, + write_page_if_not_dirtied, &context); if (dirty && attempt_increment(zone)) { write_page(page, pooled); return; @@ -1588,10 +1593,10 @@ static void finish_page_write(struct vdo_completion *completion) if (dirty) { enqueue_page(page, zone); - } else if ((zone->flusher == NULL) && vdo_has_waiters(&zone->flush_waiters) && + } else if ((zone->flusher == NULL) && vdo_waitq_has_waiters(&zone->flush_waiters) && attempt_increment(zone)) { zone->flusher = - container_of(vdo_dequeue_next_waiter(&zone->flush_waiters), + container_of(vdo_waitq_dequeue_next_waiter(&zone->flush_waiters), struct tree_page, waiter); write_page(zone->flusher, pooled); return; @@ -1724,9 +1729,9 @@ static void finish_lookup(struct data_vio *data_vio, int result) continue_data_vio_with_error(data_vio, result); } -static void abort_lookup_for_waiter(struct waiter *waiter, void *context) +static void abort_lookup_for_waiter(struct vdo_waiter *waiter, void *context) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); int result = *((int *) context); if (!data_vio->write) { @@ -1746,8 +1751,9 @@ static void abort_lookup(struct data_vio *data_vio, int result, char *what) if (data_vio->tree_lock.locked) { release_page_lock(data_vio, what); - vdo_notify_all_waiters(&data_vio->tree_lock.waiters, - abort_lookup_for_waiter, &result); + vdo_waitq_notify_all_waiters(&data_vio->tree_lock.waiters, + abort_lookup_for_waiter, + &result); } finish_lookup(data_vio, result); @@ -1813,9 +1819,9 @@ static void continue_with_loaded_page(struct data_vio *data_vio, load_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); } -static void continue_load_for_waiter(struct waiter *waiter, void *context) +static void continue_load_for_waiter(struct vdo_waiter *waiter, void *context) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); data_vio->tree_lock.height--; continue_with_loaded_page(data_vio, context); @@ -1845,7 +1851,7 @@ static void finish_block_map_page_load(struct vdo_completion *completion) /* Release our claim to the load and wake any waiters */ release_page_lock(data_vio, "load"); - vdo_notify_all_waiters(&tree_lock->waiters, continue_load_for_waiter, page); + vdo_waitq_notify_all_waiters(&tree_lock->waiters, continue_load_for_waiter, page); continue_with_loaded_page(data_vio, page); } @@ -1871,10 +1877,10 @@ static void load_page_endio(struct bio *bio) data_vio->logical.zone->thread_id); } -static void load_page(struct waiter *waiter, void *context) +static void load_page(struct vdo_waiter *waiter, void *context) { struct pooled_vio *pooled = context; - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); struct tree_lock *lock = &data_vio->tree_lock; physical_block_number_t pbn = lock->tree_slots[lock->height - 1].block_map_slot.pbn; @@ -1916,7 +1922,7 @@ static int attempt_page_lock(struct block_map_zone *zone, struct data_vio *data_ } /* Someone else is loading or allocating the page we need */ - vdo_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter); return VDO_SUCCESS; } @@ -1948,9 +1954,9 @@ static void allocation_failure(struct vdo_completion *completion) abort_lookup(data_vio, completion->result, "allocation"); } -static void continue_allocation_for_waiter(struct waiter *waiter, void *context) +static void continue_allocation_for_waiter(struct vdo_waiter *waiter, void *context) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); struct tree_lock *tree_lock = &data_vio->tree_lock; physical_block_number_t pbn = *((physical_block_number_t *) context); @@ -2010,7 +2016,7 @@ static void write_expired_elements(struct block_map_zone *zone) list_del_init(&page->entry); - result = ASSERT(!vdo_is_waiting(&page->waiter), + result = ASSERT(!vdo_waiter_is_waiting(&page->waiter), "Newly expired page not already waiting to write"); if (result != VDO_SUCCESS) { enter_zone_read_only_mode(zone, result); @@ -2089,7 +2095,7 @@ static void finish_block_map_allocation(struct vdo_completion *completion) VDO_MAPPING_STATE_UNCOMPRESSED, &tree_page->recovery_lock); - if (vdo_is_waiting(&tree_page->waiter)) { + if (vdo_waiter_is_waiting(&tree_page->waiter)) { /* This page is waiting to be written out. */ if (zone->flusher != tree_page) { /* @@ -2117,8 +2123,8 @@ static void finish_block_map_allocation(struct vdo_completion *completion) /* Release our claim to the allocation and wake any waiters */ release_page_lock(data_vio, "allocation"); - vdo_notify_all_waiters(&tree_lock->waiters, continue_allocation_for_waiter, - &pbn); + vdo_waitq_notify_all_waiters(&tree_lock->waiters, + continue_allocation_for_waiter, &pbn); if (tree_lock->height == 0) { finish_lookup(data_vio, VDO_SUCCESS); return; @@ -2324,7 +2330,7 @@ physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, */ void vdo_write_tree_page(struct tree_page *page, struct block_map_zone *zone) { - bool waiting = vdo_is_waiting(&page->waiter); + bool waiting = vdo_waiter_is_waiting(&page->waiter); if (waiting && (zone->flusher == page)) return; @@ -2630,7 +2636,7 @@ static void traverse(struct cursor *cursor) * * Implements waiter_callback_fn. */ -static void launch_cursor(struct waiter *waiter, void *context) +static void launch_cursor(struct vdo_waiter *waiter, void *context) { struct cursor *cursor = container_of(waiter, struct cursor, waiter); struct pooled_vio *pooled = context; diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h index dc807111b0e6..cc98d19309ce 100644 --- a/drivers/md/dm-vdo/block-map.h +++ b/drivers/md/dm-vdo/block-map.h @@ -68,7 +68,7 @@ struct vdo_page_cache { /* how many VPCs waiting for free page */ unsigned int waiter_count; /* queue of waiters who want a free page */ - struct wait_queue free_waiters; + struct vdo_wait_queue free_waiters; /* * Statistics are only updated on the logical zone thread, but are accessed from other * threads. @@ -129,7 +129,7 @@ struct page_info { /* page state */ enum vdo_page_buffer_state state; /* queue of completions awaiting this item */ - struct wait_queue waiting; + struct vdo_wait_queue waiting; /* state linked list entry */ struct list_head state_entry; /* LRU entry */ @@ -153,7 +153,7 @@ struct vdo_page_completion { /* The cache involved */ struct vdo_page_cache *cache; /* The waiter for the pending list */ - struct waiter waiter; + struct vdo_waiter waiter; /* The absolute physical block number of the page on disk */ physical_block_number_t pbn; /* Whether the page may be modified */ @@ -167,7 +167,7 @@ struct vdo_page_completion { struct forest; struct tree_page { - struct waiter waiter; + struct vdo_waiter waiter; /* Dirty list entry */ struct list_head entry; @@ -228,7 +228,7 @@ struct block_map_zone { struct vio_pool *vio_pool; /* The tree page which has issued or will be issuing a flush */ struct tree_page *flusher; - struct wait_queue flush_waiters; + struct vdo_wait_queue flush_waiters; /* The generation after the most recent flush */ u8 generation; u8 oldest_generation; diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 54c06e86d321..821155ca3761 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -249,7 +249,7 @@ static void initialize_lbn_lock(struct data_vio *data_vio, logical_block_number_ lock->lbn = lbn; lock->locked = false; - vdo_initialize_wait_queue(&lock->waiters); + vdo_waitq_init(&lock->waiters); zone_number = vdo_compute_logical_zone(data_vio); lock->zone = &vdo->logical_zones->zones[zone_number]; } @@ -466,7 +466,7 @@ static void attempt_logical_block_lock(struct vdo_completion *completion) } data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK; - vdo_enqueue_waiter(&lock_holder->logical.waiters, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&lock_holder->logical.waiters, &data_vio->waiter); /* * Prevent writes and read-modify-writes from blocking indefinitely on lock holders in the @@ -1191,11 +1191,11 @@ static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) /* Another data_vio is waiting for the lock, transfer it in a single lock map operation. */ next_lock_holder = - waiter_as_data_vio(vdo_dequeue_next_waiter(&lock->waiters)); + vdo_waiter_as_data_vio(vdo_waitq_dequeue_next_waiter(&lock->waiters)); /* Transfer the remaining lock waiters to the next lock holder. */ - vdo_transfer_all_waiters(&lock->waiters, - &next_lock_holder->logical.waiters); + vdo_waitq_transfer_all_waiters(&lock->waiters, + &next_lock_holder->logical.waiters); result = vdo_int_map_put(lock->zone->lbn_operations, lock->lbn, next_lock_holder, true, (void **) &lock_holder); @@ -1213,7 +1213,7 @@ static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) * If there are still waiters, other data_vios must be trying to get the lock we just * transferred. We must ensure that the new lock holder doesn't block in the packer. */ - if (vdo_has_waiters(&next_lock_holder->logical.waiters)) + if (vdo_waitq_has_waiters(&next_lock_holder->logical.waiters)) cancel_data_vio_compression(next_lock_holder); /* @@ -1235,7 +1235,7 @@ static void release_logical_lock(struct vdo_completion *completion) assert_data_vio_in_logical_zone(data_vio); - if (vdo_has_waiters(&lock->waiters)) + if (vdo_waitq_has_waiters(&lock->waiters)) transfer_lock(data_vio, lock); else release_lock(data_vio, lock); diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h index aa415b8c7d91..f5a683968d1c 100644 --- a/drivers/md/dm-vdo/data-vio.h +++ b/drivers/md/dm-vdo/data-vio.h @@ -54,7 +54,7 @@ enum async_operation_number { struct lbn_lock { logical_block_number_t lbn; bool locked; - struct wait_queue waiters; + struct vdo_wait_queue waiters; struct logical_zone *zone; }; @@ -75,7 +75,7 @@ struct tree_lock { /* The key for the lock map */ u64 key; /* The queue of waiters for the page this vio is allocating or loading */ - struct wait_queue waiters; + struct vdo_wait_queue waiters; /* The block map tree slots for this LBN */ struct block_map_tree_slot tree_slots[VDO_BLOCK_MAP_TREE_HEIGHT + 1]; }; @@ -168,13 +168,13 @@ struct reference_updater { bool increment; struct zoned_pbn zpbn; struct pbn_lock *lock; - struct waiter waiter; + struct vdo_waiter waiter; }; /* A vio for processing user data requests. */ struct data_vio { - /* The wait_queue entry structure */ - struct waiter waiter; + /* The vdo_wait_queue entry structure */ + struct vdo_waiter waiter; /* The logical block of this request */ struct lbn_lock logical; @@ -288,7 +288,7 @@ static inline struct data_vio *as_data_vio(struct vdo_completion *completion) return vio_as_data_vio(as_vio(completion)); } -static inline struct data_vio *waiter_as_data_vio(struct waiter *waiter) +static inline struct data_vio *vdo_waiter_as_data_vio(struct vdo_waiter *waiter) { if (waiter == NULL) return NULL; diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 8cc31110f5a8..02e36896ca3c 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -270,7 +270,7 @@ struct hash_lock { * to get the information they all need to deduplicate--either against each other, or * against an existing duplicate on disk. */ - struct wait_queue waiters; + struct vdo_wait_queue waiters; }; enum { @@ -351,7 +351,7 @@ static void return_hash_lock_to_pool(struct hash_zone *zone, struct hash_lock *l memset(lock, 0, sizeof(*lock)); INIT_LIST_HEAD(&lock->pool_node); INIT_LIST_HEAD(&lock->duplicate_ring); - vdo_initialize_wait_queue(&lock->waiters); + vdo_waitq_init(&lock->waiters); list_add_tail(&lock->pool_node, &zone->lock_pool); } @@ -420,7 +420,7 @@ static void set_duplicate_lock(struct hash_lock *hash_lock, struct pbn_lock *pbn */ static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock) { - return waiter_as_data_vio(vdo_dequeue_next_waiter(&lock->waiters)); + return vdo_waiter_as_data_vio(vdo_waitq_dequeue_next_waiter(&lock->waiters)); } /** @@ -536,7 +536,7 @@ static struct data_vio *retire_lock_agent(struct hash_lock *lock) */ static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio) { - vdo_enqueue_waiter(&lock->waiters, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&lock->waiters, &data_vio->waiter); /* * Make sure the agent doesn't block indefinitely in the packer since it now has at least @@ -562,9 +562,9 @@ static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio) * @waiter: The data_vio's waiter link. * @context: Not used. */ -static void abort_waiter(struct waiter *waiter, void *context __always_unused) +static void abort_waiter(struct vdo_waiter *waiter, void *context __always_unused) { - write_data_vio(waiter_as_data_vio(waiter)); + write_data_vio(vdo_waiter_as_data_vio(waiter)); } /** @@ -602,7 +602,7 @@ void vdo_clean_failed_hash_lock(struct data_vio *data_vio) /* Ensure we don't attempt to update advice when cleaning up. */ lock->update_advice = false; - vdo_notify_all_waiters(&lock->waiters, abort_waiter, NULL); + vdo_waitq_notify_all_waiters(&lock->waiters, abort_waiter, NULL); if (lock->duplicate_lock != NULL) { /* The agent must reference the duplicate zone to launch it. */ @@ -650,7 +650,7 @@ static void finish_unlocking(struct vdo_completion *completion) */ lock->verified = false; - if (vdo_has_waiters(&lock->waiters)) { + if (vdo_waitq_has_waiters(&lock->waiters)) { /* * UNLOCKING -> LOCKING transition: A new data_vio entered the hash lock while the * agent was releasing the PBN lock. The current agent exits and the waiter has to @@ -750,7 +750,7 @@ static void finish_updating(struct vdo_completion *completion) */ lock->update_advice = false; - if (vdo_has_waiters(&lock->waiters)) { + if (vdo_waitq_has_waiters(&lock->waiters)) { /* * UPDATING -> DEDUPING transition: A new data_vio arrived during the UDS update. * Send it on the verified dedupe path. The agent is done with the lock, but the @@ -812,7 +812,7 @@ static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio) struct data_vio *agent = data_vio; ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING"); - ASSERT_LOG_ONLY(!vdo_has_waiters(&lock->waiters), + ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), "shouldn't have any lock waiters in DEDUPING"); /* Just release the lock reference if other data_vios are still deduping. */ @@ -917,9 +917,9 @@ static int __must_check acquire_lock(struct hash_zone *zone, * Implements waiter_callback_fn. Binds the data_vio that was waiting to a new hash lock and waits * on that lock. */ -static void enter_forked_lock(struct waiter *waiter, void *context) +static void enter_forked_lock(struct vdo_waiter *waiter, void *context) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); struct hash_lock *new_lock = context; set_hash_lock(data_vio, new_lock); @@ -956,7 +956,7 @@ static void fork_hash_lock(struct hash_lock *old_lock, struct data_vio *new_agen set_hash_lock(new_agent, new_lock); new_lock->agent = new_agent; - vdo_notify_all_waiters(&old_lock->waiters, enter_forked_lock, new_lock); + vdo_waitq_notify_all_waiters(&old_lock->waiters, enter_forked_lock, new_lock); new_agent->is_duplicate = false; start_writing(new_lock, new_agent); @@ -1033,7 +1033,7 @@ static void start_deduping(struct hash_lock *lock, struct data_vio *agent, launch_dedupe(lock, agent, true); agent = NULL; } - while (vdo_has_waiters(&lock->waiters)) + while (vdo_waitq_has_waiters(&lock->waiters)) launch_dedupe(lock, dequeue_lock_waiter(lock), false); if (agent_is_done) { @@ -1454,7 +1454,7 @@ static void finish_writing(struct hash_lock *lock, struct data_vio *agent) lock->update_advice = true; /* If there are any waiters, we need to start deduping them. */ - if (vdo_has_waiters(&lock->waiters)) { + if (vdo_waitq_has_waiters(&lock->waiters)) { /* * WRITING -> DEDUPING transition: an asynchronously-written block failed to * compress, so the PBN lock on the written copy was already transferred. The agent @@ -1502,10 +1502,10 @@ static void finish_writing(struct hash_lock *lock, struct data_vio *agent) */ static struct data_vio *select_writing_agent(struct hash_lock *lock) { - struct wait_queue temp_queue; + struct vdo_wait_queue temp_queue; struct data_vio *data_vio; - vdo_initialize_wait_queue(&temp_queue); + vdo_waitq_init(&temp_queue); /* * Move waiters to the temp queue one-by-one until we find an allocation. Not ideal to @@ -1514,7 +1514,7 @@ static struct data_vio *select_writing_agent(struct hash_lock *lock) while (((data_vio = dequeue_lock_waiter(lock)) != NULL) && !data_vio_has_allocation(data_vio)) { /* Use the lower-level enqueue since we're just moving waiters around. */ - vdo_enqueue_waiter(&temp_queue, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&temp_queue, &data_vio->waiter); } if (data_vio != NULL) { @@ -1522,13 +1522,13 @@ static struct data_vio *select_writing_agent(struct hash_lock *lock) * Move the rest of the waiters over to the temp queue, preserving the order they * arrived at the lock. */ - vdo_transfer_all_waiters(&lock->waiters, &temp_queue); + vdo_waitq_transfer_all_waiters(&lock->waiters, &temp_queue); /* * The current agent is being replaced and will have to wait to dedupe; make it the * first waiter since it was the first to reach the lock. */ - vdo_enqueue_waiter(&lock->waiters, &lock->agent->waiter); + vdo_waitq_enqueue_waiter(&lock->waiters, &lock->agent->waiter); lock->agent = data_vio; } else { /* No one has an allocation, so keep the current agent. */ @@ -1536,7 +1536,7 @@ static struct data_vio *select_writing_agent(struct hash_lock *lock) } /* Swap all the waiters back onto the lock's queue. */ - vdo_transfer_all_waiters(&temp_queue, &lock->waiters); + vdo_waitq_transfer_all_waiters(&temp_queue, &lock->waiters); return data_vio; } @@ -1577,7 +1577,7 @@ static void start_writing(struct hash_lock *lock, struct data_vio *agent) * If the agent compresses, it might wait indefinitely in the packer, which would be bad if * there are any other data_vios waiting. */ - if (vdo_has_waiters(&lock->waiters)) + if (vdo_waitq_has_waiters(&lock->waiters)) cancel_data_vio_compression(agent); /* @@ -1928,7 +1928,7 @@ void vdo_release_hash_lock(struct data_vio *data_vio) "unregistered hash lock must not be in the lock map"); } - ASSERT_LOG_ONLY(!vdo_has_waiters(&lock->waiters), + ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), "hash lock returned to zone must have no waiters"); ASSERT_LOG_ONLY((lock->duplicate_lock == NULL), "hash lock returned to zone must not reference a PBN lock"); @@ -2812,7 +2812,7 @@ static void dump_hash_lock(const struct hash_lock *lock) lock, state, (lock->registered ? 'D' : 'U'), (unsigned long long) lock->duplicate.pbn, lock->duplicate.state, lock->reference_count, - vdo_count_waiters(&lock->waiters), lock->agent); + vdo_waitq_num_waiters(&lock->waiters), lock->agent); } static const char *index_state_to_string(struct hash_zones *zones, diff --git a/drivers/md/dm-vdo/dump.c b/drivers/md/dm-vdo/dump.c index 99266a946ed7..91bc8ed36aa7 100644 --- a/drivers/md/dm-vdo/dump.c +++ b/drivers/md/dm-vdo/dump.c @@ -146,25 +146,25 @@ void vdo_dump_all(struct vdo *vdo, const char *why) } /* - * Dump out the data_vio waiters on a wait queue. + * Dump out the data_vio waiters on a waitq. * wait_on should be the label to print for queue (e.g. logical or physical) */ -static void dump_vio_waiters(struct wait_queue *queue, char *wait_on) +static void dump_vio_waiters(struct vdo_wait_queue *waitq, char *wait_on) { - struct waiter *waiter, *first = vdo_get_first_waiter(queue); + struct vdo_waiter *waiter, *first = vdo_waitq_get_first_waiter(waitq); struct data_vio *data_vio; if (first == NULL) return; - data_vio = waiter_as_data_vio(first); + data_vio = vdo_waiter_as_data_vio(first); uds_log_info(" %s is locked. Waited on by: vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", wait_on, data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio)); for (waiter = first->next_waiter; waiter != first; waiter = waiter->next_waiter) { - data_vio = waiter_as_data_vio(waiter); + data_vio = vdo_waiter_as_data_vio(waiter); uds_log_info(" ... and : vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, data_vio->duplicate.pbn, @@ -177,7 +177,7 @@ static void dump_vio_waiters(struct wait_queue *queue, char *wait_on) * logging brevity: * * R => vio completion result not VDO_SUCCESS - * W => vio is on a wait queue + * W => vio is on a waitq * D => vio is a duplicate * p => vio is a partial block operation * z => vio is a zero block diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index a99607e23fb0..e7195c677773 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -31,9 +31,9 @@ struct flusher { /** The first unacknowledged flush generation */ sequence_number_t first_unacknowledged_generation; /** The queue of flush requests waiting to notify other threads */ - struct wait_queue notifiers; + struct vdo_wait_queue notifiers; /** The queue of flush requests waiting for VIOs to complete */ - struct wait_queue pending_flushes; + struct vdo_wait_queue pending_flushes; /** The flush generation for which notifications are being sent */ sequence_number_t notify_generation; /** The logical zone to notify next */ @@ -93,7 +93,7 @@ static inline struct vdo_flush *completion_as_vdo_flush(struct vdo_completion *c * * Return: The wait queue entry as a vdo_flush. */ -static struct vdo_flush *waiter_as_flush(struct waiter *waiter) +static struct vdo_flush *vdo_waiter_as_flush(struct vdo_waiter *waiter) { return container_of(waiter, struct vdo_flush, waiter); } @@ -195,10 +195,10 @@ static void finish_notification(struct vdo_completion *completion) assert_on_flusher_thread(flusher, __func__); - vdo_enqueue_waiter(&flusher->pending_flushes, - vdo_dequeue_next_waiter(&flusher->notifiers)); + vdo_waitq_enqueue_waiter(&flusher->pending_flushes, + vdo_waitq_dequeue_next_waiter(&flusher->notifiers)); vdo_complete_flushes(flusher); - if (vdo_has_waiters(&flusher->notifiers)) + if (vdo_waitq_has_waiters(&flusher->notifiers)) notify_flush(flusher); } @@ -248,7 +248,8 @@ static void increment_generation(struct vdo_completion *completion) */ static void notify_flush(struct flusher *flusher) { - struct vdo_flush *flush = waiter_as_flush(vdo_get_first_waiter(&flusher->notifiers)); + struct vdo_flush *flush = + vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->notifiers)); flusher->notify_generation = flush->flush_generation; flusher->logical_zone_to_notify = &flusher->vdo->logical_zones->zones[0]; @@ -280,8 +281,8 @@ static void flush_vdo(struct vdo_completion *completion) } flush->flush_generation = flusher->flush_generation++; - may_notify = !vdo_has_waiters(&flusher->notifiers); - vdo_enqueue_waiter(&flusher->notifiers, &flush->waiter); + may_notify = !vdo_waitq_has_waiters(&flusher->notifiers); + vdo_waitq_enqueue_waiter(&flusher->notifiers, &flush->waiter); if (may_notify) notify_flush(flusher); } @@ -294,7 +295,8 @@ static void check_for_drain_complete(struct flusher *flusher) { bool drained; - if (!vdo_is_state_draining(&flusher->state) || vdo_has_waiters(&flusher->pending_flushes)) + if (!vdo_is_state_draining(&flusher->state) || + vdo_waitq_has_waiters(&flusher->pending_flushes)) return; spin_lock(&flusher->lock); @@ -321,9 +323,9 @@ void vdo_complete_flushes(struct flusher *flusher) min(oldest_active_generation, READ_ONCE(zone->oldest_active_generation)); - while (vdo_has_waiters(&flusher->pending_flushes)) { + while (vdo_waitq_has_waiters(&flusher->pending_flushes)) { struct vdo_flush *flush = - waiter_as_flush(vdo_get_first_waiter(&flusher->pending_flushes)); + vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->pending_flushes)); if (flush->flush_generation >= oldest_active_generation) return; @@ -333,7 +335,7 @@ void vdo_complete_flushes(struct flusher *flusher) "acknowledged next expected flush, %llu, was: %llu", (unsigned long long) flusher->first_unacknowledged_generation, (unsigned long long) flush->flush_generation); - vdo_dequeue_next_waiter(&flusher->pending_flushes); + vdo_waitq_dequeue_next_waiter(&flusher->pending_flushes); vdo_complete_flush(flush); flusher->first_unacknowledged_generation++; } @@ -352,8 +354,8 @@ void vdo_dump_flusher(const struct flusher *flusher) (unsigned long long) flusher->flush_generation, (unsigned long long) flusher->first_unacknowledged_generation); uds_log_info(" notifiers queue is %s; pending_flushes queue is %s", - (vdo_has_waiters(&flusher->notifiers) ? "not empty" : "empty"), - (vdo_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty")); + (vdo_waitq_has_waiters(&flusher->notifiers) ? "not empty" : "empty"), + (vdo_waitq_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty")); } /** diff --git a/drivers/md/dm-vdo/flush.h b/drivers/md/dm-vdo/flush.h index 4d40908462bb..97252d6656e0 100644 --- a/drivers/md/dm-vdo/flush.h +++ b/drivers/md/dm-vdo/flush.h @@ -18,7 +18,7 @@ struct vdo_flush { /* The flush bios covered by this request */ struct bio_list bios; /* The wait queue entry for this flush */ - struct waiter waiter; + struct vdo_waiter waiter; /* Which flush this struct represents */ sequence_number_t flush_generation; }; diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index d3fc4666c3c2..9b99c9a820a3 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -519,9 +519,9 @@ static int allocate_and_lock_block(struct allocation *allocation) * @waiter: The allocating_vio that was waiting to allocate. * @context: The context (unused). */ -static void retry_allocation(struct waiter *waiter, void *context __always_unused) +static void retry_allocation(struct vdo_waiter *waiter, void *context __always_unused) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); /* Now that some slab has scrubbed, restart the allocation process. */ data_vio->allocation.wait_for_clean_slab = false; diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 2dfc39deef94..5126e670e97e 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -267,9 +267,9 @@ static void assert_on_journal_thread(struct recovery_journal *journal, * Invoked whenever a data_vio is to be released from the journal, either because its entry was * committed to disk, or because there was an error. Implements waiter_callback_fn. */ -static void continue_waiter(struct waiter *waiter, void *context) +static void continue_waiter(struct vdo_waiter *waiter, void *context) { - continue_data_vio_with_error(waiter_as_data_vio(waiter), *((int *) context)); + continue_data_vio_with_error(vdo_waiter_as_data_vio(waiter), *((int *) context)); } /** @@ -287,8 +287,8 @@ static inline bool has_block_waiters(struct recovery_journal *journal) * has waiters. */ return ((block != NULL) && - (vdo_has_waiters(&block->entry_waiters) || - vdo_has_waiters(&block->commit_waiters))); + (vdo_waitq_has_waiters(&block->entry_waiters) || + vdo_waitq_has_waiters(&block->commit_waiters))); } static void recycle_journal_blocks(struct recovery_journal *journal); @@ -343,14 +343,14 @@ static void check_for_drain_complete(struct recovery_journal *journal) recycle_journal_blocks(journal); /* Release any data_vios waiting to be assigned entries. */ - vdo_notify_all_waiters(&journal->entry_waiters, continue_waiter, - &result); + vdo_waitq_notify_all_waiters(&journal->entry_waiters, + continue_waiter, &result); } if (!vdo_is_state_draining(&journal->state) || journal->reaping || has_block_waiters(journal) || - vdo_has_waiters(&journal->entry_waiters) || + vdo_waitq_has_waiters(&journal->entry_waiters) || !suspend_lock_counter(&journal->lock_counter)) return; @@ -721,7 +721,7 @@ int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t INIT_LIST_HEAD(&journal->free_tail_blocks); INIT_LIST_HEAD(&journal->active_tail_blocks); - vdo_initialize_wait_queue(&journal->pending_writes); + vdo_waitq_init(&journal->pending_writes); journal->thread_id = vdo->thread_config.journal_thread; journal->origin = partition->offset; @@ -1047,7 +1047,7 @@ static void schedule_block_write(struct recovery_journal *journal, struct recovery_journal_block *block) { if (!block->committing) - vdo_enqueue_waiter(&journal->pending_writes, &block->write_waiter); + vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter); /* * At the end of adding entries, or discovering this partial block is now full and ready to * rewrite, we will call write_blocks() and write a whole batch. @@ -1084,9 +1084,9 @@ static void update_usages(struct recovery_journal *journal, struct data_vio *dat * * Implements waiter_callback_fn. */ -static void assign_entry(struct waiter *waiter, void *context) +static void assign_entry(struct vdo_waiter *waiter, void *context) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); struct recovery_journal_block *block = context; struct recovery_journal *journal = block->journal; @@ -1099,10 +1099,10 @@ static void assign_entry(struct waiter *waiter, void *context) update_usages(journal, data_vio); journal->available_space--; - if (!vdo_has_waiters(&block->entry_waiters)) + if (!vdo_waitq_has_waiters(&block->entry_waiters)) journal->events.blocks.started++; - vdo_enqueue_waiter(&block->entry_waiters, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter); block->entry_count++; block->uncommitted_entry_count++; journal->events.entries.started++; @@ -1127,9 +1127,10 @@ static void assign_entries(struct recovery_journal *journal) } journal->adding_entries = true; - while (vdo_has_waiters(&journal->entry_waiters) && prepare_to_assign_entry(journal)) { - vdo_notify_next_waiter(&journal->entry_waiters, assign_entry, - journal->active_block); + while (vdo_waitq_has_waiters(&journal->entry_waiters) && + prepare_to_assign_entry(journal)) { + vdo_waitq_notify_next_waiter(&journal->entry_waiters, + assign_entry, journal->active_block); } /* Now that we've finished with entries, see if we have a batch of blocks to write. */ @@ -1170,9 +1171,9 @@ static void recycle_journal_block(struct recovery_journal_block *block) * * Implements waiter_callback_fn. */ -static void continue_committed_waiter(struct waiter *waiter, void *context) +static void continue_committed_waiter(struct vdo_waiter *waiter, void *context) { - struct data_vio *data_vio = waiter_as_data_vio(waiter); + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); struct recovery_journal *journal = context; int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS); bool has_decrement; @@ -1216,11 +1217,12 @@ static void notify_commit_waiters(struct recovery_journal *journal) if (block->committing) return; - vdo_notify_all_waiters(&block->commit_waiters, continue_committed_waiter, - journal); + vdo_waitq_notify_all_waiters(&block->commit_waiters, + continue_committed_waiter, journal); if (is_read_only(journal)) { - vdo_notify_all_waiters(&block->entry_waiters, - continue_committed_waiter, journal); + vdo_waitq_notify_all_waiters(&block->entry_waiters, + continue_committed_waiter, + journal); } else if (is_block_dirty(block) || !is_block_full(block)) { /* Stop at partially-committed or partially-filled blocks. */ return; @@ -1328,9 +1330,9 @@ static void complete_write_endio(struct bio *bio) */ static void add_queued_recovery_entries(struct recovery_journal_block *block) { - while (vdo_has_waiters(&block->entry_waiters)) { + while (vdo_waitq_has_waiters(&block->entry_waiters)) { struct data_vio *data_vio = - waiter_as_data_vio(vdo_dequeue_next_waiter(&block->entry_waiters)); + vdo_waiter_as_data_vio(vdo_waitq_dequeue_next_waiter(&block->entry_waiters)); struct tree_lock *lock = &data_vio->tree_lock; struct packed_recovery_journal_entry *packed_entry; struct recovery_journal_entry new_entry; @@ -1357,7 +1359,7 @@ static void add_queued_recovery_entries(struct recovery_journal_block *block) data_vio->recovery_sequence_number = block->sequence_number; /* Enqueue the data_vio to wait for its entry to commit. */ - vdo_enqueue_waiter(&block->commit_waiters, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter); } } @@ -1366,17 +1368,18 @@ static void add_queued_recovery_entries(struct recovery_journal_block *block) * * Implements waiter_callback_fn. */ -static void write_block(struct waiter *waiter, void *context __always_unused) +static void write_block(struct vdo_waiter *waiter, void *context __always_unused) { struct recovery_journal_block *block = container_of(waiter, struct recovery_journal_block, write_waiter); struct recovery_journal *journal = block->journal; struct packed_journal_header *header = get_block_header(block); - if (block->committing || !vdo_has_waiters(&block->entry_waiters) || is_read_only(journal)) + if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) || + is_read_only(journal)) return; - block->entries_in_commit = vdo_count_waiters(&block->entry_waiters); + block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters); add_queued_recovery_entries(block); journal->pending_write_count += 1; @@ -1419,7 +1422,7 @@ static void write_blocks(struct recovery_journal *journal) return; /* Write all the full blocks. */ - vdo_notify_all_waiters(&journal->pending_writes, write_block, NULL); + vdo_waitq_notify_all_waiters(&journal->pending_writes, write_block, NULL); /* * Do we need to write the active block? Only if we have no outstanding writes, even after @@ -1459,7 +1462,7 @@ void vdo_add_recovery_journal_entry(struct recovery_journal *journal, "journal lock not held for new entry"); vdo_advance_journal_point(&journal->append_point, journal->entries_per_block); - vdo_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter); + vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter); assign_entries(journal); } @@ -1721,8 +1724,8 @@ static void dump_recovery_block(const struct recovery_journal_block *block) uds_log_info(" sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters", (unsigned long long) block->sequence_number, block->entry_count, (block->committing ? "committing" : "waiting"), - vdo_count_waiters(&block->entry_waiters), - vdo_count_waiters(&block->commit_waiters)); + vdo_waitq_num_waiters(&block->entry_waiters), + vdo_waitq_num_waiters(&block->commit_waiters)); } /** @@ -1745,7 +1748,7 @@ void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal (unsigned long long) journal->slab_journal_reap_head, (unsigned long long) stats.disk_full, (unsigned long long) stats.slab_journal_commits_requested, - vdo_count_waiters(&journal->entry_waiters)); + vdo_waitq_num_waiters(&journal->entry_waiters)); uds_log_info(" entries: started=%llu written=%llu committed=%llu", (unsigned long long) stats.entries.started, (unsigned long long) stats.entries.written, diff --git a/drivers/md/dm-vdo/recovery-journal.h b/drivers/md/dm-vdo/recovery-journal.h index c6d83019f918..19fa7ed9648a 100644 --- a/drivers/md/dm-vdo/recovery-journal.h +++ b/drivers/md/dm-vdo/recovery-journal.h @@ -113,7 +113,7 @@ struct recovery_journal_block { /* The doubly linked pointers for the free or active lists */ struct list_head list_node; /* The waiter for the pending full block list */ - struct waiter write_waiter; + struct vdo_waiter write_waiter; /* The journal to which this block belongs */ struct recovery_journal *journal; /* A pointer to the current sector in the packed block buffer */ @@ -133,9 +133,9 @@ struct recovery_journal_block { /* The number of new entries in the current commit */ journal_entry_count_t entries_in_commit; /* The queue of vios which will make entries for the next commit */ - struct wait_queue entry_waiters; + struct vdo_wait_queue entry_waiters; /* The queue of vios waiting for the current commit */ - struct wait_queue commit_waiters; + struct vdo_wait_queue commit_waiters; }; struct recovery_journal { @@ -146,7 +146,7 @@ struct recovery_journal { /* The block map which can hold locks on this journal */ struct block_map *block_map; /* The queue of vios waiting to make entries */ - struct wait_queue entry_waiters; + struct vdo_wait_queue entry_waiters; /* The number of free entries in the journal */ u64 available_space; /* The number of decrement entries which need to be made */ @@ -184,7 +184,7 @@ struct recovery_journal { /* A pointer to the active block (the one we are adding entries to now) */ struct recovery_journal_block *active_block; /* Journal blocks that need writing */ - struct wait_queue pending_writes; + struct vdo_wait_queue pending_writes; /* The new block map reap head after reaping */ sequence_number_t block_map_reap_head; /* The head block number for the block map rebuild range */ diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 670a464ddbb0..2125e256aa86 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -65,7 +65,7 @@ static bool is_slab_open(struct vdo_slab *slab) static inline bool __must_check must_make_entries_to_flush(struct slab_journal *journal) { return ((journal->slab->status != VDO_SLAB_REBUILDING) && - vdo_has_waiters(&journal->entry_waiters)); + vdo_waitq_has_waiters(&journal->entry_waiters)); } /** @@ -122,7 +122,7 @@ static bool __must_check block_is_full(struct slab_journal *journal) static void add_entries(struct slab_journal *journal); static void update_tail_block_location(struct slab_journal *journal); -static void release_journal_locks(struct waiter *waiter, void *context); +static void release_journal_locks(struct vdo_waiter *waiter, void *context); /** * is_slab_journal_blank() - Check whether a slab's journal is blank. @@ -184,7 +184,7 @@ static void check_if_slab_drained(struct vdo_slab *slab) code = vdo_get_admin_state_code(&slab->state); read_only = vdo_is_read_only(slab->allocator->depot->vdo); if (!read_only && - vdo_has_waiters(&slab->dirty_blocks) && + vdo_waitq_has_waiters(&slab->dirty_blocks) && (code != VDO_ADMIN_STATE_SUSPENDING) && (code != VDO_ADMIN_STATE_RECOVERING)) return; @@ -229,14 +229,13 @@ static u8 __must_check compute_fullness_hint(struct slab_depot *depot, */ static void check_summary_drain_complete(struct block_allocator *allocator) { - struct vdo *vdo = allocator->depot->vdo; - if (!vdo_is_state_draining(&allocator->summary_state) || (allocator->summary_write_count > 0)) return; vdo_finish_operation(&allocator->summary_state, - (vdo_is_read_only(vdo) ? VDO_READ_ONLY : VDO_SUCCESS)); + (vdo_is_read_only(allocator->depot->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS)); } /** @@ -245,11 +244,12 @@ static void check_summary_drain_complete(struct block_allocator *allocator) * @queue: The queue to notify. */ static void notify_summary_waiters(struct block_allocator *allocator, - struct wait_queue *queue) + struct vdo_wait_queue *queue) { - int result = (vdo_is_read_only(allocator->depot->vdo) ? VDO_READ_ONLY : VDO_SUCCESS); + int result = (vdo_is_read_only(allocator->depot->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS); - vdo_notify_all_waiters(queue, NULL, &result); + vdo_waitq_notify_all_waiters(queue, NULL, &result); } static void launch_write(struct slab_summary_block *summary_block); @@ -264,7 +264,7 @@ static void finish_updating_slab_summary_block(struct slab_summary_block *block) notify_summary_waiters(block->allocator, &block->current_update_waiters); block->writing = false; block->allocator->summary_write_count--; - if (vdo_has_waiters(&block->next_update_waiters)) + if (vdo_waitq_has_waiters(&block->next_update_waiters)) launch_write(block); else check_summary_drain_complete(block->allocator); @@ -320,8 +320,8 @@ static void launch_write(struct slab_summary_block *block) return; allocator->summary_write_count++; - vdo_transfer_all_waiters(&block->next_update_waiters, - &block->current_update_waiters); + vdo_waitq_transfer_all_waiters(&block->next_update_waiters, + &block->current_update_waiters); block->writing = true; if (vdo_is_read_only(depot->vdo)) { @@ -351,7 +351,7 @@ static void launch_write(struct slab_summary_block *block) * @is_clean: Whether the slab is clean. * @free_blocks: The number of free blocks. */ -static void update_slab_summary_entry(struct vdo_slab *slab, struct waiter *waiter, +static void update_slab_summary_entry(struct vdo_slab *slab, struct vdo_waiter *waiter, tail_block_offset_t tail_block_offset, bool load_ref_counts, bool is_clean, block_count_t free_blocks) @@ -382,7 +382,7 @@ static void update_slab_summary_entry(struct vdo_slab *slab, struct waiter *wait .is_dirty = !is_clean, .fullness_hint = compute_fullness_hint(allocator->depot, free_blocks), }; - vdo_enqueue_waiter(&block->next_update_waiters, waiter); + vdo_waitq_enqueue_waiter(&block->next_update_waiters, waiter); launch_write(block); } @@ -441,7 +441,7 @@ static void flush_endio(struct bio *bio) * @waiter: The journal as a flush waiter. * @context: The newly acquired flush vio. */ -static void flush_for_reaping(struct waiter *waiter, void *context) +static void flush_for_reaping(struct vdo_waiter *waiter, void *context) { struct slab_journal *journal = container_of(waiter, struct slab_journal, flush_waiter); @@ -550,7 +550,7 @@ static void adjust_slab_journal_block_reference(struct slab_journal *journal, * * Implements waiter_callback_fn. */ -static void release_journal_locks(struct waiter *waiter, void *context) +static void release_journal_locks(struct vdo_waiter *waiter, void *context) { sequence_number_t first, i; struct slab_journal *journal = @@ -734,7 +734,7 @@ static void write_slab_journal_endio(struct bio *bio) * * Callback from acquire_vio_from_pool() registered in commit_tail(). */ -static void write_slab_journal_block(struct waiter *waiter, void *context) +static void write_slab_journal_block(struct vdo_waiter *waiter, void *context) { struct pooled_vio *pooled = context; struct vio *vio = &pooled->vio; @@ -1006,7 +1006,7 @@ static bool requires_reaping(const struct slab_journal *journal) } /** finish_summary_update() - A waiter callback that resets the writing state of a slab. */ -static void finish_summary_update(struct waiter *waiter, void *context) +static void finish_summary_update(struct vdo_waiter *waiter, void *context) { struct vdo_slab *slab = container_of(waiter, struct vdo_slab, summary_waiter); int result = *((int *) context); @@ -1021,7 +1021,7 @@ static void finish_summary_update(struct waiter *waiter, void *context) check_if_slab_drained(slab); } -static void write_reference_block(struct waiter *waiter, void *context); +static void write_reference_block(struct vdo_waiter *waiter, void *context); /** * launch_reference_block_write() - Launch the write of a dirty reference block by first acquiring @@ -1032,7 +1032,7 @@ static void write_reference_block(struct waiter *waiter, void *context); * This can be asynchronous since the writer will have to wait if all VIOs in the pool are * currently in use. */ -static void launch_reference_block_write(struct waiter *waiter, void *context) +static void launch_reference_block_write(struct vdo_waiter *waiter, void *context) { struct vdo_slab *slab = context; @@ -1047,7 +1047,8 @@ static void launch_reference_block_write(struct waiter *waiter, void *context) static void save_dirty_reference_blocks(struct vdo_slab *slab) { - vdo_notify_all_waiters(&slab->dirty_blocks, launch_reference_block_write, slab); + vdo_waitq_notify_all_waiters(&slab->dirty_blocks, + launch_reference_block_write, slab); check_if_slab_drained(slab); } @@ -1084,7 +1085,7 @@ static void finish_reference_block_write(struct vdo_completion *completion) /* Re-queue the block if it was re-dirtied while it was writing. */ if (block->is_dirty) { - vdo_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); + vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); if (vdo_is_state_draining(&slab->state)) { /* We must be saving, and this block will otherwise not be relaunched. */ save_dirty_reference_blocks(slab); @@ -1097,7 +1098,7 @@ static void finish_reference_block_write(struct vdo_completion *completion) * Mark the slab as clean in the slab summary if there are no dirty or writing blocks * and no summary update in progress. */ - if ((slab->active_count > 0) || vdo_has_waiters(&slab->dirty_blocks)) { + if ((slab->active_count > 0) || vdo_waitq_has_waiters(&slab->dirty_blocks)) { check_if_slab_drained(slab); return; } @@ -1175,7 +1176,7 @@ static void handle_io_error(struct vdo_completion *completion) * @waiter: The waiter of the dirty block. * @context: The VIO returned by the pool. */ -static void write_reference_block(struct waiter *waiter, void *context) +static void write_reference_block(struct vdo_waiter *waiter, void *context) { size_t block_offset; physical_block_number_t pbn; @@ -1213,7 +1214,7 @@ static void reclaim_journal_space(struct slab_journal *journal) { block_count_t length = journal_length(journal); struct vdo_slab *slab = journal->slab; - block_count_t write_count = vdo_count_waiters(&slab->dirty_blocks); + block_count_t write_count = vdo_waitq_num_waiters(&slab->dirty_blocks); block_count_t written; if ((length < journal->flushing_threshold) || (write_count == 0)) @@ -1228,8 +1229,8 @@ static void reclaim_journal_space(struct slab_journal *journal) } for (written = 0; written < write_count; written++) { - vdo_notify_next_waiter(&slab->dirty_blocks, - launch_reference_block_write, slab); + vdo_waitq_notify_next_waiter(&slab->dirty_blocks, + launch_reference_block_write, slab); } } @@ -1263,7 +1264,7 @@ static void dirty_block(struct reference_block *block) block->is_dirty = true; if (!block->is_writing) - vdo_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); + vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); } /** @@ -1678,7 +1679,7 @@ static int __must_check adjust_reference_count(struct vdo_slab *slab, * This callback is invoked by add_entries() once it has determined that we are ready to make * another entry in the slab journal. Implements waiter_callback_fn. */ -static void add_entry_from_waiter(struct waiter *waiter, void *context) +static void add_entry_from_waiter(struct vdo_waiter *waiter, void *context) { int result; struct reference_updater *updater = @@ -1744,7 +1745,7 @@ static void add_entry_from_waiter(struct waiter *waiter, void *context) */ static inline bool is_next_entry_a_block_map_increment(struct slab_journal *journal) { - struct waiter *waiter = vdo_get_first_waiter(&journal->entry_waiters); + struct vdo_waiter *waiter = vdo_waitq_get_first_waiter(&journal->entry_waiters); struct reference_updater *updater = container_of(waiter, struct reference_updater, waiter); @@ -1767,7 +1768,7 @@ static void add_entries(struct slab_journal *journal) } journal->adding_entries = true; - while (vdo_has_waiters(&journal->entry_waiters)) { + while (vdo_waitq_has_waiters(&journal->entry_waiters)) { struct slab_journal_block_header *header = &journal->tail_header; if (journal->partial_write_in_progress || @@ -1864,8 +1865,8 @@ static void add_entries(struct slab_journal *journal) } } - vdo_notify_next_waiter(&journal->entry_waiters, - add_entry_from_waiter, journal); + vdo_waitq_notify_next_waiter(&journal->entry_waiters, + add_entry_from_waiter, journal); } journal->adding_entries = false; @@ -1873,7 +1874,7 @@ static void add_entries(struct slab_journal *journal) /* If there are no waiters, and we are flushing or saving, commit the tail block. */ if (vdo_is_state_draining(&journal->slab->state) && !vdo_is_state_suspending(&journal->slab->state) && - !vdo_has_waiters(&journal->entry_waiters)) + !vdo_waitq_has_waiters(&journal->entry_waiters)) commit_tail(journal); } @@ -2259,7 +2260,7 @@ static void load_reference_block_endio(struct bio *bio) * @waiter: The waiter of the block to load. * @context: The VIO returned by the pool. */ -static void load_reference_block(struct waiter *waiter, void *context) +static void load_reference_block(struct vdo_waiter *waiter, void *context) { struct pooled_vio *pooled = context; struct vio *vio = &pooled->vio; @@ -2284,7 +2285,7 @@ static void load_reference_blocks(struct vdo_slab *slab) slab->free_blocks = slab->block_count; slab->active_count = slab->reference_block_count; for (i = 0; i < slab->reference_block_count; i++) { - struct waiter *waiter = &slab->reference_blocks[i].waiter; + struct vdo_waiter *waiter = &slab->reference_blocks[i].waiter; waiter->callback = load_reference_block; acquire_vio_from_pool(slab->allocator->vio_pool, waiter); @@ -2455,7 +2456,7 @@ static void handle_load_error(struct vdo_completion *completion) * * This is the success callback from acquire_vio_from_pool() when loading a slab journal. */ -static void read_slab_journal_tail(struct waiter *waiter, void *context) +static void read_slab_journal_tail(struct vdo_waiter *waiter, void *context) { struct slab_journal *journal = container_of(waiter, struct slab_journal, resource_waiter); @@ -2662,7 +2663,7 @@ static void uninitialize_scrubber_vio(struct slab_scrubber *scrubber) */ static void finish_scrubbing(struct slab_scrubber *scrubber, int result) { - bool notify = vdo_has_waiters(&scrubber->waiters); + bool notify = vdo_waitq_has_waiters(&scrubber->waiters); bool done = !has_slabs_to_scrub(scrubber); struct block_allocator *allocator = container_of(scrubber, struct block_allocator, scrubber); @@ -2709,7 +2710,7 @@ static void finish_scrubbing(struct slab_scrubber *scrubber, int result) * Fortunately if there were waiters, we can't have been freed yet. */ if (notify) - vdo_notify_all_waiters(&scrubber->waiters, NULL, NULL); + vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL); } static void scrub_next_slab(struct slab_scrubber *scrubber); @@ -2933,7 +2934,7 @@ static void scrub_next_slab(struct slab_scrubber *scrubber) * Note: this notify call is always safe only because scrubbing can only be started when * the VDO is quiescent. */ - vdo_notify_all_waiters(&scrubber->waiters, NULL, NULL); + vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL); if (vdo_is_read_only(completion->vdo)) { finish_scrubbing(scrubber, VDO_READ_ONLY); @@ -3053,7 +3054,7 @@ static struct vdo_slab *next_slab(struct slab_iterator *iterator) * This callback is invoked on all vios waiting to make slab journal entries after the VDO has gone * into read-only mode. Implements waiter_callback_fn. */ -static void abort_waiter(struct waiter *waiter, void *context __always_unused) +static void abort_waiter(struct vdo_waiter *waiter, void *context __always_unused) { struct reference_updater *updater = container_of(waiter, struct reference_updater, waiter); @@ -3079,8 +3080,8 @@ static void notify_block_allocator_of_read_only_mode(void *listener, while (iterator.next != NULL) { struct vdo_slab *slab = next_slab(&iterator); - vdo_notify_all_waiters(&slab->journal.entry_waiters, - abort_waiter, &slab->journal); + vdo_waitq_notify_all_waiters(&slab->journal.entry_waiters, + abort_waiter, &slab->journal); check_if_slab_drained(slab); } @@ -3210,7 +3211,7 @@ int vdo_allocate_block(struct block_allocator *allocator, * some other error otherwise. */ int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, - struct waiter *waiter) + struct vdo_waiter *waiter) { if (vdo_is_read_only(allocator->depot->vdo)) return VDO_READ_ONLY; @@ -3218,7 +3219,7 @@ int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, if (vdo_is_state_quiescent(&allocator->scrubber.admin_state)) return VDO_NO_SPACE; - vdo_enqueue_waiter(&allocator->scrubber.waiters, waiter); + vdo_waitq_enqueue_waiter(&allocator->scrubber.waiters, waiter); return VDO_SUCCESS; } @@ -3244,7 +3245,7 @@ void vdo_modify_reference_count(struct vdo_completion *completion, return; } - vdo_enqueue_waiter(&slab->journal.entry_waiters, &updater->waiter); + vdo_waitq_enqueue_waiter(&slab->journal.entry_waiters, &updater->waiter); if ((slab->status != VDO_SLAB_REBUILT) && requires_reaping(&slab->journal)) register_slab_for_scrubbing(slab, true); @@ -3587,7 +3588,7 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) } uds_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", - vdo_count_waiters(&journal->entry_waiters), + vdo_waitq_num_waiters(&journal->entry_waiters), uds_bool_to_string(journal->waiting_to_commit), uds_bool_to_string(journal->updating_slab_summary), (unsigned long long) journal->head, @@ -3608,7 +3609,7 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) uds_log_info(" slab: free=%u/%u blocks=%u dirty=%zu active=%zu journal@(%llu,%u)", slab->free_blocks, slab->block_count, slab->reference_block_count, - vdo_count_waiters(&slab->dirty_blocks), + vdo_waitq_num_waiters(&slab->dirty_blocks), slab->active_count, (unsigned long long) slab->slab_journal_point.sequence_number, slab->slab_journal_point.entry_count); @@ -3628,7 +3629,7 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) uds_log_info("slab_scrubber slab_count %u waiters %zu %s%s", READ_ONCE(scrubber->slab_count), - vdo_count_waiters(&scrubber->waiters), + vdo_waitq_num_waiters(&scrubber->waiters), vdo_get_admin_state_code(&scrubber->admin_state)->name, scrubber->high_priority_only ? ", high_priority_only " : ""); } diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h index 169021b0811a..efdef566709a 100644 --- a/drivers/md/dm-vdo/slab-depot.h +++ b/drivers/md/dm-vdo/slab-depot.h @@ -60,13 +60,13 @@ struct journal_lock { struct slab_journal { /* A waiter object for getting a VIO pool entry */ - struct waiter resource_waiter; + struct vdo_waiter resource_waiter; /* A waiter object for updating the slab summary */ - struct waiter slab_summary_waiter; + struct vdo_waiter slab_summary_waiter; /* A waiter object for getting a vio with which to flush */ - struct waiter flush_waiter; + struct vdo_waiter flush_waiter; /* The queue of VIOs waiting to make an entry */ - struct wait_queue entry_waiters; + struct vdo_wait_queue entry_waiters; /* The parent slab reference of this journal */ struct vdo_slab *slab; @@ -149,7 +149,7 @@ struct slab_journal { */ struct reference_block { /* This block waits on the ref_counts to tell it to write */ - struct waiter waiter; + struct vdo_waiter waiter; /* The slab to which this reference_block belongs */ struct vdo_slab *slab; /* The number of references in this block that represent allocations */ @@ -241,12 +241,12 @@ struct vdo_slab { struct search_cursor search_cursor; /* A list of the dirty blocks waiting to be written out */ - struct wait_queue dirty_blocks; + struct vdo_wait_queue dirty_blocks; /* The number of blocks which are currently writing */ size_t active_count; /* A waiter object for updating the slab summary */ - struct waiter summary_waiter; + struct vdo_waiter summary_waiter; /* The latest slab journal for which there has been a reference count update */ struct journal_point slab_journal_point; @@ -271,7 +271,7 @@ struct slab_scrubber { /* The queue of slabs to scrub once there are no high_priority_slabs */ struct list_head slabs; /* The queue of VIOs waiting for a slab to be scrubbed */ - struct wait_queue waiters; + struct vdo_wait_queue waiters; /* * The number of slabs that are unrecovered or being scrubbed. This field is modified by @@ -341,9 +341,9 @@ struct slab_summary_block { /* Whether this block has a write outstanding */ bool writing; /* Ring of updates waiting on the outstanding write */ - struct wait_queue current_update_waiters; + struct vdo_wait_queue current_update_waiters; /* Ring of updates waiting on the next write */ - struct wait_queue next_update_waiters; + struct vdo_wait_queue next_update_waiters; /* The active slab_summary_entry array for this block */ struct slab_summary_entry *entries; /* The vio used to write this block */ @@ -522,7 +522,7 @@ int __must_check vdo_allocate_block(struct block_allocator *allocator, physical_block_number_t *block_number_ptr); int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, - struct waiter *waiter); + struct vdo_waiter *waiter); void vdo_modify_reference_count(struct vdo_completion *completion, struct reference_updater *updater); diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index f83b56acc8e4..6acaba149c75 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -25,7 +25,7 @@ struct vio_pool { /** The list of objects which are available */ struct list_head available; /** The queue of requestors waiting for objects from the pool */ - struct wait_queue waiting; + struct vdo_wait_queue waiting; /** The number of objects currently in use */ size_t busy_count; /** The list of objects which are in use */ @@ -364,7 +364,7 @@ void free_vio_pool(struct vio_pool *pool) return; /* Remove all available vios from the object pool. */ - ASSERT_LOG_ONLY(!vdo_has_waiters(&pool->waiting), + ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&pool->waiting), "VIO pool must not have any waiters when being freed"); ASSERT_LOG_ONLY((pool->busy_count == 0), "VIO pool must not have %zu busy entries when being freed", @@ -400,7 +400,7 @@ bool is_vio_pool_busy(struct vio_pool *pool) * @pool: The vio pool. * @waiter: Object that is requesting a vio. */ -void acquire_vio_from_pool(struct vio_pool *pool, struct waiter *waiter) +void acquire_vio_from_pool(struct vio_pool *pool, struct vdo_waiter *waiter) { struct pooled_vio *pooled; @@ -408,7 +408,7 @@ void acquire_vio_from_pool(struct vio_pool *pool, struct waiter *waiter) "acquire from active vio_pool called from correct thread"); if (list_empty(&pool->available)) { - vdo_enqueue_waiter(&pool->waiting, waiter); + vdo_waitq_enqueue_waiter(&pool->waiting, waiter); return; } @@ -430,8 +430,8 @@ void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio) vio->vio.completion.error_handler = NULL; vio->vio.completion.parent = NULL; - if (vdo_has_waiters(&pool->waiting)) { - vdo_notify_next_waiter(&pool->waiting, NULL, vio); + if (vdo_waitq_has_waiters(&pool->waiting)) { + vdo_waitq_notify_next_waiter(&pool->waiting, NULL, vio); return; } diff --git a/drivers/md/dm-vdo/vio.h b/drivers/md/dm-vdo/vio.h index 3c72fded69b0..71585424f85b 100644 --- a/drivers/md/dm-vdo/vio.h +++ b/drivers/md/dm-vdo/vio.h @@ -193,7 +193,7 @@ int __must_check make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t th void *context, struct vio_pool **pool_ptr); void free_vio_pool(struct vio_pool *pool); bool __must_check is_vio_pool_busy(struct vio_pool *pool); -void acquire_vio_from_pool(struct vio_pool *pool, struct waiter *waiter); +void acquire_vio_from_pool(struct vio_pool *pool, struct vdo_waiter *waiter); void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio); #endif /* VIO_H */ diff --git a/drivers/md/dm-vdo/wait-queue.c b/drivers/md/dm-vdo/wait-queue.c index 8acc24e79d2b..9c12a9893823 100644 --- a/drivers/md/dm-vdo/wait-queue.c +++ b/drivers/md/dm-vdo/wait-queue.c @@ -12,211 +12,213 @@ #include "status-codes.h" /** - * vdo_enqueue_waiter() - Add a waiter to the tail end of a wait queue. - * @queue: The queue to which to add the waiter. - * @waiter: The waiter to add to the queue. + * vdo_waitq_enqueue_waiter() - Add a waiter to the tail end of a waitq. + * @waitq: The vdo_wait_queue to which to add the waiter. + * @waiter: The waiter to add to the waitq. * - * The waiter must not already be waiting in a queue. - * - * Return: VDO_SUCCESS or an error code. + * The waiter must not already be waiting in a waitq. */ -void vdo_enqueue_waiter(struct wait_queue *queue, struct waiter *waiter) +void vdo_waitq_enqueue_waiter(struct vdo_wait_queue *waitq, struct vdo_waiter *waiter) { BUG_ON(waiter->next_waiter != NULL); - if (queue->last_waiter == NULL) { + if (waitq->last_waiter == NULL) { /* - * The queue is empty, so form the initial circular list by self-linking the + * The waitq is empty, so form the initial circular list by self-linking the * initial waiter. */ waiter->next_waiter = waiter; } else { - /* Splice the new waiter in at the end of the queue. */ - waiter->next_waiter = queue->last_waiter->next_waiter; - queue->last_waiter->next_waiter = waiter; + /* Splice the new waiter in at the end of the waitq. */ + waiter->next_waiter = waitq->last_waiter->next_waiter; + waitq->last_waiter->next_waiter = waiter; } /* In both cases, the waiter we added to the ring becomes the last waiter. */ - queue->last_waiter = waiter; - queue->queue_length += 1; + waitq->last_waiter = waiter; + waitq->length += 1; } /** - * vdo_transfer_all_waiters() - Transfer all waiters from one wait queue to a second queue, - * emptying the first queue. - * @from_queue: The queue containing the waiters to move. - * @to_queue: The queue that will receive the waiters from the first queue. + * vdo_waitq_transfer_all_waiters() - Transfer all waiters from one waitq to + * a second waitq, emptying the first waitq. + * @from_waitq: The waitq containing the waiters to move. + * @to_waitq: The waitq that will receive the waiters from the first waitq. */ -void vdo_transfer_all_waiters(struct wait_queue *from_queue, struct wait_queue *to_queue) +void vdo_waitq_transfer_all_waiters(struct vdo_wait_queue *from_waitq, + struct vdo_wait_queue *to_waitq) { - /* If the source queue is empty, there's nothing to do. */ - if (!vdo_has_waiters(from_queue)) + /* If the source waitq is empty, there's nothing to do. */ + if (!vdo_waitq_has_waiters(from_waitq)) return; - if (vdo_has_waiters(to_queue)) { + if (vdo_waitq_has_waiters(to_waitq)) { /* - * Both queues are non-empty. Splice the two circular lists together by swapping - * the next (head) pointers in the list tails. + * Both are non-empty. Splice the two circular lists together + * by swapping the next (head) pointers in the list tails. */ - struct waiter *from_head = from_queue->last_waiter->next_waiter; - struct waiter *to_head = to_queue->last_waiter->next_waiter; + struct vdo_waiter *from_head = from_waitq->last_waiter->next_waiter; + struct vdo_waiter *to_head = to_waitq->last_waiter->next_waiter; - to_queue->last_waiter->next_waiter = from_head; - from_queue->last_waiter->next_waiter = to_head; + to_waitq->last_waiter->next_waiter = from_head; + from_waitq->last_waiter->next_waiter = to_head; } - to_queue->last_waiter = from_queue->last_waiter; - to_queue->queue_length += from_queue->queue_length; - vdo_initialize_wait_queue(from_queue); + to_waitq->last_waiter = from_waitq->last_waiter; + to_waitq->length += from_waitq->length; + vdo_waitq_init(from_waitq); } /** - * vdo_notify_all_waiters() - Notify all the entries waiting in a queue. - * @queue: The wait queue containing the waiters to notify. + * vdo_waitq_notify_all_waiters() - Notify all the entries waiting in a waitq. + * @waitq: The vdo_wait_queue containing the waiters to notify. * @callback: The function to call to notify each waiter, or NULL to invoke the callback field * registered in each waiter. * @context: The context to pass to the callback function. * - * Notifies all the entries waiting in a queue to continue execution by invoking a callback - * function on each of them in turn. The queue is copied and emptied before invoking any callbacks, - * and only the waiters that were in the queue at the start of the call will be notified. + * Notifies all the entries waiting in a waitq to continue execution by invoking a callback + * function on each of them in turn. The waitq is copied and emptied before invoking any callbacks, + * and only the waiters that were in the waitq at the start of the call will be notified. */ -void vdo_notify_all_waiters(struct wait_queue *queue, waiter_callback_fn callback, - void *context) +void vdo_waitq_notify_all_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context) { /* - * Copy and empty the queue first, avoiding the possibility of an infinite loop if entries - * are returned to the queue by the callback function. + * Copy and empty the waitq first, avoiding the possibility of an infinite + * loop if entries are returned to the waitq by the callback function. */ - struct wait_queue waiters; + struct vdo_wait_queue waiters; - vdo_initialize_wait_queue(&waiters); - vdo_transfer_all_waiters(queue, &waiters); + vdo_waitq_init(&waiters); + vdo_waitq_transfer_all_waiters(waitq, &waiters); - /* Drain the copied queue, invoking the callback on every entry. */ - while (vdo_has_waiters(&waiters)) - vdo_notify_next_waiter(&waiters, callback, context); + /* Drain the copied waitq, invoking the callback on every entry. */ + while (vdo_waitq_has_waiters(&waiters)) + vdo_waitq_notify_next_waiter(&waiters, callback, context); } /** - * vdo_get_first_waiter() - Return the waiter that is at the head end of a wait queue. - * @queue: The queue from which to get the first waiter. + * vdo_waitq_get_first_waiter() - Return the waiter that is at the head end of a waitq. + * @waitq: The vdo_wait_queue from which to get the first waiter. * - * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. + * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty. */ -struct waiter *vdo_get_first_waiter(const struct wait_queue *queue) +struct vdo_waiter *vdo_waitq_get_first_waiter(const struct vdo_wait_queue *waitq) { - struct waiter *last_waiter = queue->last_waiter; + struct vdo_waiter *last_waiter = waitq->last_waiter; if (last_waiter == NULL) { /* There are no waiters, so we're done. */ return NULL; } - /* The queue is circular, so the last entry links to the head of the queue. */ + /* The waitq is circular, so the last entry links to the head of the waitq. */ return last_waiter->next_waiter; } /** - * vdo_dequeue_matching_waiters() - Remove all waiters that match based on the specified matching - * method and append them to a wait_queue. - * @queue: The wait queue to process. - * @match_method: The method to determine matching. + * vdo_waitq_dequeue_matching_waiters() - Remove all waiters that match based on the specified + * matching method and append them to a vdo_wait_queue. + * @waitq: The vdo_wait_queue to process. + * @waiter_match: The method to determine matching. * @match_context: Contextual info for the match method. - * @matched_queue: A wait_queue to store matches. + * @matched_waitq: A wait_waitq to store matches. */ -void vdo_dequeue_matching_waiters(struct wait_queue *queue, waiter_match_fn match_method, - void *match_context, struct wait_queue *matched_queue) +void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_match_fn waiter_match, + void *match_context, + struct vdo_wait_queue *matched_waitq) { - struct wait_queue matched_waiters, iteration_queue; + // FIXME: copying a waitq just to iterate it, with matching, is unfortunate + struct vdo_wait_queue matched_waiters, iteration_waitq; - vdo_initialize_wait_queue(&matched_waiters); + vdo_waitq_init(&matched_waiters); + vdo_waitq_init(&iteration_waitq); + vdo_waitq_transfer_all_waiters(waitq, &iteration_waitq); - vdo_initialize_wait_queue(&iteration_queue); - vdo_transfer_all_waiters(queue, &iteration_queue); - while (vdo_has_waiters(&iteration_queue)) { - struct waiter *waiter = vdo_dequeue_next_waiter(&iteration_queue); + while (vdo_waitq_has_waiters(&iteration_waitq)) { + struct vdo_waiter *waiter = vdo_waitq_dequeue_next_waiter(&iteration_waitq); - vdo_enqueue_waiter((match_method(waiter, match_context) ? - &matched_waiters : queue), waiter); + vdo_waitq_enqueue_waiter((waiter_match(waiter, match_context) ? + &matched_waiters : waitq), waiter); } - vdo_transfer_all_waiters(&matched_waiters, matched_queue); + vdo_waitq_transfer_all_waiters(&matched_waiters, matched_waitq); } /** - * vdo_dequeue_next_waiter() - Remove the first waiter from the head end of a wait queue. - * @queue: The wait queue from which to remove the first entry. + * vdo_waitq_dequeue_next_waiter() - Remove the first waiter from the head end of a waitq. + * @waitq: The vdo_wait_queue from which to remove the first entry. * * The caller will be responsible for waking the waiter by invoking the correct callback function * to resume its execution. * - * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. + * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty. */ -struct waiter *vdo_dequeue_next_waiter(struct wait_queue *queue) +struct vdo_waiter *vdo_waitq_dequeue_next_waiter(struct vdo_wait_queue *waitq) { - struct waiter *first_waiter = vdo_get_first_waiter(queue); - struct waiter *last_waiter = queue->last_waiter; + struct vdo_waiter *first_waiter = vdo_waitq_get_first_waiter(waitq); + struct vdo_waiter *last_waiter = waitq->last_waiter; if (first_waiter == NULL) return NULL; if (first_waiter == last_waiter) { - /* The queue has a single entry, so just empty it out by nulling the tail. */ - queue->last_waiter = NULL; + /* The waitq has a single entry, so just empty it out by nulling the tail. */ + waitq->last_waiter = NULL; } else { /* - * The queue has more than one entry, so splice the first waiter out of the - * circular queue. + * The waitq has more than one entry, so splice the first waiter out of the + * circular waitq. */ last_waiter->next_waiter = first_waiter->next_waiter; } - /* The waiter is no longer in a wait queue. */ + /* The waiter is no longer in a waitq. */ first_waiter->next_waiter = NULL; - queue->queue_length -= 1; + waitq->length -= 1; return first_waiter; } /** - * vdo_notify_next_waiter() - Notify the next entry waiting in a queue. - * @queue: The wait queue containing the waiter to notify. + * vdo_waitq_notify_next_waiter() - Notify the next entry waiting in a waitq. + * @waitq: The vdo_wait_queue containing the waiter to notify. * @callback: The function to call to notify the waiter, or NULL to invoke the callback field * registered in the waiter. * @context: The context to pass to the callback function. * - * Notifies the next entry waiting in a queue to continue execution by invoking a callback function - * on it after removing it from the queue. + * Notifies the next entry waiting in a waitq to continue execution by invoking a callback function + * on it after removing it from the waitq. * - * Return: true if there was a waiter in the queue. + * Return: true if there was a waiter in the waitq. */ -bool vdo_notify_next_waiter(struct wait_queue *queue, waiter_callback_fn callback, - void *context) +bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context) { - struct waiter *waiter = vdo_dequeue_next_waiter(queue); + struct vdo_waiter *waiter = vdo_waitq_dequeue_next_waiter(waitq); if (waiter == NULL) return false; if (callback == NULL) callback = waiter->callback; - (*callback)(waiter, context); + callback(waiter, context); return true; } /** - * vdo_get_next_waiter() - Get the waiter after this one, for debug iteration. - * @queue: The wait queue. + * vdo_waitq_get_next_waiter() - Get the waiter after this one, for debug iteration. + * @waitq: The vdo_wait_queue. * @waiter: A waiter. * * Return: The next waiter, or NULL. */ -const struct waiter *vdo_get_next_waiter(const struct wait_queue *queue, - const struct waiter *waiter) +const struct vdo_waiter *vdo_waitq_get_next_waiter(const struct vdo_wait_queue *waitq, + const struct vdo_waiter *waiter) { - struct waiter *first_waiter = vdo_get_first_waiter(queue); + struct vdo_waiter *first_waiter = vdo_waitq_get_first_waiter(waitq); if (waiter == NULL) return first_waiter; diff --git a/drivers/md/dm-vdo/wait-queue.h b/drivers/md/dm-vdo/wait-queue.h index 50f1e2a1ea67..b92f12dd5b4b 100644 --- a/drivers/md/dm-vdo/wait-queue.h +++ b/drivers/md/dm-vdo/wait-queue.h @@ -10,122 +10,132 @@ #include /** - * DOC: Wait queues. + * A vdo_wait_queue is a circular singly linked list of entries waiting to be notified + * of a change in a condition. Keeping a circular list allows the vdo_wait_queue + * structure to simply be a pointer to the tail (newest) entry, supporting + * constant-time enqueue and dequeue operations. A null pointer is an empty waitq. * - * A wait queue is a circular list of entries waiting to be notified of a change in a condition. - * Keeping a circular list allows the queue structure to simply be a pointer to the tail (newest) - * entry in the queue, supporting constant-time enqueue and dequeue operations. A null pointer is - * an empty queue. + * An empty waitq: + * waitq0.last_waiter -> NULL * - * An empty queue: - * queue0.last_waiter -> NULL + * A singleton waitq: + * waitq1.last_waiter -> entry1 -> entry1 -> [...] * - * A singleton queue: - * queue1.last_waiter -> entry1 -> entry1 -> [...] + * A three-element waitq: + * waitq2.last_waiter -> entry3 -> entry1 -> entry2 -> entry3 -> [...] * - * A three-element queue: - * queue2.last_waiter -> entry3 -> entry1 -> entry2 -> entry3 -> [...] + * linux/wait.h's wait_queue_head is _not_ used because vdo_wait_queue's + * interface is much less complex (doesn't need locking, priorities or timers). + * Made possible by vdo's thread-based resource allocation and locking; and + * the polling nature of vdo_wait_queue consumers. + * + * FIXME: could be made to use a linux/list.h's list_head but its extra barriers + * really aren't needed. Nor is a doubly linked list, but vdo_wait_queue could + * make use of __list_del_clearprev() -- but that would compromise the ability + * to make full use of linux's list interface. */ -struct waiter; +struct vdo_waiter; -struct wait_queue { +struct vdo_wait_queue { /* The tail of the queue, the last (most recently added) entry */ - struct waiter *last_waiter; + struct vdo_waiter *last_waiter; /* The number of waiters currently in the queue */ - size_t queue_length; + size_t length; }; /** - * typedef waiter_callback_fn - Callback type for functions which will be called to resume - * processing of a waiter after it has been removed from its wait - * queue. + * vdo_waiter_callback_fn - Callback type that will be called to resume processing + * of a waiter after it has been removed from its wait queue. */ -typedef void (*waiter_callback_fn)(struct waiter *waiter, void *context); +typedef void (*vdo_waiter_callback_fn)(struct vdo_waiter *waiter, void *context); /** - * typedef waiter_match_fn - Method type for waiter matching methods. + * vdo_waiter_match_fn - Method type for waiter matching methods. * - * A waiter_match_fn method returns false if the waiter does not match. + * Returns false if the waiter does not match. */ -typedef bool (*waiter_match_fn)(struct waiter *waiter, void *context); +typedef bool (*vdo_waiter_match_fn)(struct vdo_waiter *waiter, void *context); -/* The queue entry structure for entries in a wait_queue. */ -struct waiter { +/* The structure for entries in a vdo_wait_queue. */ +struct vdo_waiter { /* - * The next waiter in the queue. If this entry is the last waiter, then this is actually a - * pointer back to the head of the queue. + * The next waiter in the waitq. If this entry is the last waiter, then this + * is actually a pointer back to the head of the waitq. */ - struct waiter *next_waiter; + struct vdo_waiter *next_waiter; - /* Optional waiter-specific callback to invoke when waking this waiter. */ - waiter_callback_fn callback; + /* Optional waiter-specific callback to invoke when dequeuing this waiter. */ + vdo_waiter_callback_fn callback; }; /** - * is_waiting() - Check whether a waiter is waiting. + * vdo_waiter_is_waiting() - Check whether a waiter is waiting. * @waiter: The waiter to check. * - * Return: true if the waiter is on some wait_queue. + * Return: true if the waiter is on some vdo_wait_queue. */ -static inline bool vdo_is_waiting(struct waiter *waiter) +static inline bool vdo_waiter_is_waiting(struct vdo_waiter *waiter) { return (waiter->next_waiter != NULL); } /** - * initialize_wait_queue() - Initialize a wait queue. - * @queue: The queue to initialize. + * vdo_waitq_init() - Initialize a vdo_wait_queue. + * @waitq: The vdo_wait_queue to initialize. */ -static inline void vdo_initialize_wait_queue(struct wait_queue *queue) +static inline void vdo_waitq_init(struct vdo_wait_queue *waitq) { - *queue = (struct wait_queue) { + *waitq = (struct vdo_wait_queue) { .last_waiter = NULL, - .queue_length = 0, + .length = 0, }; } /** - * has_waiters() - Check whether a wait queue has any entries waiting in it. - * @queue: The queue to query. + * vdo_waitq_has_waiters() - Check whether a vdo_wait_queue has any entries waiting. + * @waitq: The vdo_wait_queue to query. * - * Return: true if there are any waiters in the queue. + * Return: true if there are any waiters in the waitq. */ -static inline bool __must_check vdo_has_waiters(const struct wait_queue *queue) +static inline bool __must_check vdo_waitq_has_waiters(const struct vdo_wait_queue *waitq) { - return (queue->last_waiter != NULL); + return (waitq->last_waiter != NULL); } -void vdo_enqueue_waiter(struct wait_queue *queue, struct waiter *waiter); +void vdo_waitq_enqueue_waiter(struct vdo_wait_queue *waitq, + struct vdo_waiter *waiter); -void vdo_notify_all_waiters(struct wait_queue *queue, waiter_callback_fn callback, - void *context); +void vdo_waitq_notify_all_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context); -bool vdo_notify_next_waiter(struct wait_queue *queue, waiter_callback_fn callback, - void *context); +bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context); -void vdo_transfer_all_waiters(struct wait_queue *from_queue, - struct wait_queue *to_queue); +void vdo_waitq_transfer_all_waiters(struct vdo_wait_queue *from_waitq, + struct vdo_wait_queue *to_waitq); -struct waiter *vdo_get_first_waiter(const struct wait_queue *queue); +struct vdo_waiter *vdo_waitq_get_first_waiter(const struct vdo_wait_queue *waitq); -void vdo_dequeue_matching_waiters(struct wait_queue *queue, waiter_match_fn match_method, - void *match_context, struct wait_queue *matched_queue); +void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_match_fn waiter_match, + void *match_context, + struct vdo_wait_queue *matched_waitq); -struct waiter *vdo_dequeue_next_waiter(struct wait_queue *queue); +struct vdo_waiter *vdo_waitq_dequeue_next_waiter(struct vdo_wait_queue *waitq); /** - * count_waiters() - Count the number of waiters in a wait queue. - * @queue: The wait queue to query. + * vdo_waitq_num_waiters() - Return the number of waiters in a vdo_wait_queue. + * @waitq: The vdo_wait_queue to query. * - * Return: The number of waiters in the queue. + * Return: The number of waiters in the waitq. */ -static inline size_t __must_check vdo_count_waiters(const struct wait_queue *queue) +static inline size_t __must_check vdo_waitq_num_waiters(const struct vdo_wait_queue *waitq) { - return queue->queue_length; + return waitq->length; } -const struct waiter * __must_check vdo_get_next_waiter(const struct wait_queue *queue, - const struct waiter *waiter); +const struct vdo_waiter * __must_check +vdo_waitq_get_next_waiter(const struct vdo_wait_queue *waitq, const struct vdo_waiter *waiter); #endif /* VDO_WAIT_QUEUE_H */ From cd1227dd8341dca720ec71ad7dd2824fbf60ed4e Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 20 Nov 2023 17:29:17 -0500 Subject: [PATCH 0595/1038] dm vdo wait-queue: remove unused debug function vdo_waitq_get_next_waiter Reviewed-by: Ken Raeburn Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/wait-queue.c | 18 ------------------ drivers/md/dm-vdo/wait-queue.h | 3 --- 2 files changed, 21 deletions(-) diff --git a/drivers/md/dm-vdo/wait-queue.c b/drivers/md/dm-vdo/wait-queue.c index 9c12a9893823..4231d3243fa1 100644 --- a/drivers/md/dm-vdo/wait-queue.c +++ b/drivers/md/dm-vdo/wait-queue.c @@ -207,21 +207,3 @@ bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq, return true; } - -/** - * vdo_waitq_get_next_waiter() - Get the waiter after this one, for debug iteration. - * @waitq: The vdo_wait_queue. - * @waiter: A waiter. - * - * Return: The next waiter, or NULL. - */ -const struct vdo_waiter *vdo_waitq_get_next_waiter(const struct vdo_wait_queue *waitq, - const struct vdo_waiter *waiter) -{ - struct vdo_waiter *first_waiter = vdo_waitq_get_first_waiter(waitq); - - if (waiter == NULL) - return first_waiter; - - return ((waiter->next_waiter != first_waiter) ? waiter->next_waiter : NULL); -} diff --git a/drivers/md/dm-vdo/wait-queue.h b/drivers/md/dm-vdo/wait-queue.h index b92f12dd5b4b..e514bdcf7d32 100644 --- a/drivers/md/dm-vdo/wait-queue.h +++ b/drivers/md/dm-vdo/wait-queue.h @@ -135,7 +135,4 @@ static inline size_t __must_check vdo_waitq_num_waiters(const struct vdo_wait_qu return waitq->length; } -const struct vdo_waiter * __must_check -vdo_waitq_get_next_waiter(const struct vdo_wait_queue *waitq, const struct vdo_waiter *waiter); - #endif /* VDO_WAIT_QUEUE_H */ From e752e5c33b960085f1ae12e391b18dd8fc5ceef1 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 20 Nov 2023 17:29:18 -0500 Subject: [PATCH 0596/1038] dm vdo wait-queue: optimize vdo_waitq_dequeue_matching_waiters Remove temporary 'matched_waiters' waitq and just enqueue matched waiters directly to the caller provided 'matched_waitq'. Reviewed-by: Ken Raeburn Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/wait-queue.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/md/dm-vdo/wait-queue.c b/drivers/md/dm-vdo/wait-queue.c index 4231d3243fa1..7e4cf9f03249 100644 --- a/drivers/md/dm-vdo/wait-queue.c +++ b/drivers/md/dm-vdo/wait-queue.c @@ -129,10 +129,8 @@ void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, void *match_context, struct vdo_wait_queue *matched_waitq) { - // FIXME: copying a waitq just to iterate it, with matching, is unfortunate - struct vdo_wait_queue matched_waiters, iteration_waitq; + struct vdo_wait_queue iteration_waitq; - vdo_waitq_init(&matched_waiters); vdo_waitq_init(&iteration_waitq); vdo_waitq_transfer_all_waiters(waitq, &iteration_waitq); @@ -140,10 +138,8 @@ void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, struct vdo_waiter *waiter = vdo_waitq_dequeue_next_waiter(&iteration_waitq); vdo_waitq_enqueue_waiter((waiter_match(waiter, match_context) ? - &matched_waiters : waitq), waiter); + matched_waitq : waitq), waiter); } - - vdo_waitq_transfer_all_waiters(&matched_waiters, matched_waitq); } /** From 29f0ef873c5ab5d8d59553447e4175e90287e731 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 20 Nov 2023 17:29:19 -0500 Subject: [PATCH 0597/1038] dm vdo block-map: optimize enter_zone_read_only_mode Rather than incrementally dequeue from the zone->flush_waiters vdo_wait_queue, simply re-initialize it. Reviewed-by: Ken Raeburn Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index a1f2c9d38192..7f9c4bc05f02 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -641,12 +641,10 @@ static void enter_zone_read_only_mode(struct block_map_zone *zone, int result) vdo_enter_read_only_mode(zone->block_map->vdo, result); /* - * We are in read-only mode, so we won't ever write any page out. Just take all waiters off - * the queue so the zone can drain. + * We are in read-only mode, so we won't ever write any page out. + * Just take all waiters off the waitq so the zone can drain. */ - while (vdo_waitq_has_waiters(&zone->flush_waiters)) - vdo_waitq_dequeue_next_waiter(&zone->flush_waiters); - + vdo_waitq_init(&zone->flush_waiters); check_for_drain_complete(zone); } From a4bba246ecdea4bf73602dcedbc9091656d63872 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 20 Nov 2023 17:29:20 -0500 Subject: [PATCH 0598/1038] dm vdo wait-queue: rename to vdo_waitq_dequeue_waiter Rename vdo_waitq_dequeue_next_waiter to vdo_waitq_dequeue_waiter. The "next" aspect of returned waiter is implied. "next" also isn't informative ("oldest" would be). Removing "next_" adds symmetry to vdo_waitq_enqueue_waiter(). Also fix whitespace and comments from previous waitq commit. Reviewed-by: Ken Raeburn Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 7 +++---- drivers/md/dm-vdo/data-vio.c | 2 +- drivers/md/dm-vdo/dedupe.c | 4 ++-- drivers/md/dm-vdo/flush.c | 4 ++-- drivers/md/dm-vdo/recovery-journal.c | 2 +- drivers/md/dm-vdo/wait-queue.c | 18 +++++++++--------- drivers/md/dm-vdo/wait-queue.h | 4 ++-- 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 7f9c4bc05f02..c5cb9da5d33e 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -913,7 +913,7 @@ static void allocate_free_page(struct page_info *info) /* * Remove all entries which match the page number in question and push them onto the page - * info's wait queue. + * info's waitq. */ vdo_waitq_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page, &pbn, &info->waiting); @@ -1593,9 +1593,8 @@ static void finish_page_write(struct vdo_completion *completion) enqueue_page(page, zone); } else if ((zone->flusher == NULL) && vdo_waitq_has_waiters(&zone->flush_waiters) && attempt_increment(zone)) { - zone->flusher = - container_of(vdo_waitq_dequeue_next_waiter(&zone->flush_waiters), - struct tree_page, waiter); + zone->flusher = container_of(vdo_waitq_dequeue_waiter(&zone->flush_waiters), + struct tree_page, waiter); write_page(zone->flusher, pooled); return; } diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 821155ca3761..711396e7a77d 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -1191,7 +1191,7 @@ static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) /* Another data_vio is waiting for the lock, transfer it in a single lock map operation. */ next_lock_holder = - vdo_waiter_as_data_vio(vdo_waitq_dequeue_next_waiter(&lock->waiters)); + vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&lock->waiters)); /* Transfer the remaining lock waiters to the next lock holder. */ vdo_waitq_transfer_all_waiters(&lock->waiters, diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 02e36896ca3c..f882d56581dc 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -413,14 +413,14 @@ static void set_duplicate_lock(struct hash_lock *hash_lock, struct pbn_lock *pbn } /** - * dequeue_lock_waiter() - Remove the first data_vio from the lock's wait queue and return it. + * dequeue_lock_waiter() - Remove the first data_vio from the lock's waitq and return it. * @lock: The lock containing the wait queue. * * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. */ static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock) { - return vdo_waiter_as_data_vio(vdo_waitq_dequeue_next_waiter(&lock->waiters)); + return vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&lock->waiters)); } /** diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index e7195c677773..a6eeb425d721 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -196,7 +196,7 @@ static void finish_notification(struct vdo_completion *completion) assert_on_flusher_thread(flusher, __func__); vdo_waitq_enqueue_waiter(&flusher->pending_flushes, - vdo_waitq_dequeue_next_waiter(&flusher->notifiers)); + vdo_waitq_dequeue_waiter(&flusher->notifiers)); vdo_complete_flushes(flusher); if (vdo_waitq_has_waiters(&flusher->notifiers)) notify_flush(flusher); @@ -335,7 +335,7 @@ void vdo_complete_flushes(struct flusher *flusher) "acknowledged next expected flush, %llu, was: %llu", (unsigned long long) flusher->first_unacknowledged_generation, (unsigned long long) flush->flush_generation); - vdo_waitq_dequeue_next_waiter(&flusher->pending_flushes); + vdo_waitq_dequeue_waiter(&flusher->pending_flushes); vdo_complete_flush(flush); flusher->first_unacknowledged_generation++; } diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 5126e670e97e..a6981e5dd017 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -1332,7 +1332,7 @@ static void add_queued_recovery_entries(struct recovery_journal_block *block) { while (vdo_waitq_has_waiters(&block->entry_waiters)) { struct data_vio *data_vio = - vdo_waiter_as_data_vio(vdo_waitq_dequeue_next_waiter(&block->entry_waiters)); + vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters)); struct tree_lock *lock = &data_vio->tree_lock; struct packed_recovery_journal_entry *packed_entry; struct recovery_journal_entry new_entry; diff --git a/drivers/md/dm-vdo/wait-queue.c b/drivers/md/dm-vdo/wait-queue.c index 7e4cf9f03249..6e1e739277ef 100644 --- a/drivers/md/dm-vdo/wait-queue.c +++ b/drivers/md/dm-vdo/wait-queue.c @@ -135,7 +135,7 @@ void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, vdo_waitq_transfer_all_waiters(waitq, &iteration_waitq); while (vdo_waitq_has_waiters(&iteration_waitq)) { - struct vdo_waiter *waiter = vdo_waitq_dequeue_next_waiter(&iteration_waitq); + struct vdo_waiter *waiter = vdo_waitq_dequeue_waiter(&iteration_waitq); vdo_waitq_enqueue_waiter((waiter_match(waiter, match_context) ? matched_waitq : waitq), waiter); @@ -143,15 +143,15 @@ void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, } /** - * vdo_waitq_dequeue_next_waiter() - Remove the first waiter from the head end of a waitq. + * vdo_waitq_dequeue_waiter() - Remove the first (oldest) waiter from a waitq. * @waitq: The vdo_wait_queue from which to remove the first entry. * - * The caller will be responsible for waking the waiter by invoking the correct callback function - * to resume its execution. + * The caller will be responsible for waking the waiter by continuing its + * execution appropriately. * * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty. */ -struct vdo_waiter *vdo_waitq_dequeue_next_waiter(struct vdo_wait_queue *waitq) +struct vdo_waiter *vdo_waitq_dequeue_waiter(struct vdo_wait_queue *waitq) { struct vdo_waiter *first_waiter = vdo_waitq_get_first_waiter(waitq); struct vdo_waiter *last_waiter = waitq->last_waiter; @@ -160,12 +160,12 @@ struct vdo_waiter *vdo_waitq_dequeue_next_waiter(struct vdo_wait_queue *waitq) return NULL; if (first_waiter == last_waiter) { - /* The waitq has a single entry, so just empty it out by nulling the tail. */ + /* The waitq has a single entry, so empty it by nulling the tail. */ waitq->last_waiter = NULL; } else { /* - * The waitq has more than one entry, so splice the first waiter out of the - * circular waitq. + * The waitq has multiple waiters, so splice the first waiter out + * of the circular waitq. */ last_waiter->next_waiter = first_waiter->next_waiter; } @@ -192,7 +192,7 @@ struct vdo_waiter *vdo_waitq_dequeue_next_waiter(struct vdo_wait_queue *waitq) bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq, vdo_waiter_callback_fn callback, void *context) { - struct vdo_waiter *waiter = vdo_waitq_dequeue_next_waiter(waitq); + struct vdo_waiter *waiter = vdo_waitq_dequeue_waiter(waitq); if (waiter == NULL) return false; diff --git a/drivers/md/dm-vdo/wait-queue.h b/drivers/md/dm-vdo/wait-queue.h index e514bdcf7d32..7e8ee6afe7c7 100644 --- a/drivers/md/dm-vdo/wait-queue.h +++ b/drivers/md/dm-vdo/wait-queue.h @@ -106,6 +106,8 @@ static inline bool __must_check vdo_waitq_has_waiters(const struct vdo_wait_queu void vdo_waitq_enqueue_waiter(struct vdo_wait_queue *waitq, struct vdo_waiter *waiter); +struct vdo_waiter *vdo_waitq_dequeue_waiter(struct vdo_wait_queue *waitq); + void vdo_waitq_notify_all_waiters(struct vdo_wait_queue *waitq, vdo_waiter_callback_fn callback, void *context); @@ -122,8 +124,6 @@ void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, void *match_context, struct vdo_wait_queue *matched_waitq); -struct vdo_waiter *vdo_waitq_dequeue_next_waiter(struct vdo_wait_queue *waitq); - /** * vdo_waitq_num_waiters() - Return the number of waiters in a vdo_wait_queue. * @waitq: The vdo_wait_queue to query. From db6b0a7ffeef58df70290b5d52bc469f7d9021d4 Mon Sep 17 00:00:00 2001 From: Bruce Johnston Date: Mon, 20 Nov 2023 17:29:54 -0500 Subject: [PATCH 0599/1038] dm vdo dedupe: switch to using int-map instead of pointer-map Use get_unaligned_le64() on the hash lock's record name to serve as the key to use with the int hash-map. Switching to using int hash-map removes the only consumer of pointer hash-map, as such it is removed. Reviewed-by: Matthew Sakai Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dedupe.c | 48 +-- drivers/md/dm-vdo/dedupe.h | 2 +- drivers/md/dm-vdo/pointer-map.c | 696 -------------------------------- drivers/md/dm-vdo/pointer-map.h | 81 ---- 4 files changed, 23 insertions(+), 804 deletions(-) delete mode 100644 drivers/md/dm-vdo/pointer-map.c delete mode 100644 drivers/md/dm-vdo/pointer-map.h diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index f882d56581dc..feb35b0d7be3 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -133,10 +133,10 @@ #include "completion.h" #include "constants.h" #include "data-vio.h" +#include "int-map.h" #include "io-submitter.h" #include "packer.h" #include "physical-zone.h" -#include "pointer-map.h" #include "slab-depot.h" #include "statistics.h" #include "types.h" @@ -370,6 +370,17 @@ struct pbn_lock *vdo_get_duplicate_lock(struct data_vio *data_vio) return data_vio->hash_lock->duplicate_lock; } +/** + * hash_lock_key() - Return hash_lock's record name as a hash code. + * @lock: The hash lock. + * + * Return: The key to use for the int map. + */ +static inline u64 hash_lock_key(struct hash_lock *lock) +{ + return get_unaligned_le64(&lock->hash.name); +} + /** * get_hash_lock_state_name() - Get the string representation of a hash lock state. * @state: The hash lock state. @@ -865,7 +876,7 @@ static int __must_check acquire_lock(struct hash_zone *zone, int result; /* - * Borrow and prepare a lock from the pool so we don't have to do two pointer_map accesses + * Borrow and prepare a lock from the pool so we don't have to do two int_map accesses * in the common case of no lock contention. */ result = ASSERT(!list_empty(&zone->lock_pool), @@ -882,8 +893,8 @@ static int __must_check acquire_lock(struct hash_zone *zone, */ new_lock->hash = *hash; - result = vdo_pointer_map_put(zone->hash_lock_map, &new_lock->hash, new_lock, - (replace_lock != NULL), (void **) &lock); + result = vdo_int_map_put(zone->hash_lock_map, hash_lock_key(new_lock), + new_lock, (replace_lock != NULL), (void **) &lock); if (result != VDO_SUCCESS) { return_hash_lock_to_pool(zone, uds_forget(new_lock)); return result; @@ -1904,6 +1915,7 @@ void vdo_acquire_hash_lock(struct vdo_completion *completion) */ void vdo_release_hash_lock(struct data_vio *data_vio) { + u64 lock_key; struct hash_lock *lock = data_vio->hash_lock; struct hash_zone *zone = data_vio->hash_zone; @@ -1917,14 +1929,15 @@ void vdo_release_hash_lock(struct data_vio *data_vio) return; } + lock_key = hash_lock_key(lock); if (lock->registered) { struct hash_lock *removed; - removed = vdo_pointer_map_remove(zone->hash_lock_map, &lock->hash); + removed = vdo_int_map_remove(zone->hash_lock_map, lock_key); ASSERT_LOG_ONLY(lock == removed, "hash lock being released must have been mapped"); } else { - ASSERT_LOG_ONLY(lock != vdo_pointer_map_get(zone->hash_lock_map, &lock->hash), + ASSERT_LOG_ONLY(lock != vdo_int_map_get(zone->hash_lock_map, lock_key), "unregistered hash lock must not be in the lock map"); } @@ -2011,22 +2024,6 @@ void vdo_share_compressed_write_lock(struct data_vio *data_vio, ASSERT_LOG_ONLY(claimed, "impossible to fail to claim an initial increment"); } -/** compare_keys() - Implements pointer_key_comparator_fn. */ -static bool compare_keys(const void *this_key, const void *that_key) -{ - /* Null keys are not supported. */ - return (memcmp(this_key, that_key, sizeof(struct uds_record_name)) == 0); -} - -/** hash_key() - Implements pointer_key_comparator_fn. */ -static u32 hash_key(const void *key) -{ - const struct uds_record_name *name = key; - - /* Use a fragment of the record name as a hash code. */ - return get_unaligned_le32(&name->name[4]); -} - static void dedupe_kobj_release(struct kobject *directory) { uds_free(container_of(directory, struct hash_zones, dedupe_directory)); @@ -2407,8 +2404,7 @@ static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zone data_vio_count_t i; struct hash_zone *zone = &zones->zones[zone_number]; - result = vdo_make_pointer_map(VDO_LOCK_MAP_CAPACITY, 0, compare_keys, - hash_key, &zone->hash_lock_map); + result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->hash_lock_map); if (result != VDO_SUCCESS) return result; @@ -2532,7 +2528,7 @@ void vdo_free_hash_zones(struct hash_zones *zones) struct hash_zone *zone = &zones->zones[i]; uds_free_funnel_queue(uds_forget(zone->timed_out_complete)); - vdo_free_pointer_map(uds_forget(zone->hash_lock_map)); + vdo_free_int_map(uds_forget(zone->hash_lock_map)); uds_free(uds_forget(zone->lock_array)); } @@ -2847,7 +2843,7 @@ static void dump_hash_zone(const struct hash_zone *zone) } uds_log_info("struct hash_zone %u: mapSize=%zu", - zone->zone_number, vdo_pointer_map_size(zone->hash_lock_map)); + zone->zone_number, vdo_int_map_size(zone->hash_lock_map)); for (i = 0; i < LOCK_POOL_CAPACITY; i++) dump_hash_lock(&zone->lock_array[i]); } diff --git a/drivers/md/dm-vdo/dedupe.h b/drivers/md/dm-vdo/dedupe.h index 90c5779bfe70..773dde5f9365 100644 --- a/drivers/md/dm-vdo/dedupe.h +++ b/drivers/md/dm-vdo/dedupe.h @@ -40,7 +40,7 @@ struct hash_zone { thread_id_t thread_id; /* Mapping from record name fields to hash_locks */ - struct pointer_map *hash_lock_map; + struct int_map *hash_lock_map; /* List containing all unused hash_locks */ struct list_head lock_pool; diff --git a/drivers/md/dm-vdo/pointer-map.c b/drivers/md/dm-vdo/pointer-map.c deleted file mode 100644 index 61eb2d3cd8e5..000000000000 --- a/drivers/md/dm-vdo/pointer-map.c +++ /dev/null @@ -1,696 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -/** - * DOC: - * - * Hash table implementation of a map from integers to pointers, implemented using the Hopscotch - * Hashing algorithm by Herlihy, Shavit, and Tzafrir (see - * http://en.wikipedia.org/wiki/Hopscotch_hashing). This implementation does not contain any of the - * locking/concurrency features of the algorithm, just the collision resolution scheme. - * - * Hopscotch Hashing is based on hashing with open addressing and linear probing. All the entries - * are stored in a fixed array of buckets, with no dynamic allocation for collisions. Unlike linear - * probing, all the entries that hash to a given bucket are stored within a fixed neighborhood - * starting at that bucket. Chaining is effectively represented as a bit vector relative to each - * bucket instead of as pointers or explicit offsets. - * - * When an empty bucket cannot be found within a given neighborhood, subsequent neighborhoods are - * searched, and one or more entries will "hop" into those neighborhoods. When this process works, - * an empty bucket will move into the desired neighborhood, allowing the entry to be added. When - * that process fails (typically when the buckets are around 90% full), the table must be resized - * and the all entries rehashed and added to the expanded table. - * - * Unlike linear probing, the number of buckets that must be searched in the worst case has a fixed - * upper bound (the size of the neighborhood). Those entries occupy a small number of memory cache - * lines, leading to improved use of the cache (fewer misses on both successful and unsuccessful - * searches). Hopscotch hashing outperforms linear probing at much higher load factors, so even - * with the increased memory burden for maintaining the hop vectors, less memory is needed to - * achieve that performance. Hopscotch is also immune to "contamination" from deleting entries - * since entries are genuinely removed instead of being replaced by a placeholder. - * - * The published description of the algorithm used a bit vector, but the paper alludes to an offset - * scheme which is used by this implementation. Since the entries in the neighborhood are within N - * entries of the hash bucket at the start of the neighborhood, a pair of small offset fields each - * log2(N) bits wide is all that's needed to maintain the hops as a linked list. In order to encode - * "no next hop" (i.e. NULL) as the natural initial value of zero, the offsets are biased by one - * (i.e. 0 => NULL, 1 => offset=0, 2 => offset=1, etc.) We can represent neighborhoods of up to 255 - * entries with just 8+8=16 bits per entry. The hop list is sorted by hop offset so the first entry - * in the list is always the bucket closest to the start of the neighborhood. - * - * While individual accesses tend to be very fast, the table resize operations are very, very - * expensive. If an upper bound on the latency of adding an entry to the table is needed, we either - * need to ensure the table is pre-sized to be large enough so no resize is ever needed, or we'll - * need to develop an approach to incrementally resize the table. - */ - -#include "pointer-map.h" - -#include - -#include "errors.h" -#include "logger.h" -#include "memory-alloc.h" -#include "numeric.h" -#include "permassert.h" - -enum { - DEFAULT_CAPACITY = 16, /* the number of neighborhoods in a new table */ - NEIGHBORHOOD = 255, /* the number of buckets in each neighborhood */ - MAX_PROBES = 1024, /* limit on the number of probes for a free bucket */ - NULL_HOP_OFFSET = 0, /* the hop offset value terminating the hop list */ - DEFAULT_LOAD = 75 /* a compromise between memory use and performance */ -}; - -/** - * struct bucket - Hash buckets. - * - * Buckets are packed together to reduce memory usage and improve cache efficiency. It would be - * tempting to encode the hop offsets separately and maintain alignment of key/value pairs, but - * it's crucial to keep the hop fields near the buckets that they use them so they'll tend to share - * cache lines. - */ -struct __packed bucket { - /** - * @first_hop: The biased offset of the first entry in the hop list of the neighborhood - * that hashes to this bucket. - */ - u8 first_hop; - /** @next_hop: the biased offset of the next bucket in the hop list. */ - u8 next_hop; - /** @key: The key stored in this bucket. */ - const void *key; - /** @value: The value stored in this bucket (NULL if empty). */ - void *value; -}; - -/** - * struct pointer_map - The concrete definition of the opaque pointer_map type. - * - * To avoid having to wrap the neighborhoods of the last entries back around to the start of the - * bucket array, we allocate a few more buckets at the end of the array instead, which is why - * capacity and bucket_count are different. - */ -struct pointer_map { - /** @size: The number of entries stored in the map. */ - size_t size; - /** @capacity: The number of neighborhoods in the map. */ - size_t capacity; - /** @bucket_count: The number of buckets in the bucket array. */ - size_t bucket_count; - /** @buckets: The array of hash buckets. */ - struct bucket *buckets; - /** @comparator: The function for comparing keys for equality. */ - pointer_key_comparator *comparator; - /** @hasher: The function for getting a hash code from a key. */ - pointer_key_hasher *hasher; -}; - -/** - * allocate_buckets() - Initialize a pointer_map. - * @map: The map to initialize. - * @capacity: The initial capacity of the map. - * - * Return: UDS_SUCCESS or an error code. - */ -static int allocate_buckets(struct pointer_map *map, size_t capacity) -{ - map->size = 0; - map->capacity = capacity; - - /* - * Allocate NEIGHBORHOOD - 1 extra buckets so the last bucket can have a full neighborhood - * without have to wrap back around to element zero. - */ - map->bucket_count = capacity + (NEIGHBORHOOD - 1); - return uds_allocate(map->bucket_count, - struct bucket, - "pointer_map buckets", - &map->buckets); -} - -/** - * vdo_make_pointer_map() - Allocate and initialize a pointer_map. - * @initial_capacity: The number of entries the map should initially be capable of holding (zero - * tells the map to use its own small default). - * @initial_load: The load factor of the map, expressed as an integer percentage (typically in the - * range 50 to 90, with zero telling the map to use its own default). - * @comparator: The function to use to compare the referents of two pointer keys for equality. - * @hasher: The function to use obtain the hash code associated with each pointer key - * @map_ptr: A pointer to hold the new pointer_map. - * - * Return: UDS_SUCCESS or an error code. - */ -int vdo_make_pointer_map(size_t initial_capacity, - unsigned int initial_load, - pointer_key_comparator comparator, - pointer_key_hasher hasher, - struct pointer_map **map_ptr) -{ - int result; - struct pointer_map *map; - size_t capacity; - - /* Use the default initial load if the caller did not specify one. */ - if (initial_load == 0) - initial_load = DEFAULT_LOAD; - if (initial_load > 100) - return UDS_INVALID_ARGUMENT; - - result = uds_allocate(1, struct pointer_map, "pointer_map", &map); - if (result != UDS_SUCCESS) - return result; - - map->hasher = hasher; - map->comparator = comparator; - - /* Use the default capacity if the caller did not specify one. */ - capacity = (initial_capacity > 0) ? initial_capacity : DEFAULT_CAPACITY; - - /* - * Scale up the capacity by the specified initial load factor. (i.e to hold 1000 entries at - * 80% load we need a capacity of 1250) - */ - capacity = capacity * 100 / initial_load; - - result = allocate_buckets(map, capacity); - if (result != UDS_SUCCESS) { - vdo_free_pointer_map(uds_forget(map)); - return result; - } - - *map_ptr = map; - return UDS_SUCCESS; -} - -/** - * vdo_free_pointer_map() - Free a pointer_map. - * @map: The pointer_map to free. - * - * The map does not own the pointer keys and values stored in the map and they are not freed by - * this call. - */ -void vdo_free_pointer_map(struct pointer_map *map) -{ - if (map == NULL) - return; - - uds_free(uds_forget(map->buckets)); - uds_free(uds_forget(map)); -} - -/** - * vdo_pointer_map_size() - Get the number of entries stored in a pointer_map. - * @map: The pointer_map to query. - * - * Return: The number of entries in the map. - */ -size_t vdo_pointer_map_size(const struct pointer_map *map) -{ - return map->size; -} - -/** - * dereference_hop() - Convert a biased hop offset within a neighborhood to a pointer to the bucket - * it references. - * @neighborhood: The first bucket in the neighborhood. - * @hop_offset: The biased hop offset to the desired bucket. - * - * Return: NULL if hop_offset is zero, otherwise a pointer to the bucket in the neighborhood at - * hop_offset - 1. - */ -static struct bucket *dereference_hop(struct bucket *neighborhood, unsigned int hop_offset) -{ - BUILD_BUG_ON(NULL_HOP_OFFSET != 0); - if (hop_offset == NULL_HOP_OFFSET) - return NULL; - - return &neighborhood[hop_offset - 1]; -} - -/** - * insert_in_hop_list() - Add a bucket into the hop list for the neighborhood, inserting it into - * the list so the hop list remains sorted by hop offset. - * @neighborhood: The first bucket in the neighborhood. - * @new_bucket: The bucket to add to the hop list. - */ -static void insert_in_hop_list(struct bucket *neighborhood, struct bucket *new_bucket) -{ - /* Zero indicates a NULL hop offset, so bias the hop offset by one. */ - int hop_offset = 1 + (new_bucket - neighborhood); - - /* Handle the special case of adding a bucket at the start of the list. */ - int next_hop = neighborhood->first_hop; - - if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { - new_bucket->next_hop = next_hop; - neighborhood->first_hop = hop_offset; - return; - } - - /* Search the hop list for the insertion point that maintains the sort order. */ - for (;;) { - struct bucket *bucket = dereference_hop(neighborhood, next_hop); - - next_hop = bucket->next_hop; - - if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { - new_bucket->next_hop = next_hop; - bucket->next_hop = hop_offset; - return; - } - } -} - -/** - * select_bucket() - Select and return the hash bucket for a given search key. - * @map: The map to search. - * @key: The mapping key. - */ -static struct bucket *select_bucket(const struct pointer_map *map, const void *key) -{ - /* - * Scale the 32-bit hash to a bucket index by treating it as a binary fraction and - * multiplying that by the capacity. If the hash is uniformly distributed over [0 .. - * 2^32-1], then (hash * capacity / 2^32) should be uniformly distributed over [0 .. - * capacity-1]. The multiply and shift is much faster than a divide (modulus) on X86 CPUs. - */ - u64 hash = map->hasher(key); - - return &map->buckets[(hash * map->capacity) >> 32]; -} - -/** - * search_hop_list() - Search the hop list. - * @map: The map being searched. - * @bucket: The map bucket to search for the key. - * @key: The mapping key. - * @previous_ptr: if not NULL, a pointer in which to store the bucket in the list preceding the one - * that had the matching key. - * - * Searches the hop list associated with given hash bucket for a given search key. If the key is - * found, returns a pointer to the entry (bucket or collision), otherwise returns NULL. - * - * Return: an entry that matches the key, or NULL if not found. - */ -static struct bucket *search_hop_list(struct pointer_map *map, - struct bucket *bucket, - const void *key, - struct bucket **previous_ptr) -{ - struct bucket *previous = NULL; - unsigned int next_hop = bucket->first_hop; - - while (next_hop != NULL_HOP_OFFSET) { - /* Check the neighboring bucket indexed by the offset for the desired key. */ - struct bucket *entry = dereference_hop(bucket, next_hop); - - if ((entry->value != NULL) && map->comparator(key, entry->key)) { - if (previous_ptr != NULL) - *previous_ptr = previous; - return entry; - } - next_hop = entry->next_hop; - previous = entry; - } - return NULL; -} - -/** - * vdo_pointer_map_get() - Retrieve the value associated with a given key from the pointer_map. - * @map: The pointer_map to query. - * @key: The key to look up (may be NULL if the comparator and hasher functions support it). - * - * Return: the value associated with the given key, or NULL if the key is not mapped to any value. - */ -void *vdo_pointer_map_get(struct pointer_map *map, const void *key) -{ - struct bucket *match = search_hop_list(map, select_bucket(map, key), key, NULL); - - return ((match != NULL) ? match->value : NULL); -} - -/** - * resize_buckets() - Increase the number of hash buckets and rehash all the existing entries, - * storing them in the new buckets. - * @map: The map to resize. - */ -static int resize_buckets(struct pointer_map *map) -{ - int result; - size_t i; - - /* Copy the top-level map data to the stack. */ - struct pointer_map old_map = *map; - - /* Re-initialize the map to be empty and 50% larger. */ - size_t new_capacity = map->capacity / 2 * 3; - - uds_log_info("%s: attempting resize from %zu to %zu, current size=%zu", - __func__, - map->capacity, - new_capacity, - map->size); - result = allocate_buckets(map, new_capacity); - if (result != UDS_SUCCESS) { - *map = old_map; - return result; - } - - /* Populate the new hash table from the entries in the old bucket array. */ - for (i = 0; i < old_map.bucket_count; i++) { - struct bucket *entry = &old_map.buckets[i]; - - if (entry->value == NULL) - continue; - - result = vdo_pointer_map_put(map, entry->key, entry->value, true, NULL); - if (result != UDS_SUCCESS) { - /* Destroy the new partial map and restore the map from the stack. */ - uds_free(uds_forget(map->buckets)); - *map = old_map; - return result; - } - } - - /* Destroy the old bucket array. */ - uds_free(uds_forget(old_map.buckets)); - return UDS_SUCCESS; -} - -/** - * find_empty_bucket() - Probe the bucket array starting at the given bucket for the next empty - * bucket, returning a pointer to it. - * @map: The map containing the buckets to search. - * @bucket: The bucket at which to start probing. - * @max_probes: The maximum number of buckets to search. - * - * NULL will be returned if the search reaches the end of the bucket array or if the number of - * linear probes exceeds a specified limit. - * - * Return: The next empty bucket, or NULL if the search failed. - */ -static struct bucket * -find_empty_bucket(struct pointer_map *map, struct bucket *bucket, unsigned int max_probes) -{ - /* - * Limit the search to either the nearer of the end of the bucket array or a fixed distance - * beyond the initial bucket. - */ - ptrdiff_t remaining = &map->buckets[map->bucket_count] - bucket; - struct bucket *sentinel = &bucket[min_t(ptrdiff_t, remaining, max_probes)]; - struct bucket *entry; - - for (entry = bucket; entry < sentinel; entry++) - if (entry->value == NULL) - return entry; - return NULL; -} - -/** - * move_empty_bucket() - Move an empty bucket closer to the start of the bucket array. - * @map: The map containing the bucket. - - * @hole: The empty bucket to fill with an entry that precedes it in one of its enclosing - * neighborhoods. - * - * This searches the neighborhoods that contain the empty bucket for a non-empty bucket closer to - * the start of the array. If such a bucket is found, this swaps the two buckets by moving the - * entry to the empty bucket. - * - * Return: The bucket that was vacated by moving its entry to the provided hole, or NULL if no - * entry could be moved. - */ -static struct bucket * -move_empty_bucket(struct pointer_map *map __always_unused, struct bucket *hole) -{ - /* - * Examine every neighborhood that the empty bucket is part of, starting with the one in - * which it is the last bucket. No boundary check is needed for the negative array - * arithmetic since this function is only called when hole is at least NEIGHBORHOOD cells - * deeper into the array than a valid bucket. - */ - struct bucket *bucket; - - for (bucket = &hole[1 - NEIGHBORHOOD]; bucket < hole; bucket++) { - /* - * Find the entry that is nearest to the bucket, which means it will be nearest to - * the hash bucket whose neighborhood is full. - */ - struct bucket *new_hole = dereference_hop(bucket, bucket->first_hop); - - if (new_hole == NULL) { - /* - * There are no buckets in this neighborhood that are in use by this one - * (they must all be owned by overlapping neighborhoods). - */ - continue; - } - - /* - * Skip this bucket if its first entry is actually further away than the hole that - * we're already trying to fill. - */ - if (hole < new_hole) - continue; - - /* - * We've found an entry in this neighborhood that we can "hop" further away, moving - * the hole closer to the hash bucket, if not all the way into its neighborhood. - */ - - /* - * The entry that will be the new hole is the first bucket in the list, so setting - * first_hop is all that's needed remove it from the list. - */ - bucket->first_hop = new_hole->next_hop; - new_hole->next_hop = NULL_HOP_OFFSET; - - /* Move the entry into the original hole. */ - hole->key = new_hole->key; - hole->value = new_hole->value; - new_hole->value = NULL; - - /* Insert the filled hole into the hop list for the neighborhood. */ - insert_in_hop_list(bucket, hole); - return new_hole; - } - - /* We couldn't find an entry to relocate to the hole. */ - return NULL; -} - -/** - * update_mapping() - Find and update any existing mapping for a given key, returning the value - * associated with the key in the provided pointer. - * @map: The pointer_map to attempt to modify. - * @neighborhood: The first bucket in the neighborhood that would contain the search key. - * @key: The key with which to associate the new value. - * @new_value: The value to be associated with the key. - * @update: Whether to overwrite an existing value. - * @old_value_ptr: A pointer in which to store the old value (unmodified if no mapping was found). - * - * Return: true if the map contains a mapping for the key, false if it does not. - */ -static bool update_mapping(struct pointer_map *map, - struct bucket *neighborhood, - const void *key, - void *new_value, - bool update, - void **old_value_ptr) -{ - struct bucket *bucket = search_hop_list(map, neighborhood, key, NULL); - - if (bucket == NULL) { - /* There is no bucket containing the key in the neighborhood. */ - return false; - } - - /* - * Return the value of the current mapping (if desired) and update the mapping with the new - * value (if desired). - */ - if (old_value_ptr != NULL) - *old_value_ptr = bucket->value; - if (update) { - /* - * We're dropping the old key pointer on the floor here, assuming it's a property - * of the value or that it's otherwise safe to just forget. - */ - bucket->key = key; - bucket->value = new_value; - } - return true; -} - -/** - * find_or_make_vacancy() - Find an empty bucket in a specified neighborhood for a new mapping or - * attempt to re-arrange mappings so there is such a bucket. - * @map: The pointer_map to search or modify. - * @neighborhood: The first bucket in the neighborhood in which an empty bucket is needed for a new - * mapping. - * - * This operation may fail (returning NULL) if an empty bucket is not available or could not be - * relocated to the neighborhood. - * - * Return: A pointer to an empty bucket in the desired neighborhood, or NULL if a vacancy could not - * be found or arranged. - */ -static struct bucket *find_or_make_vacancy(struct pointer_map *map, struct bucket *neighborhood) -{ - /* Probe within and beyond the neighborhood for the first empty bucket. */ - struct bucket *hole = find_empty_bucket(map, neighborhood, MAX_PROBES); - - /* - * Keep trying until the empty bucket is in the bucket's neighborhood or we are unable to - * move it any closer by swapping it with a filled bucket. - */ - while (hole != NULL) { - int distance = hole - neighborhood; - - if (distance < NEIGHBORHOOD) { - /* - * We've found or relocated an empty bucket close enough to the initial - * hash bucket to be referenced by its hop vector. - */ - return hole; - } - - /* - * The nearest empty bucket isn't within the neighborhood that must contain the new - * entry, so try to swap it with bucket that is closer. - */ - hole = move_empty_bucket(map, hole); - } - - return NULL; -} - -/** - * vdo_pointer_map_put() - Try to associate a value (a pointer) with an integer in a pointer_map. - * @map: The pointer_map to attempt to modify. - * @key: The key with which to associate the new value (may be NULL if the comparator and hasher - * functions support it). - * @new_value: The value to be associated with the key. - * @update: Whether to overwrite an existing value. - * @old_value_ptr: A pointer in which to store either the old value (if the key was already mapped) - * or NULL if the map did not contain the key; NULL may be provided if the caller - * does not need to know the old value. - * - * If the map already contains a mapping for the provided key, the old value is only replaced with - * the specified value if update is true. In either case the old value is returned. If the map does - * not already contain a value for the specified key, the new value is added regardless of the - * value of update. - * - * If the value stored in the map is updated, then the key stored in the map will also be updated - * with the key provided by this call. The old key will not be returned due to the memory - * management assumptions described in the interface header comment. - * - * Return: UDS_SUCCESS or an error code. - */ -int vdo_pointer_map_put(struct pointer_map *map, - const void *key, - void *new_value, - bool update, - void **old_value_ptr) -{ - struct bucket *neighborhood, *bucket; - - if (new_value == NULL) - return UDS_INVALID_ARGUMENT; - - /* - * Select the bucket at the start of the neighborhood that must contain any entry for the - * provided key. - */ - neighborhood = select_bucket(map, key); - - /* - * Check whether the neighborhood already contains an entry for the key, in which case we - * optionally update it, returning the old value. - */ - if (update_mapping(map, neighborhood, key, new_value, update, old_value_ptr)) - return UDS_SUCCESS; - - /* - * Find an empty bucket in the desired neighborhood for the new entry or re-arrange entries - * in the map so there is such a bucket. This operation will usually succeed; the loop body - * will only be executed on the rare occasions that we have to resize the map. - */ - while ((bucket = find_or_make_vacancy(map, neighborhood)) == NULL) { - /* - * There is no empty bucket in which to put the new entry in the current map, so - * we're forced to allocate a new bucket array with a larger capacity, re-hash all - * the entries into those buckets, and try again (a very expensive operation for - * large maps). - */ - int result = resize_buckets(map); - - if (result != UDS_SUCCESS) - return result; - - /* - * Resizing the map invalidates all pointers to buckets, so - * recalculate the neighborhood pointer. - */ - neighborhood = select_bucket(map, key); - } - - /* Put the new entry in the empty bucket, adding it to the neighborhood. */ - bucket->key = key; - bucket->value = new_value; - insert_in_hop_list(neighborhood, bucket); - map->size += 1; - - /* - * There was no existing entry, so there was no old value to be - * returned. - */ - if (old_value_ptr != NULL) - *old_value_ptr = NULL; - return UDS_SUCCESS; -} - -/** - * vdo_pointer_map_remove() - Remove the mapping for a given key from the pointer_map. - * @map: The pointer_map from which to remove the mapping. - * @key: The key whose mapping is to be removed (may be NULL if the comparator and hasher functions - * support it). - * - * Return: the value that was associated with the key, or NULL if it was not mapped. - */ -void *vdo_pointer_map_remove(struct pointer_map *map, const void *key) -{ - void *value; - - /* Select the bucket to search and search it for an existing entry. */ - struct bucket *bucket = select_bucket(map, key); - struct bucket *previous; - struct bucket *victim = search_hop_list(map, bucket, key, &previous); - - if (victim == NULL) { - /* There is no matching entry to remove. */ - return NULL; - } - - /* - * We found an entry to remove. Save the mapped value to return later and empty the bucket. - */ - map->size -= 1; - value = victim->value; - victim->value = NULL; - victim->key = 0; - - /* The victim bucket is now empty, but it still needs to be spliced out of the hop list. */ - if (previous == NULL) { - /* The victim is the head of the list, so swing first_hop. */ - bucket->first_hop = victim->next_hop; - } else { - previous->next_hop = victim->next_hop; - } - - victim->next_hop = NULL_HOP_OFFSET; - return value; -} diff --git a/drivers/md/dm-vdo/pointer-map.h b/drivers/md/dm-vdo/pointer-map.h deleted file mode 100644 index 83465d01bfa7..000000000000 --- a/drivers/md/dm-vdo/pointer-map.h +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2023 Red Hat - */ - -#ifndef VDO_POINTER_MAP_H -#define VDO_POINTER_MAP_H - -#include -#include - -/* - * A pointer_map associates pointer values (void *) with the data referenced by - * pointer keys (void *). NULL pointer values are not supported. A - * NULL key value is supported when the instance's key comparator and hasher functions - * support it. - * - * The map is implemented as hash table, which should provide constant-time insert, query, and - * remove operations, although the insert may occasionally grow the table, which is linear in the - * number of entries in the map. The table will grow as needed to hold new entries, but will not - * shrink as entries are removed. - * - * The key and value pointers passed to the map are retained and used by the map, but are not owned - * by the map. Freeing the map does not attempt to free the pointers. The client is entirely - * responsible for the memory management of the keys and values. The current interface and - * implementation assume that keys will be properties of the values, or that keys will not be - * memory managed, or that keys will not need to be freed as a result of being replaced when a key - * is re-mapped. - */ - -struct pointer_map; - -/** - * typedef pointer_key_comparator - The prototype of functions that compare the referents of two - * pointer keys for equality. - * @this_key: The first element to compare. - * @that_key: The second element to compare. - * - * If two keys are equal, then both keys must have the same the hash code associated with them by - * the hasher function defined below. - * - * Return: true if and only if the referents of the two key pointers are to be treated as the same - * key by the map. - */ -typedef bool pointer_key_comparator(const void *this_key, const void *that_key); - -/** - * typedef pointer_key_hasher - The prototype of functions that get or calculate a hash code - * associated with the referent of pointer key. - * @key: The pointer key to hash. - * - * The hash code must be uniformly distributed over all u32 values. The hash code associated - * with a given key must not change while the key is in the map. If the comparator function says - * two keys are equal, then this function must return the same hash code for both keys. This - * function may be called many times for a key while an entry is stored for it in the map. - * - * Return: The hash code for the key. - */ -typedef u32 pointer_key_hasher(const void *key); - -int __must_check vdo_make_pointer_map(size_t initial_capacity, - unsigned int initial_load, - pointer_key_comparator comparator, - pointer_key_hasher hasher, - struct pointer_map **map_ptr); - -void vdo_free_pointer_map(struct pointer_map *map); - -size_t vdo_pointer_map_size(const struct pointer_map *map); - -void *vdo_pointer_map_get(struct pointer_map *map, const void *key); - -int __must_check vdo_pointer_map_put(struct pointer_map *map, - const void *key, - void *new_value, - bool update, - void **old_value_ptr); - -void *vdo_pointer_map_remove(struct pointer_map *map, const void *key); - -#endif /* VDO_POINTER_MAP_H */ From ffb8d9654100804998223ba2659044279cc6bc46 Mon Sep 17 00:00:00 2001 From: Bruce Johnston Date: Mon, 20 Nov 2023 17:29:55 -0500 Subject: [PATCH 0600/1038] dm vdo int-map: rename functions to use a common vdo_int_map preamble Reviewed-by: Matthew Sakai Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 12 ++++++------ drivers/md/dm-vdo/dedupe.c | 4 ++-- drivers/md/dm-vdo/int-map.c | 20 ++++++++++++-------- drivers/md/dm-vdo/int-map.h | 10 +++++----- drivers/md/dm-vdo/io-submitter.c | 8 ++++---- drivers/md/dm-vdo/logical-zone.c | 4 ++-- drivers/md/dm-vdo/physical-zone.c | 8 ++++---- 7 files changed, 35 insertions(+), 31 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index c5cb9da5d33e..953819943cd3 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -232,7 +232,7 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache) if (result != UDS_SUCCESS) return result; - result = vdo_make_int_map(cache->page_count, 0, &cache->page_map); + result = vdo_int_map_create(cache->page_count, 0, &cache->page_map); if (result != UDS_SUCCESS) return result; @@ -1346,8 +1346,8 @@ int vdo_invalidate_page_cache(struct vdo_page_cache *cache) } /* Reset the page map by re-allocating it. */ - vdo_free_int_map(uds_forget(cache->page_map)); - return vdo_make_int_map(cache->page_count, 0, &cache->page_map); + vdo_int_map_free(uds_forget(cache->page_map)); + return vdo_int_map_create(cache->page_count, 0, &cache->page_map); } /** @@ -2751,7 +2751,7 @@ static int __must_check initialize_block_map_zone(struct block_map *map, INIT_LIST_HEAD(&zone->dirty_lists->eras[i][VDO_CACHE_PAGE]); } - result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->loading_pages); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->loading_pages); if (result != VDO_SUCCESS) return result; @@ -2831,7 +2831,7 @@ static void uninitialize_block_map_zone(struct block_map_zone *zone) uds_free(uds_forget(zone->dirty_lists)); free_vio_pool(uds_forget(zone->vio_pool)); - vdo_free_int_map(uds_forget(zone->loading_pages)); + vdo_int_map_free(uds_forget(zone->loading_pages)); if (cache->infos != NULL) { struct page_info *info; @@ -2839,7 +2839,7 @@ static void uninitialize_block_map_zone(struct block_map_zone *zone) free_vio(uds_forget(info->vio)); } - vdo_free_int_map(uds_forget(cache->page_map)); + vdo_int_map_free(uds_forget(cache->page_map)); uds_free(uds_forget(cache->infos)); uds_free(uds_forget(cache->pages)); } diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index feb35b0d7be3..1a04e699ed8f 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -2404,7 +2404,7 @@ static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zone data_vio_count_t i; struct hash_zone *zone = &zones->zones[zone_number]; - result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->hash_lock_map); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->hash_lock_map); if (result != VDO_SUCCESS) return result; @@ -2528,7 +2528,7 @@ void vdo_free_hash_zones(struct hash_zones *zones) struct hash_zone *zone = &zones->zones[i]; uds_free_funnel_queue(uds_forget(zone->timed_out_complete)); - vdo_free_int_map(uds_forget(zone->hash_lock_map)); + vdo_int_map_free(uds_forget(zone->hash_lock_map)); uds_free(uds_forget(zone->lock_array)); } diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index 463a4951e0d1..94f13df7a2e1 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -171,7 +171,7 @@ static int allocate_buckets(struct int_map *map, size_t capacity) } /** - * vdo_make_int_map() - Allocate and initialize an int_map. + * vdo_int_map_create() - Allocate and initialize an int_map. * @initial_capacity: The number of entries the map should initially be capable of holding (zero * tells the map to use its own small default). * @initial_load: The load factor of the map, expressed as an integer percentage (typically in the @@ -180,7 +180,8 @@ static int allocate_buckets(struct int_map *map, size_t capacity) * * Return: UDS_SUCCESS or an error code. */ -int vdo_make_int_map(size_t initial_capacity, unsigned int initial_load, struct int_map **map_ptr) +int vdo_int_map_create(size_t initial_capacity, unsigned int initial_load, + struct int_map **map_ptr) { struct int_map *map; int result; @@ -207,7 +208,7 @@ int vdo_make_int_map(size_t initial_capacity, unsigned int initial_load, struct result = allocate_buckets(map, capacity); if (result != UDS_SUCCESS) { - vdo_free_int_map(uds_forget(map)); + vdo_int_map_free(uds_forget(map)); return result; } @@ -216,13 +217,13 @@ int vdo_make_int_map(size_t initial_capacity, unsigned int initial_load, struct } /** - * vdo_free_int_map() - Free an int_map. + * vdo_int_map_free() - Free an int_map. * @map: The int_map to free. * * NOTE: The map does not own the pointer values stored in the map and they are not freed by this * call. */ -void vdo_free_int_map(struct int_map *map) +void vdo_int_map_free(struct int_map *map) { if (map == NULL) return; @@ -464,7 +465,8 @@ find_empty_bucket(struct int_map *map, struct bucket *bucket, unsigned int max_p * Return: The bucket that was vacated by moving its entry to the provided hole, or NULL if no * entry could be moved. */ -static struct bucket *move_empty_bucket(struct int_map *map __always_unused, struct bucket *hole) +static struct bucket *move_empty_bucket(struct int_map *map __always_unused, + struct bucket *hole) { /* * Examine every neighborhood that the empty bucket is part of, starting with the one in @@ -572,7 +574,8 @@ static bool update_mapping(struct int_map *map, * Return: a pointer to an empty bucket in the desired neighborhood, or NULL if a vacancy could not * be found or arranged. */ -static struct bucket *find_or_make_vacancy(struct int_map *map, struct bucket *neighborhood) +static struct bucket *find_or_make_vacancy(struct int_map *map, + struct bucket *neighborhood) { /* Probe within and beyond the neighborhood for the first empty bucket. */ struct bucket *hole = find_empty_bucket(map, neighborhood, MAX_PROBES); @@ -619,7 +622,8 @@ static struct bucket *find_or_make_vacancy(struct int_map *map, struct bucket *n * * Return: UDS_SUCCESS or an error code. */ -int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, void **old_value_ptr) +int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, + void **old_value_ptr) { struct bucket *neighborhood, *bucket; diff --git a/drivers/md/dm-vdo/int-map.h b/drivers/md/dm-vdo/int-map.h index e85f12b30edc..edafb7569f51 100644 --- a/drivers/md/dm-vdo/int-map.h +++ b/drivers/md/dm-vdo/int-map.h @@ -23,17 +23,17 @@ struct int_map; -int __must_check -vdo_make_int_map(size_t initial_capacity, unsigned int initial_load, struct int_map **map_ptr); +int __must_check vdo_int_map_create(size_t initial_capacity, unsigned int initial_load, + struct int_map **map_ptr); -void vdo_free_int_map(struct int_map *map); +void vdo_int_map_free(struct int_map *map); size_t vdo_int_map_size(const struct int_map *map); void *vdo_int_map_get(struct int_map *map, u64 key); -int __must_check -vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, void **old_value_ptr); +int __must_check vdo_int_map_put(struct int_map *map, u64 key, void *new_value, + bool update, void **old_value_ptr); void *vdo_int_map_remove(struct int_map *map, u64 key); diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 39f5f202d69d..9471b6caabe6 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -401,8 +401,8 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter * uneven. So for now, we'll assume that all requests *may* wind up on one thread, * and thus all in the same map. */ - result = vdo_make_int_map(max_requests_active * 2, 0, - &bio_queue_data->map); + result = vdo_int_map_create(max_requests_active * 2, 0, + &bio_queue_data->map); if (result != 0) { /* * Clean up the partially initialized bio-queue entirely and indicate that @@ -422,7 +422,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter * Clean up the partially initialized bio-queue entirely and indicate that * initialization failed. */ - vdo_free_int_map(uds_forget(bio_queue_data->map)); + vdo_int_map_free(uds_forget(bio_queue_data->map)); uds_log_error("bio queue initialization failed %d", result); vdo_cleanup_io_submitter(io_submitter); vdo_free_io_submitter(io_submitter); @@ -471,7 +471,7 @@ void vdo_free_io_submitter(struct io_submitter *io_submitter) io_submitter->num_bio_queues_used--; /* vdo_destroy() will free the work queue, so just give up our reference to it. */ uds_forget(io_submitter->bio_queue_data[i].queue); - vdo_free_int_map(uds_forget(io_submitter->bio_queue_data[i].map)); + vdo_int_map_free(uds_forget(io_submitter->bio_queue_data[i].map)); } uds_free(io_submitter); } diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index 7140e202798f..df69fb68db3d 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -57,7 +57,7 @@ static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number struct logical_zone *zone = &zones->zones[zone_number]; zone_count_t allocation_zone_number; - result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->lbn_operations); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->lbn_operations); if (result != VDO_SUCCESS) return result; @@ -137,7 +137,7 @@ void vdo_free_logical_zones(struct logical_zones *zones) uds_free(uds_forget(zones->manager)); for (index = 0; index < zones->zone_count; index++) - vdo_free_int_map(uds_forget(zones->zones[index].lbn_operations)); + vdo_int_map_free(uds_forget(zones->zones[index].lbn_operations)); uds_free(zones); } diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index 9b99c9a820a3..8ecc80ecbb53 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -330,13 +330,13 @@ static int initialize_zone(struct vdo *vdo, struct physical_zones *zones) zone_count_t zone_number = zones->zone_count; struct physical_zone *zone = &zones->zones[zone_number]; - result = vdo_make_int_map(VDO_LOCK_MAP_CAPACITY, 0, &zone->pbn_operations); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->pbn_operations); if (result != VDO_SUCCESS) return result; result = make_pbn_lock_pool(LOCK_POOL_CAPACITY, &zone->lock_pool); if (result != VDO_SUCCESS) { - vdo_free_int_map(zone->pbn_operations); + vdo_int_map_free(zone->pbn_operations); return result; } @@ -347,7 +347,7 @@ static int initialize_zone(struct vdo *vdo, struct physical_zones *zones) result = vdo_make_default_thread(vdo, zone->thread_id); if (result != VDO_SUCCESS) { free_pbn_lock_pool(uds_forget(zone->lock_pool)); - vdo_free_int_map(zone->pbn_operations); + vdo_int_map_free(zone->pbn_operations); return result; } return result; @@ -401,7 +401,7 @@ void vdo_free_physical_zones(struct physical_zones *zones) struct physical_zone *zone = &zones->zones[index]; free_pbn_lock_pool(uds_forget(zone->lock_pool)); - vdo_free_int_map(uds_forget(zone->pbn_operations)); + vdo_int_map_free(uds_forget(zone->pbn_operations)); } uds_free(zones); From 9165dac82273579b150bb56f76c6cf3222eb1a58 Mon Sep 17 00:00:00 2001 From: Bruce Johnston Date: Mon, 20 Nov 2023 17:29:56 -0500 Subject: [PATCH 0601/1038] dm vdo int-map: remove unused parameter from vdo_int_map_create Reviewed-by: Matthew Sakai Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 6 +++--- drivers/md/dm-vdo/dedupe.c | 2 +- drivers/md/dm-vdo/int-map.c | 13 ++----------- drivers/md/dm-vdo/int-map.h | 3 +-- drivers/md/dm-vdo/io-submitter.c | 2 +- drivers/md/dm-vdo/logical-zone.c | 2 +- drivers/md/dm-vdo/physical-zone.c | 2 +- 7 files changed, 10 insertions(+), 20 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 953819943cd3..f9f68e8d4b0c 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -232,7 +232,7 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache) if (result != UDS_SUCCESS) return result; - result = vdo_int_map_create(cache->page_count, 0, &cache->page_map); + result = vdo_int_map_create(cache->page_count, &cache->page_map); if (result != UDS_SUCCESS) return result; @@ -1347,7 +1347,7 @@ int vdo_invalidate_page_cache(struct vdo_page_cache *cache) /* Reset the page map by re-allocating it. */ vdo_int_map_free(uds_forget(cache->page_map)); - return vdo_int_map_create(cache->page_count, 0, &cache->page_map); + return vdo_int_map_create(cache->page_count, &cache->page_map); } /** @@ -2751,7 +2751,7 @@ static int __must_check initialize_block_map_zone(struct block_map *map, INIT_LIST_HEAD(&zone->dirty_lists->eras[i][VDO_CACHE_PAGE]); } - result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->loading_pages); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->loading_pages); if (result != VDO_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 1a04e699ed8f..b3ffb85518c3 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -2404,7 +2404,7 @@ static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zone data_vio_count_t i; struct hash_zone *zone = &zones->zones[zone_number]; - result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->hash_lock_map); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->hash_lock_map); if (result != VDO_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index 94f13df7a2e1..99ccbb1339c6 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -174,25 +174,16 @@ static int allocate_buckets(struct int_map *map, size_t capacity) * vdo_int_map_create() - Allocate and initialize an int_map. * @initial_capacity: The number of entries the map should initially be capable of holding (zero * tells the map to use its own small default). - * @initial_load: The load factor of the map, expressed as an integer percentage (typically in the - * range 50 to 90, with zero telling the map to use its own default). * @map_ptr: Output, a pointer to hold the new int_map. * * Return: UDS_SUCCESS or an error code. */ -int vdo_int_map_create(size_t initial_capacity, unsigned int initial_load, - struct int_map **map_ptr) +int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) { struct int_map *map; int result; size_t capacity; - /* Use the default initial load if the caller did not specify one. */ - if (initial_load == 0) - initial_load = DEFAULT_LOAD; - if (initial_load > 100) - return UDS_INVALID_ARGUMENT; - result = uds_allocate(1, struct int_map, "struct int_map", &map); if (result != UDS_SUCCESS) return result; @@ -204,7 +195,7 @@ int vdo_int_map_create(size_t initial_capacity, unsigned int initial_load, * Scale up the capacity by the specified initial load factor. (i.e to hold 1000 entries at * 80% load we need a capacity of 1250) */ - capacity = capacity * 100 / initial_load; + capacity = capacity * 100 / DEFAULT_LOAD; result = allocate_buckets(map, capacity); if (result != UDS_SUCCESS) { diff --git a/drivers/md/dm-vdo/int-map.h b/drivers/md/dm-vdo/int-map.h index edafb7569f51..1858ad799887 100644 --- a/drivers/md/dm-vdo/int-map.h +++ b/drivers/md/dm-vdo/int-map.h @@ -23,8 +23,7 @@ struct int_map; -int __must_check vdo_int_map_create(size_t initial_capacity, unsigned int initial_load, - struct int_map **map_ptr); +int __must_check vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr); void vdo_int_map_free(struct int_map *map); diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 9471b6caabe6..74f33a3ddce5 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -401,7 +401,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter * uneven. So for now, we'll assume that all requests *may* wind up on one thread, * and thus all in the same map. */ - result = vdo_int_map_create(max_requests_active * 2, 0, + result = vdo_int_map_create(max_requests_active * 2, &bio_queue_data->map); if (result != 0) { /* diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index df69fb68db3d..cfbf1701ca84 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -57,7 +57,7 @@ static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number struct logical_zone *zone = &zones->zones[zone_number]; zone_count_t allocation_zone_number; - result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->lbn_operations); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->lbn_operations); if (result != VDO_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index 8ecc80ecbb53..3bcf6f1ba77f 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -330,7 +330,7 @@ static int initialize_zone(struct vdo *vdo, struct physical_zones *zones) zone_count_t zone_number = zones->zone_count; struct physical_zone *zone = &zones->zones[zone_number]; - result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, 0, &zone->pbn_operations); + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->pbn_operations); if (result != VDO_SUCCESS) return result; From 4c79d55678b8bf993cb94c6181180997112bb353 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 30 Nov 2023 19:54:56 -0500 Subject: [PATCH 0602/1038] dm vdo: fix how dm_kcopyd_client_create() failure is checked dm_kcopyd_client_create() returns an ERR_PTR so its return must be checked with IS_ERR(). Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo-target.c | 8 ++++++-- drivers/md/dm-vdo/slab-depot.c | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo-target.c b/drivers/md/dm-vdo-target.c index 72d2291dd93f..4fbc148681e5 100644 --- a/drivers/md/dm-vdo-target.c +++ b/drivers/md/dm-vdo-target.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1683,8 +1684,11 @@ static int grow_layout(struct vdo *vdo, block_count_t old_size, block_count_t ne /* Make a copy completion if there isn't one */ if (vdo->partition_copier == NULL) { vdo->partition_copier = dm_kcopyd_client_create(NULL); - if (vdo->partition_copier == NULL) - return -ENOMEM; + if (IS_ERR(vdo->partition_copier)) { + result = PTR_ERR(vdo->partition_copier); + vdo->partition_copier = NULL; + return result; + } } /* Free any unused preparation. */ diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 2125e256aa86..56d975c98752 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -3445,8 +3446,10 @@ static void initiate_load(struct admin_state *state) handle_operation_error, allocator->thread_id, NULL); allocator->eraser = dm_kcopyd_client_create(NULL); - if (allocator->eraser == NULL) { - vdo_fail_completion(&allocator->completion, -ENOMEM); + if (IS_ERR(allocator->eraser)) { + vdo_fail_completion(&allocator->completion, + PTR_ERR(allocator->eraser)); + allocator->eraser = NULL; return; } allocator->slabs_to_erase = get_slab_iterator(allocator); From f36b1d3ba533d21b5b793623f05761b0297d114e Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:18:33 -0500 Subject: [PATCH 0603/1038] dm vdo: use a proper Makefile for dm-vdo Requires moving dm-vdo-target.c into drivers/md/dm-vdo/ This change adds a proper drivers/md/dm-vdo/Makefile and eliminates the abnormal use of patsubst in drivers/md/Makefile -- which was the cause of at least one build failure that was reported by the upstream build bot. Also, split out VDO's drivers/md/dm-vdo/Kconfig and include it from drivers/md/Kconfig Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- MAINTAINERS | 1 - drivers/md/Kconfig | 18 +------- drivers/md/Makefile | 3 +- drivers/md/dm-vdo/Kconfig | 17 +++++++ drivers/md/dm-vdo/Makefile | 61 +++++++++++++++++++++++++ drivers/md/{ => dm-vdo}/dm-vdo-target.c | 52 ++++++++++----------- 6 files changed, 107 insertions(+), 45 deletions(-) create mode 100644 drivers/md/dm-vdo/Kconfig create mode 100644 drivers/md/dm-vdo/Makefile rename drivers/md/{ => dm-vdo}/dm-vdo-target.c (99%) diff --git a/MAINTAINERS b/MAINTAINERS index 74b82cc8db48..45513eb1bc42 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6124,7 +6124,6 @@ M: dm-devel@lists.linux.dev L: dm-devel@lists.linux.dev S: Maintained F: Documentation/admin-guide/device-mapper/vdo*.rst -F: drivers/md/dm-vdo-target.c F: drivers/md/dm-vdo/ DEVLINK diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index cef25cf4800c..68ce56fc61d0 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -498,22 +498,6 @@ config DM_FLAKEY help A target that intermittently fails I/O for debugging purposes. -config DM_VDO - tristate "VDO: deduplication and compression target" - depends on 64BIT - depends on BLK_DEV_DM - select DM_BUFIO - select LZ4_COMPRESS - select LZ4_DECOMPRESS - help - This device mapper target presents a block device with - deduplication, compression and thin-provisioning. - - To compile this code as a module, choose M here: the module will - be called dm-vdo. - - If unsure, say N. - config DM_VERITY tristate "Verity target support" depends on BLK_DEV_DM @@ -650,4 +634,6 @@ config DM_AUDIT Enables audit logging of several security relevant events in the particular device-mapper targets, especially the integrity target. +source "drivers/md/dm-vdo/Kconfig" + endif # MD diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 2ac0e6331104..476a214e4bdc 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -25,7 +25,6 @@ dm-ebs-y += dm-ebs-target.o dm-era-y += dm-era-target.o dm-clone-y += dm-clone-target.o dm-clone-metadata.o dm-verity-y += dm-verity-target.o -dm-vdo-y += dm-vdo-target.o $(patsubst drivers/md/dm-vdo/%.c,dm-vdo/%.o,$(wildcard $(src)/dm-vdo/*.c)) dm-zoned-y += dm-zoned-target.o dm-zoned-metadata.o dm-zoned-reclaim.o md-mod-y += md.o md-bitmap.o @@ -69,7 +68,7 @@ obj-$(CONFIG_DM_ZERO) += dm-zero.o obj-$(CONFIG_DM_RAID) += dm-raid.o obj-$(CONFIG_DM_THIN_PROVISIONING) += dm-thin-pool.o obj-$(CONFIG_DM_VERITY) += dm-verity.o -obj-$(CONFIG_DM_VDO) += dm-vdo.o +obj-$(CONFIG_DM_VDO) += dm-vdo/ obj-$(CONFIG_DM_CACHE) += dm-cache.o obj-$(CONFIG_DM_CACHE_SMQ) += dm-cache-smq.o obj-$(CONFIG_DM_EBS) += dm-ebs.o diff --git a/drivers/md/dm-vdo/Kconfig b/drivers/md/dm-vdo/Kconfig new file mode 100644 index 000000000000..111ecd2c2a24 --- /dev/null +++ b/drivers/md/dm-vdo/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config DM_VDO + tristate "VDO: deduplication and compression target" + depends on 64BIT + depends on BLK_DEV_DM + select DM_BUFIO + select LZ4_COMPRESS + select LZ4_DECOMPRESS + help + This device mapper target presents a block device with + deduplication, compression and thin-provisioning. + + To compile this code as a module, choose M here: the module will + be called dm-vdo. + + If unsure, say N. diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile new file mode 100644 index 000000000000..8f8d161a6dbe --- /dev/null +++ b/drivers/md/dm-vdo/Makefile @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_DM_VDO) += dm-vdo.o + +dm-vdo-objs := \ + action-manager.o \ + admin-state.o \ + block-map.o \ + chapter-index.o \ + completion.o \ + config.o \ + constants.o \ + data-vio.o \ + dedupe.o \ + delta-index.o \ + dm-vdo-target.o \ + dump.o \ + encodings.o \ + errors.o \ + flush.o \ + funnel-queue.o \ + funnel-requestqueue.o \ + funnel-workqueue.o \ + geometry.o \ + index-layout.o \ + index.o \ + index-page-map.o \ + index-session.o \ + int-map.o \ + io-factory.o \ + io-submitter.o \ + logger.o \ + logical-zone.o \ + memory-alloc.o \ + message-stats.o \ + murmurhash3.o \ + open-chapter.o \ + packer.o \ + permassert.o \ + physical-zone.o \ + pool-sysfs.o \ + pool-sysfs-stats.o \ + priority-table.o \ + radix-sort.o \ + recovery-journal.o \ + repair.o \ + slab-depot.o \ + sparse-cache.o \ + status-codes.o \ + string-utils.o \ + sysfs.o \ + thread-cond-var.o \ + thread-device.o \ + thread-registry.o \ + uds-sysfs.o \ + uds-threads.o \ + vdo.o \ + vio.o \ + volume-index.o \ + volume.o \ + wait-queue.o diff --git a/drivers/md/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c similarity index 99% rename from drivers/md/dm-vdo-target.c rename to drivers/md/dm-vdo/dm-vdo-target.c index 4fbc148681e5..fcba50d8d088 100644 --- a/drivers/md/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -13,32 +13,32 @@ #include #include -#include "dm-vdo/admin-state.h" -#include "dm-vdo/block-map.h" -#include "dm-vdo/completion.h" -#include "dm-vdo/constants.h" -#include "dm-vdo/data-vio.h" -#include "dm-vdo/dedupe.h" -#include "dm-vdo/dump.h" -#include "dm-vdo/encodings.h" -#include "dm-vdo/errors.h" -#include "dm-vdo/flush.h" -#include "dm-vdo/io-submitter.h" -#include "dm-vdo/logger.h" -#include "dm-vdo/memory-alloc.h" -#include "dm-vdo/message-stats.h" -#include "dm-vdo/pool-sysfs.h" -#include "dm-vdo/recovery-journal.h" -#include "dm-vdo/repair.h" -#include "dm-vdo/slab-depot.h" -#include "dm-vdo/status-codes.h" -#include "dm-vdo/string-utils.h" -#include "dm-vdo/thread-device.h" -#include "dm-vdo/thread-registry.h" -#include "dm-vdo/types.h" -#include "dm-vdo/uds-sysfs.h" -#include "dm-vdo/vdo.h" -#include "dm-vdo/vio.h" +#include "admin-state.h" +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "dump.h" +#include "encodings.h" +#include "errors.h" +#include "flush.h" +#include "io-submitter.h" +#include "logger.h" +#include "memory-alloc.h" +#include "message-stats.h" +#include "pool-sysfs.h" +#include "recovery-journal.h" +#include "repair.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "string-utils.h" +#include "thread-device.h" +#include "thread-registry.h" +#include "types.h" +#include "uds-sysfs.h" +#include "vdo.h" +#include "vio.h" #define CURRENT_VERSION "8.3.0.65" From 6bda10727d1651abc32763fbf081602df61cd9ad Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:35:45 -0500 Subject: [PATCH 0604/1038] dm vdo block-map: fix a few small nits Rename 'pages' to 'num_pages' in distribute_page_over_waitq(). Update assert message in validate_completed_page() to model others. Tweak line-wrapping on a comment that was needlessly long. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index f9f68e8d4b0c..64f893b0721a 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -535,19 +535,19 @@ static void complete_waiter_with_page(struct vdo_waiter *waiter, void *page_info static unsigned int distribute_page_over_waitq(struct page_info *info, struct vdo_wait_queue *waitq) { - size_t pages; + size_t num_pages; update_lru(info); - pages = vdo_waitq_num_waiters(waitq); + num_pages = vdo_waitq_num_waiters(waitq); /* * Increment the busy count once for each pending completion so that this page does not * stop being busy until all completions have been processed (VDO-83). */ - info->busy += pages; + info->busy += num_pages; vdo_waitq_notify_all_waiters(waitq, complete_waiter_with_page, info); - return pages; + return num_pages; } /** @@ -614,7 +614,8 @@ static int __must_check validate_completed_page(struct vdo_page_completion *comp return result; if (writable) { - result = ASSERT(completion->writable, "VDO Page Completion is writable"); + result = ASSERT(completion->writable, + "VDO Page Completion must be writable"); if (result != UDS_SUCCESS) return result; } @@ -741,8 +742,8 @@ static void handle_rebuild_read_error(struct vdo_completion *completion) assert_on_cache_thread(cache, __func__); /* - * We are doing a read-only rebuild, so treat this as a successful read of an uninitialized - * page. + * We are doing a read-only rebuild, so treat this as a successful read + * of an uninitialized page. */ vio_record_metadata_io_error(as_vio(completion)); ADD_ONCE(cache->stats.failed_reads, 1); From 8810d3d59494fb8404d5ca5f72a5b05ed5fbeaa1 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:35:46 -0500 Subject: [PATCH 0605/1038] dm vdo block-map: use uds_log_ratelimit() rather than open code it Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 64f893b0721a..eb838e6ae3e6 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -1014,13 +1014,9 @@ static void handle_page_write_error(struct vdo_completion *completion) /* If we're already read-only, write failures are to be expected. */ if (result != VDO_READ_ONLY) { - static DEFINE_RATELIMIT_STATE(error_limiter, DEFAULT_RATELIMIT_INTERVAL, - DEFAULT_RATELIMIT_BURST); - - if (__ratelimit(&error_limiter)) { - uds_log_error("failed to write block map page %llu", - (unsigned long long) info->pbn); - } + uds_log_ratelimit(uds_log_error, + "failed to write block map page %llu", + (unsigned long long) info->pbn); } set_info_state(info, PS_DIRTY); From 36778716a2cf7003d7b45bfb9ed6e88f47511b35 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:35:47 -0500 Subject: [PATCH 0606/1038] dm vdo block-map: remove extra vdo arg from initialize_block_map_zone The block_map is passed to initialize_block_map_zone, but the block_map's vdo member is already initialized with the same vdo instance, so just use it. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index eb838e6ae3e6..eda42383a2c1 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -2719,12 +2719,12 @@ void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, */ static int __must_check initialize_block_map_zone(struct block_map *map, zone_count_t zone_number, - struct vdo *vdo, page_count_t cache_size, block_count_t maximum_age) { int result; block_count_t i; + struct vdo *vdo = map->vdo; struct block_map_zone *zone = &map->zones[zone_number]; BUILD_BUG_ON(sizeof(struct page_descriptor) != sizeof(u64)); @@ -2898,8 +2898,7 @@ int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical map->zone_count = vdo->thread_config.logical_zone_count; for (zone = 0; zone < map->zone_count; zone++) { - result = initialize_block_map_zone(map, zone, vdo, cache_size, - maximum_age); + result = initialize_block_map_zone(map, zone, cache_size, maximum_age); if (result != VDO_SUCCESS) { vdo_free_block_map(map); return result; From 3ccf136a4925c852197dd5fb5f7ee88514757e4e Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:35:48 -0500 Subject: [PATCH 0607/1038] dm vdo block-map: avoid extra dereferences to access vdo object The vdo_page_cache's 'vdo' is the same as the block_map's vdo instance, so use that to save 2 extra dereferences. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index eda42383a2c1..0ce136612869 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -562,7 +562,7 @@ static void set_persistent_error(struct vdo_page_cache *cache, const char *conte { struct page_info *info; /* If we're already read-only, there's no need to log. */ - struct vdo *vdo = cache->zone->block_map->vdo; + struct vdo *vdo = cache->vdo; if ((result != VDO_READ_ONLY) && !vdo_is_read_only(vdo)) { uds_log_error_strerror(result, "VDO Page Cache persistent error: %s", @@ -1111,7 +1111,7 @@ static void write_pages(struct vdo_completion *flush_completion) state_entry); list_del_init(&info->state_entry); - if (vdo_is_read_only(info->cache->zone->block_map->vdo)) { + if (vdo_is_read_only(info->cache->vdo)) { struct vdo_completion *completion = &info->vio->completion; vdo_reset_completion(completion); @@ -1233,7 +1233,7 @@ void vdo_get_page(struct vdo_page_completion *page_completion, cache->zone->thread_id, parent); completion->requeue = requeue; - if (page_completion->writable && vdo_is_read_only(cache->zone->block_map->vdo)) { + if (page_completion->writable && vdo_is_read_only(cache->vdo)) { vdo_fail_completion(completion, VDO_READ_ONLY); return; } From b06d5c37b88b97359e98820eecae99e243512c1b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:35:49 -0500 Subject: [PATCH 0608/1038] dm vdo block-map: rename struct cursors member to 'completion' 'completion' is more informative name for a 'struct vdo_completion' than 'parent'. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 20 +++++++++----------- drivers/md/dm-vdo/block-map.h | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 0ce136612869..50dda7ae7074 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -98,7 +98,7 @@ struct cursors { struct block_map_zone *zone; struct vio_pool *pool; vdo_entry_callback_fn entry_callback; - struct vdo_completion *parent; + struct vdo_completion *completion; root_count_t active_roots; struct cursor cursors[]; }; @@ -2501,7 +2501,7 @@ static void replace_forest(struct block_map *map) static void finish_cursor(struct cursor *cursor) { struct cursors *cursors = cursor->parent; - struct vdo_completion *parent = cursors->parent; + struct vdo_completion *completion = cursors->completion; return_vio_to_pool(cursors->pool, uds_forget(cursor->vio)); if (--cursors->active_roots > 0) @@ -2509,7 +2509,7 @@ static void finish_cursor(struct cursor *cursor) uds_free(cursors); - vdo_finish_completion(parent); + vdo_finish_completion(completion); } static void traverse(struct cursor *cursor); @@ -2595,12 +2595,10 @@ static void traverse(struct cursor *cursor) if (cursor->height < VDO_BLOCK_MAP_TREE_HEIGHT - 1) { int result = cursor->parent->entry_callback(location.pbn, - cursor->parent->parent); - + cursor->parent->completion); if (result != VDO_SUCCESS) { page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; - vdo_write_tree_page(tree_page, - cursor->parent->zone); + vdo_write_tree_page(tree_page, cursor->parent->zone); continue; } } @@ -2676,10 +2674,10 @@ static struct boundary compute_boundary(struct block_map *map, root_count_t root /** * vdo_traverse_forest() - Walk the entire forest of a block map. * @callback: A function to call with the pbn of each allocated node in the forest. - * @parent: The completion to notify on each traversed PBN, and when the traversal is complete. + * @completion: The completion to notify on each traversed PBN, and when traversal completes. */ void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, - struct vdo_completion *parent) + struct vdo_completion *completion) { root_count_t root; struct cursors *cursors; @@ -2688,14 +2686,14 @@ void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, result = uds_allocate_extended(struct cursors, map->root_count, struct cursor, __func__, &cursors); if (result != VDO_SUCCESS) { - vdo_fail_completion(parent, result); + vdo_fail_completion(completion, result); return; } cursors->zone = &map->zones[0]; cursors->pool = cursors->zone->vio_pool; cursors->entry_callback = callback; - cursors->parent = parent; + cursors->completion = completion; cursors->active_roots = map->root_count; for (root = 0; root < map->root_count; root++) { struct cursor *cursor = &cursors->cursors[root]; diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h index cc98d19309ce..c574bd524bc2 100644 --- a/drivers/md/dm-vdo/block-map.h +++ b/drivers/md/dm-vdo/block-map.h @@ -313,7 +313,7 @@ physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, void vdo_write_tree_page(struct tree_page *page, struct block_map_zone *zone); void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, - struct vdo_completion *parent); + struct vdo_completion *completion); int __must_check vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical_blocks, struct vdo *vdo, From 86492a3f698de5d66777ba18e06e030cb2c5cf9a Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:42:27 -0500 Subject: [PATCH 0609/1038] dm vdo: slight cleanup of UDS error codes No need to increment each UDS_ error code manually (relative to UDS_ERROR_CODE_BASE). Also, remove unused PRP_BLOCK_START and PRP_BLOCK_END. Lastly, UDS_SUCCESS and VDO_SUCCESS are used interchangeably; so best to explicitly set VDO_SUCCESS equal to UDS_SUCCESS. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/errors.h | 35 ++++++++++++++++---------------- drivers/md/dm-vdo/status-codes.h | 4 +--- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/drivers/md/dm-vdo/errors.h b/drivers/md/dm-vdo/errors.h index 0129fa37d8d8..c37f198a159c 100644 --- a/drivers/md/dm-vdo/errors.h +++ b/drivers/md/dm-vdo/errors.h @@ -15,43 +15,42 @@ enum uds_status_codes { /* Successful return */ UDS_SUCCESS = 0, - /* Used as a base value for reporting internal errors */ UDS_ERROR_CODE_BASE = 1024, /* Index overflow */ - UDS_OVERFLOW = UDS_ERROR_CODE_BASE + 0, + UDS_OVERFLOW = UDS_ERROR_CODE_BASE, /* Invalid argument passed to internal routine */ - UDS_INVALID_ARGUMENT = UDS_ERROR_CODE_BASE + 1, + UDS_INVALID_ARGUMENT, /* UDS data structures are in an invalid state */ - UDS_BAD_STATE = UDS_ERROR_CODE_BASE + 2, + UDS_BAD_STATE, /* Attempt to enter the same name into an internal structure twice */ - UDS_DUPLICATE_NAME = UDS_ERROR_CODE_BASE + 3, + UDS_DUPLICATE_NAME, /* An assertion failed */ - UDS_ASSERTION_FAILED = UDS_ERROR_CODE_BASE + 4, + UDS_ASSERTION_FAILED, /* A request has been queued for later processing (not an error) */ - UDS_QUEUED = UDS_ERROR_CODE_BASE + 5, + UDS_QUEUED, /* A problem has occurred with a buffer */ - UDS_BUFFER_ERROR = UDS_ERROR_CODE_BASE + 6, + UDS_BUFFER_ERROR, /* No directory was found where one was expected */ - UDS_NO_DIRECTORY = UDS_ERROR_CODE_BASE + 7, + UDS_NO_DIRECTORY, /* This error range has already been registered */ - UDS_ALREADY_REGISTERED = UDS_ERROR_CODE_BASE + 8, + UDS_ALREADY_REGISTERED, /* Attempt to read or write data outside the valid range */ - UDS_OUT_OF_RANGE = UDS_ERROR_CODE_BASE + 9, + UDS_OUT_OF_RANGE, /* Could not load modules */ - UDS_EMODULE_LOAD = UDS_ERROR_CODE_BASE + 10, + UDS_EMODULE_LOAD, /* The index session is disabled */ - UDS_DISABLED = UDS_ERROR_CODE_BASE + 11, + UDS_DISABLED, /* Unknown error */ - UDS_UNKNOWN_ERROR = UDS_ERROR_CODE_BASE + 12, + UDS_UNKNOWN_ERROR, /* The index configuration or volume format is no longer supported */ - UDS_UNSUPPORTED_VERSION = UDS_ERROR_CODE_BASE + 13, + UDS_UNSUPPORTED_VERSION, /* Some index structure is corrupt */ - UDS_CORRUPT_DATA = UDS_ERROR_CODE_BASE + 14, + UDS_CORRUPT_DATA, /* No index state found */ - UDS_NO_INDEX = UDS_ERROR_CODE_BASE + 15, + UDS_NO_INDEX, /* Attempt to access incomplete index save data */ - UDS_INDEX_NOT_SAVED_CLEANLY = UDS_ERROR_CODE_BASE + 16, + UDS_INDEX_NOT_SAVED_CLEANLY, /* One more than the last UDS_INTERNAL error code */ UDS_ERROR_CODE_LAST, /* One more than the last error this block will ever use */ diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h index c5da37ec8b55..9e0126a83c6b 100644 --- a/drivers/md/dm-vdo/status-codes.h +++ b/drivers/md/dm-vdo/status-codes.h @@ -12,14 +12,12 @@ enum { UDS_BLOCK_SIZE = UDS_ERROR_CODE_BLOCK_END - UDS_ERROR_CODE_BASE, VDO_BLOCK_START = UDS_ERROR_CODE_BLOCK_END, VDO_BLOCK_END = VDO_BLOCK_START + UDS_BLOCK_SIZE, - PRP_BLOCK_START = VDO_BLOCK_END, - PRP_BLOCK_END = PRP_BLOCK_START + UDS_BLOCK_SIZE, }; /* VDO-specific status codes. */ enum vdo_status_codes { /* successful result */ - VDO_SUCCESS, + VDO_SUCCESS = UDS_SUCCESS, /* base of all VDO errors */ VDO_STATUS_CODE_BASE = VDO_BLOCK_START, /* we haven't written this yet */ From c10497b3b11d0d040eaff25bef47dec569ae0779 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:42:28 -0500 Subject: [PATCH 0610/1038] dm vdo: rename uds_map_to_system_error to uds_status_to_errno Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/errors.c | 2 +- drivers/md/dm-vdo/errors.h | 2 +- drivers/md/dm-vdo/index-layout.c | 4 ++-- drivers/md/dm-vdo/index-session.c | 20 ++++++++++---------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c index d19504a1445e..e8599599a377 100644 --- a/drivers/md/dm-vdo/errors.c +++ b/drivers/md/dm-vdo/errors.c @@ -219,7 +219,7 @@ const char *uds_string_error_name(int errnum, char *buf, size_t buflen) * system-generated value (such as -EIO), or an internal UDS status code. The result will be a * negative errno value. */ -int uds_map_to_system_error(int error) +int uds_status_to_errno(int error) { char error_name[UDS_MAX_ERROR_NAME_SIZE]; char error_message[UDS_MAX_ERROR_MESSAGE_SIZE]; diff --git a/drivers/md/dm-vdo/errors.h b/drivers/md/dm-vdo/errors.h index c37f198a159c..cf15d7243204 100644 --- a/drivers/md/dm-vdo/errors.h +++ b/drivers/md/dm-vdo/errors.h @@ -71,7 +71,7 @@ const char * __must_check uds_string_error(int errnum, char *buf, size_t buflen) const char *uds_string_error_name(int errnum, char *buf, size_t buflen); -int uds_map_to_system_error(int error); +int uds_status_to_errno(int error); int uds_register_error_block(const char *block_name, int first_error, int last_reserved_error, const struct error_info *infos, diff --git a/drivers/md/dm-vdo/index-layout.c b/drivers/md/dm-vdo/index-layout.c index 4dffa82a3c43..6446a1f58641 100644 --- a/drivers/md/dm-vdo/index-layout.c +++ b/drivers/md/dm-vdo/index-layout.c @@ -267,13 +267,13 @@ int uds_compute_index_size(const struct uds_parameters *parameters, u64 *index_s result = uds_make_configuration(parameters, &index_config); if (result != UDS_SUCCESS) { uds_log_error_strerror(result, "cannot compute index size"); - return uds_map_to_system_error(result); + return uds_status_to_errno(result); } result = compute_sizes(index_config, &sizes); uds_free_configuration(index_config); if (result != UDS_SUCCESS) - return uds_map_to_system_error(result); + return uds_status_to_errno(result); *index_size = sizes.total_size; return UDS_SUCCESS; diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c index 615c7b840d7b..73c2927f74ce 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/index-session.c @@ -210,7 +210,7 @@ static void handle_callbacks(struct uds_request *request) if (request->status == UDS_SUCCESS) update_session_stats(request); - request->status = uds_map_to_system_error(request->status); + request->status = uds_status_to_errno(request->status); request->callback(request); release_index_session(index_session); } @@ -276,7 +276,7 @@ int uds_create_index_session(struct uds_index_session **session) return -EINVAL; } - return uds_map_to_system_error(make_empty_index_session(session)); + return uds_status_to_errno(make_empty_index_session(session)); } static int __must_check start_loading_index_session(struct uds_index_session *index_session) @@ -374,7 +374,7 @@ int uds_open_index(enum uds_open_index_type open_type, result = start_loading_index_session(session); if (result != UDS_SUCCESS) - return uds_map_to_system_error(result); + return uds_status_to_errno(result); session->parameters = *parameters; format_dev_t(name, parameters->bdev->bd_dev); @@ -386,7 +386,7 @@ int uds_open_index(enum uds_open_index_type open_type, get_open_type_string(open_type)); finish_loading_index_session(session, result); - return uds_map_to_system_error(result); + return uds_status_to_errno(result); } static void wait_for_no_requests_in_progress(struct uds_index_session *index_session) @@ -470,7 +470,7 @@ int uds_suspend_index_session(struct uds_index_session *session, bool save) uds_unlock_mutex(&session->request_mutex); if (no_work) - return uds_map_to_system_error(result); + return uds_status_to_errno(result); if (rebuilding) suspend_rebuild(session); @@ -484,7 +484,7 @@ int uds_suspend_index_session(struct uds_index_session *session, bool save) session->state |= IS_FLAG_SUSPENDED; uds_broadcast_cond(&session->request_cond); uds_unlock_mutex(&session->request_mutex); - return uds_map_to_system_error(result); + return uds_status_to_errno(result); } static int replace_device(struct uds_index_session *session, struct block_device *bdev) @@ -536,7 +536,7 @@ int uds_resume_index_session(struct uds_index_session *session, session->state &= ~IS_FLAG_WAITING; uds_broadcast_cond(&session->request_cond); uds_unlock_mutex(&session->request_mutex); - return uds_map_to_system_error(result); + return uds_status_to_errno(result); } } @@ -635,7 +635,7 @@ int uds_close_index(struct uds_index_session *index_session) } uds_unlock_mutex(&index_session->request_mutex); if (result != UDS_SUCCESS) - return uds_map_to_system_error(result); + return uds_status_to_errno(result); uds_log_debug("Closing index"); wait_for_no_requests_in_progress(index_session); @@ -646,7 +646,7 @@ int uds_close_index(struct uds_index_session *index_session) index_session->state &= ~IS_FLAG_CLOSING; uds_broadcast_cond(&index_session->request_cond); uds_unlock_mutex(&index_session->request_mutex); - return uds_map_to_system_error(result); + return uds_status_to_errno(result); } /* This will save and close an open index before destroying the session. */ @@ -702,7 +702,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) uds_destroy_mutex(&index_session->request_mutex); uds_log_debug("Destroyed index session"); uds_free(index_session); - return uds_map_to_system_error(result); + return uds_status_to_errno(result); } /* Wait until all callbacks for index operations are complete. */ From f7c1c2e085cd8232e24ebae6a0a72998010ea8cc Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:42:29 -0500 Subject: [PATCH 0611/1038] dm vdo: rename vdo_map_to_system_error to vdo_status_to_errno Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/data-vio.c | 2 +- drivers/md/dm-vdo/dm-vdo-target.c | 10 +++++----- drivers/md/dm-vdo/status-codes.c | 4 ++-- drivers/md/dm-vdo/status-codes.h | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 711396e7a77d..7d8100f29e13 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -274,7 +274,7 @@ static void acknowledge_data_vio(struct data_vio *data_vio) { struct vdo *vdo = vdo_from_data_vio(data_vio); struct bio *bio = data_vio->user_bio; - int error = vdo_map_to_system_error(data_vio->vio.completion.result); + int error = vdo_status_to_errno(data_vio->vio.completion.result); if (bio == NULL) return; diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index fcba50d8d088..e2e60a29e873 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -1117,7 +1117,7 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, vdo_write_stats(vdo, result_buffer, maxlen); result = 1; } else { - result = vdo_map_to_system_error(process_vdo_message(vdo, argc, argv)); + result = vdo_status_to_errno(process_vdo_message(vdo, argc, argv)); } uds_unregister_thread_device_id(); @@ -1613,7 +1613,7 @@ static int construct_new_vdo_registered(struct dm_target *ti, unsigned int argc, if (result != VDO_SUCCESS) { release_instance(instance); free_device_config(config); - return vdo_map_to_system_error(result); + return vdo_status_to_errno(result); } return VDO_SUCCESS; @@ -1852,7 +1852,7 @@ static int prepare_to_modify(struct dm_target *ti, struct device_config *config, if (result != VDO_SUCCESS) { if (result == VDO_PARAMETER_MISMATCH) { /* - * If we don't trap this case, vdo_map_to_system_error() will remap + * If we don't trap this case, vdo_status_to_errno() will remap * it to -EIO, which is misleading and ahistorical. */ result = -EINVAL; @@ -1892,7 +1892,7 @@ static int update_existing_vdo(const char *device_name, struct dm_target *ti, result = prepare_to_modify(ti, config, vdo); if (result != VDO_SUCCESS) { free_device_config(config); - return vdo_map_to_system_error(result); + return vdo_status_to_errno(result); } set_device_config(ti, vdo, config); @@ -2850,7 +2850,7 @@ static int vdo_preresume(struct dm_target *ti) if ((result == VDO_PARAMETER_MISMATCH) || (result == VDO_INVALID_ADMIN_STATE)) result = -EINVAL; uds_unregister_thread_device_id(); - return vdo_map_to_system_error(result); + return vdo_status_to_errno(result); } static void vdo_resume(struct dm_target *ti) diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index 33b1de29b205..b4d7eb7f94ff 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -87,7 +87,7 @@ int vdo_register_status_codes(void) } /** - * vdo_map_to_system_error() - Given an error code, return a value we can return to the OS. + * vdo_status_to_errno() - Given an error code, return a value we can return to the OS. * @error: The error code to convert. * * The input error code may be a system-generated value (such as -EIO), an errno macro used in our @@ -96,7 +96,7 @@ int vdo_register_status_codes(void) * * Return: A system error code value. */ -int vdo_map_to_system_error(int error) +int vdo_status_to_errno(int error) { char error_name[UDS_MAX_ERROR_NAME_SIZE]; char error_message[UDS_MAX_ERROR_MESSAGE_SIZE]; diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h index 9e0126a83c6b..5d1e8bbe54b4 100644 --- a/drivers/md/dm-vdo/status-codes.h +++ b/drivers/md/dm-vdo/status-codes.h @@ -105,6 +105,6 @@ extern const struct error_info vdo_status_list[]; int vdo_register_status_codes(void); -int vdo_map_to_system_error(int error); +int vdo_status_to_errno(int error); #endif /* VDO_STATUS_CODES_H */ From 97b6f0e752ce862759c95ab0e83d84623236bb8d Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 21:45:31 -0500 Subject: [PATCH 0612/1038] dm vdo data-vio: rename is_trim flag to is_discard Eliminate use of "trim" in favor of "discard" since it reflects the top-level Linux discard primative rather than the ATA specific ditto. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 4 ++-- drivers/md/dm-vdo/data-vio.c | 16 ++++++++-------- drivers/md/dm-vdo/data-vio.h | 2 +- drivers/md/dm-vdo/types.h | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 50dda7ae7074..54ad6939cab7 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -2194,8 +2194,8 @@ static void allocate_block_map_page(struct block_map_zone *zone, { int result; - if (!data_vio->write || data_vio->is_trim) { - /* This is a pure read or a trim, so there's nothing left to do here. */ + if (!data_vio->write || data_vio->is_discard) { + /* This is a pure read or a discard, so there's nothing left to do here. */ finish_lookup(data_vio, VDO_SUCCESS); return; } diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 7d8100f29e13..328b645dee16 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -553,7 +553,7 @@ static void launch_bio(struct vdo *vdo, struct data_vio *data_vio, struct bio *b if (bio_op(bio) == REQ_OP_DISCARD) { data_vio->remaining_discard = bio->bi_iter.bi_size; data_vio->write = true; - data_vio->is_trim = true; + data_vio->is_discard = true; if (data_vio->is_partial) { vdo_count_bios(&vdo->stats.bios_in_partial, bio); data_vio->read = true; @@ -1990,10 +1990,10 @@ static void handle_allocation_error(struct vdo_completion *completion) handle_data_vio_error(completion); } -static int assert_is_trim(struct data_vio *data_vio) +static int assert_is_discard(struct data_vio *data_vio) { - int result = ASSERT(data_vio->is_trim, - "data_vio with no block map page is a trim"); + int result = ASSERT(data_vio->is_discard, + "data_vio with no block map page is a discard"); return ((result == VDO_SUCCESS) ? result : VDO_READ_ONLY); } @@ -2019,19 +2019,19 @@ void continue_data_vio_with_block_map_slot(struct vdo_completion *completion) if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) { /* - * This is a trim for a block on a block map page which has not been allocated, so + * This is a discard for a block on a block map page which has not been allocated, so * there's nothing more we need to do. */ completion->callback = complete_data_vio; - continue_data_vio_with_error(data_vio, assert_is_trim(data_vio)); + continue_data_vio_with_error(data_vio, assert_is_discard(data_vio)); return; } /* - * We need an allocation if this is neither a full-block trim nor a + * We need an allocation if this is neither a full-block discard nor a * full-block zero write. */ - if (!data_vio->is_zero && (!data_vio->is_trim || data_vio->is_partial)) { + if (!data_vio->is_zero && (!data_vio->is_discard || data_vio->is_partial)) { data_vio_allocate_data_block(data_vio, VIO_WRITE_LOCK, allocate_block, handle_allocation_error); return; diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h index f5a683968d1c..78744d064e96 100644 --- a/drivers/md/dm-vdo/data-vio.h +++ b/drivers/md/dm-vdo/data-vio.h @@ -199,7 +199,7 @@ struct data_vio { u16 write : 1; u16 fua : 1; u16 is_zero : 1; - u16 is_trim : 1; + u16 is_discard : 1; u16 is_partial : 1; u16 is_duplicate : 1; u16 first_reference_operation_complete : 1; diff --git a/drivers/md/dm-vdo/types.h b/drivers/md/dm-vdo/types.h index abc6d36f6522..dbe892b10f26 100644 --- a/drivers/md/dm-vdo/types.h +++ b/drivers/md/dm-vdo/types.h @@ -144,7 +144,7 @@ struct block_map_slot { /* * Four bits of each five-byte block map entry contain a mapping state value used to distinguish - * unmapped or trimmed logical blocks (which are treated as mapped to the zero block) from entries + * unmapped or discarded logical blocks (which are treated as mapped to the zero block) from entries * that have been mapped to a physical block, including the zero block. * * FIXME: these should maybe be defines. From 1ccef45aa889a0a3cd3f82681bfd90e0d4c2b203 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 22:01:25 -0500 Subject: [PATCH 0613/1038] dm vdo slab-depot: fix various small nits Comment typo, whitespace issues, mark function inline. Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/slab-depot.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 56d975c98752..42126bd60242 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -1360,7 +1360,7 @@ static unsigned int calculate_slab_priority(struct vdo_slab *slab) /* * Slabs are essentially prioritized by an approximation of the number of free blocks in the slab - * so slabs with lots of free blocks with be opened for allocation before slabs that have few free + * so slabs with lots of free blocks will be opened for allocation before slabs that have few free * blocks. */ static void prioritize_slab(struct vdo_slab *slab) @@ -1374,14 +1374,14 @@ static void prioritize_slab(struct vdo_slab *slab) /** * adjust_free_block_count() - Adjust the free block count and (if needed) reprioritize the slab. - * @increment: should be true if the free block count went up. + * @incremented: true if the free block count went up. */ -static void adjust_free_block_count(struct vdo_slab *slab, bool increment) +static void adjust_free_block_count(struct vdo_slab *slab, bool incremented) { struct block_allocator *allocator = slab->allocator; WRITE_ONCE(allocator->allocated_blocks, - allocator->allocated_blocks + (increment ? -1 : 1)); + allocator->allocated_blocks + (incremented ? -1 : 1)); /* The open slab doesn't need to be reprioritized until it is closed. */ if (slab == allocator->open_slab) @@ -1747,9 +1747,8 @@ static void add_entry_from_waiter(struct vdo_waiter *waiter, void *context) static inline bool is_next_entry_a_block_map_increment(struct slab_journal *journal) { struct vdo_waiter *waiter = vdo_waitq_get_first_waiter(&journal->entry_waiters); - struct reference_updater *updater = container_of(waiter, - struct reference_updater, - waiter); + struct reference_updater *updater = + container_of(waiter, struct reference_updater, waiter); return (updater->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING); } @@ -2642,7 +2641,7 @@ static struct vdo_slab *get_next_slab(struct slab_scrubber *scrubber) * * Return: true if the scrubber has slabs to scrub. */ -static bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubber) +static inline bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubber) { return (get_next_slab(scrubber) != NULL); } @@ -2817,8 +2816,8 @@ static int apply_block_entries(struct packed_slab_journal_block *block, static void apply_journal_entries(struct vdo_completion *completion) { int result; - struct slab_scrubber *scrubber - = container_of(as_vio(completion), struct slab_scrubber, vio); + struct slab_scrubber *scrubber = + container_of(as_vio(completion), struct slab_scrubber, vio); struct vdo_slab *slab = scrubber->slab; struct slab_journal *journal = &slab->journal; From ac9ae5769d61e4fe1c5e4245c5cbcd6c0f0f0e9e Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 22:01:26 -0500 Subject: [PATCH 0614/1038] dm vdo dedupe: fix various small nits Remove extra blank line, mark function inline, add missing braces, and fix a typo in a comment. Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dedupe.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index b3ffb85518c3..8533f1669a01 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -1061,7 +1061,7 @@ static void start_deduping(struct hash_lock *lock, struct data_vio *agent, * increment_stat() - Increment a statistic counter in a non-atomic yet thread-safe manner. * @stat: The statistic field to increment. */ -static void increment_stat(u64 *stat) +static inline void increment_stat(u64 *stat) { /* * Must only be mutated on the hash zone thread. Prevents any compiler shenanigans from @@ -1130,7 +1130,6 @@ static bool blocks_equal(char *block1, char *block2) { int i; - for (i = 0; i < VDO_BLOCK_SIZE; i += sizeof(u64)) { if (*((u64 *) &block1[i]) != *((u64 *) &block2[i])) return false; @@ -1456,13 +1455,13 @@ static void finish_writing(struct hash_lock *lock, struct data_vio *agent) lock->duplicate = agent->new_mapped; lock->verified = true; - if (vdo_is_state_compressed(lock->duplicate.state) && - lock->registered) + if (vdo_is_state_compressed(lock->duplicate.state) && lock->registered) { /* * Compression means the location we gave in the UDS query is not the location * we're using to deduplicate. */ lock->update_advice = true; + } /* If there are any waiters, we need to start deduping them. */ if (vdo_waitq_has_waiters(&lock->waiters)) { @@ -2366,7 +2365,7 @@ static void timeout_index_operations_callback(struct vdo_completion *completion) DEDUPE_CONTEXT_TIMED_OUT)) { /* * This context completed between the time the timeout fired, and now. We - * can treat it as a a successful query, its requestor is already enqueued + * can treat it as a successful query, its requestor is already enqueued * to process it. */ continue; From 5c45cd10c085f46148f1b6030c5d710d842dd1d0 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 26 Jan 2024 22:01:27 -0500 Subject: [PATCH 0615/1038] dm vdo index: fix various small nits Add braces around multi-line while loops and if statements. Also remove excess newlines. Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/index-layout.c | 10 +++++----- drivers/md/dm-vdo/index-page-map.c | 3 +-- drivers/md/dm-vdo/index-session.c | 12 ++++++++---- drivers/md/dm-vdo/index.c | 3 +-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/drivers/md/dm-vdo/index-layout.c b/drivers/md/dm-vdo/index-layout.c index 6446a1f58641..f0511988ea07 100644 --- a/drivers/md/dm-vdo/index-layout.c +++ b/drivers/md/dm-vdo/index-layout.c @@ -386,9 +386,10 @@ static void define_sub_index_nonce(struct index_layout *layout) encode_u64_le(buffer, &offset, sil->sub_index.start_block); encode_u16_le(buffer, &offset, 0); sil->nonce = generate_secondary_nonce(primary_nonce, buffer, sizeof(buffer)); - if (sil->nonce == 0) + if (sil->nonce == 0) { sil->nonce = generate_secondary_nonce(~primary_nonce + 1, buffer, sizeof(buffer)); + } } static void setup_sub_index(struct index_layout *layout, u64 start_block, @@ -651,10 +652,11 @@ static int discard_index_state_data(struct index_layout *layout) saved_result = result; } - if (saved_result != UDS_SUCCESS) + if (saved_result != UDS_SUCCESS) { return uds_log_error_strerror(result, "%s: cannot destroy all index saves", __func__); + } return UDS_SUCCESS; } @@ -1242,9 +1244,7 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, "unknown superblock magic label"); if ((super->version < SUPER_VERSION_MINIMUM) || - (super->version == 4) || - (super->version == 5) || - (super->version == 6) || + (super->version == 4) || (super->version == 5) || (super->version == 6) || (super->version > SUPER_VERSION_MAXIMUM)) { return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, "unknown superblock version number %u", diff --git a/drivers/md/dm-vdo/index-page-map.c b/drivers/md/dm-vdo/index-page-map.c index 190d12848ba2..acfff74f37bc 100644 --- a/drivers/md/dm-vdo/index-page-map.c +++ b/drivers/md/dm-vdo/index-page-map.c @@ -100,8 +100,7 @@ void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_nu u32 slot = chapter_number * map->entries_per_chapter; *lowest_list = ((index_page_number == 0) ? - 0 : - map->entries[slot + index_page_number - 1] + 1); + 0 : map->entries[slot + index_page_number - 1] + 1); *highest_list = ((index_page_number < map->entries_per_chapter) ? map->entries[slot + index_page_number] : map->geometry->delta_lists_per_chapter - 1); diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c index 73c2927f74ce..c261453b9871 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/index-session.c @@ -392,9 +392,10 @@ int uds_open_index(enum uds_open_index_type open_type, static void wait_for_no_requests_in_progress(struct uds_index_session *index_session) { uds_lock_mutex(&index_session->request_mutex); - while (index_session->request_count > 0) + while (index_session->request_count > 0) { uds_wait_cond(&index_session->request_cond, &index_session->request_mutex); + } uds_unlock_mutex(&index_session->request_mutex); } @@ -619,9 +620,10 @@ int uds_close_index(struct uds_index_session *index_session) /* Wait for any current index state change to complete. */ uds_lock_mutex(&index_session->request_mutex); while ((index_session->state & IS_FLAG_WAITING) || - (index_session->state & IS_FLAG_CLOSING)) + (index_session->state & IS_FLAG_CLOSING)) { uds_wait_cond(&index_session->request_cond, &index_session->request_mutex); + } if (index_session->state & IS_FLAG_SUSPENDED) { uds_log_info("Index session is suspended"); @@ -660,9 +662,10 @@ int uds_destroy_index_session(struct uds_index_session *index_session) /* Wait for any current index state change to complete. */ uds_lock_mutex(&index_session->request_mutex); while ((index_session->state & IS_FLAG_WAITING) || - (index_session->state & IS_FLAG_CLOSING)) + (index_session->state & IS_FLAG_CLOSING)) { uds_wait_cond(&index_session->request_cond, &index_session->request_mutex); + } if (index_session->state & IS_FLAG_DESTROYING) { uds_unlock_mutex(&index_session->request_mutex); @@ -686,9 +689,10 @@ int uds_destroy_index_session(struct uds_index_session *index_session) /* Wait until the load exits before proceeding. */ uds_lock_mutex(&index_session->request_mutex); - while (index_session->state & IS_FLAG_LOADING) + while (index_session->state & IS_FLAG_LOADING) { uds_wait_cond(&index_session->request_cond, &index_session->request_mutex); + } uds_unlock_mutex(&index_session->request_mutex); } diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index 28ae54efaced..3d2a673b3f52 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -1330,8 +1330,7 @@ int uds_save_index(struct uds_index *index) uds_wait_for_idle_index(index); index->prev_save = index->last_save; index->last_save = ((index->newest_virtual_chapter == 0) ? - NO_LAST_SAVE : - index->newest_virtual_chapter - 1); + NO_LAST_SAVE : index->newest_virtual_chapter - 1); uds_log_info("beginning save (vcn %llu)", (unsigned long long) index->last_save); result = uds_save_index_state(index->layout, index); From 7f67d0f1c83939e3282a1311de58341d615d42b8 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 29 Jan 2024 19:51:45 -0500 Subject: [PATCH 0616/1038] dm vdo: rename struct geometry to index_geometry Signed-off-by: Mike Snitzer Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/chapter-index.c | 14 ++++---- drivers/md/dm-vdo/chapter-index.h | 10 +++--- drivers/md/dm-vdo/config.c | 18 +++++----- drivers/md/dm-vdo/config.h | 2 +- drivers/md/dm-vdo/geometry.c | 57 +++++++++++++++--------------- drivers/md/dm-vdo/geometry.h | 40 +++++++++++---------- drivers/md/dm-vdo/hash-utils.h | 4 +-- drivers/md/dm-vdo/index-layout.c | 2 +- drivers/md/dm-vdo/index-page-map.c | 6 ++-- drivers/md/dm-vdo/index-page-map.h | 6 ++-- drivers/md/dm-vdo/index.c | 12 +++---- drivers/md/dm-vdo/open-chapter.c | 4 +-- drivers/md/dm-vdo/open-chapter.h | 4 +-- drivers/md/dm-vdo/sparse-cache.c | 8 ++--- drivers/md/dm-vdo/sparse-cache.h | 2 +- drivers/md/dm-vdo/volume-index.c | 12 +++---- drivers/md/dm-vdo/volume.c | 38 ++++++++++---------- drivers/md/dm-vdo/volume.h | 2 +- 18 files changed, 121 insertions(+), 120 deletions(-) diff --git a/drivers/md/dm-vdo/chapter-index.c b/drivers/md/dm-vdo/chapter-index.c index 27a2d95a2bef..7285a842e888 100644 --- a/drivers/md/dm-vdo/chapter-index.c +++ b/drivers/md/dm-vdo/chapter-index.c @@ -13,7 +13,7 @@ #include "uds.h" int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, - const struct geometry *geometry, u64 volume_nonce) + const struct index_geometry *geometry, u64 volume_nonce) { int result; size_t memory_size; @@ -79,7 +79,7 @@ int uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, u32 list_number; const u8 *found_name; bool found; - const struct geometry *geometry = chapter_index->geometry; + const struct index_geometry *geometry = chapter_index->geometry; u64 chapter_number = chapter_index->virtual_chapter_number; u32 record_pages = geometry->record_pages_per_chapter; @@ -128,7 +128,7 @@ int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, struct delta_index_stats stats; u64 nonce = chapter_index->volume_nonce; u64 chapter_number = chapter_index->virtual_chapter_number; - const struct geometry *geometry = chapter_index->geometry; + const struct index_geometry *geometry = chapter_index->geometry; u32 list_count = geometry->delta_lists_per_chapter; unsigned int removals = 0; struct delta_index_entry entry; @@ -206,8 +206,8 @@ int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, /* Make a new chapter index page, initializing it with the data from a given index_page buffer. */ int uds_initialize_chapter_index_page(struct delta_index_page *index_page, - const struct geometry *geometry, u8 *page_buffer, - u64 volume_nonce) + const struct index_geometry *geometry, + u8 *page_buffer, u64 volume_nonce) { return uds_initialize_delta_index_page(index_page, volume_nonce, geometry->chapter_mean_delta, @@ -217,7 +217,7 @@ int uds_initialize_chapter_index_page(struct delta_index_page *index_page, /* Validate a chapter index page read during rebuild. */ int uds_validate_chapter_index_page(const struct delta_index_page *index_page, - const struct geometry *geometry) + const struct index_geometry *geometry) { int result; const struct delta_index *delta_index = &index_page->delta_index; @@ -266,7 +266,7 @@ int uds_validate_chapter_index_page(const struct delta_index_page *index_page, * the name. */ int uds_search_chapter_index_page(struct delta_index_page *index_page, - const struct geometry *geometry, + const struct index_geometry *geometry, const struct uds_record_name *name, u16 *record_page_ptr) { diff --git a/drivers/md/dm-vdo/chapter-index.h b/drivers/md/dm-vdo/chapter-index.h index ad1f61b6ab9a..e1bc7e31a2a5 100644 --- a/drivers/md/dm-vdo/chapter-index.h +++ b/drivers/md/dm-vdo/chapter-index.h @@ -24,7 +24,7 @@ enum { }; struct open_chapter_index { - const struct geometry *geometry; + const struct index_geometry *geometry; struct delta_index delta_index; u64 virtual_chapter_number; u64 volume_nonce; @@ -32,7 +32,7 @@ struct open_chapter_index { }; int __must_check uds_make_open_chapter_index(struct open_chapter_index **chapter_index, - const struct geometry *geometry, + const struct index_geometry *geometry, u64 volume_nonce); void uds_free_open_chapter_index(struct open_chapter_index *chapter_index); @@ -49,14 +49,14 @@ int __must_check uds_pack_open_chapter_index_page(struct open_chapter_index *cha bool last_page, u32 *lists_packed); int __must_check uds_initialize_chapter_index_page(struct delta_index_page *index_page, - const struct geometry *geometry, + const struct index_geometry *geometry, u8 *page_buffer, u64 volume_nonce); int __must_check uds_validate_chapter_index_page(const struct delta_index_page *index_page, - const struct geometry *geometry); + const struct index_geometry *geometry); int __must_check uds_search_chapter_index_page(struct delta_index_page *index_page, - const struct geometry *geometry, + const struct index_geometry *geometry, const struct uds_record_name *name, u16 *record_page_ptr); diff --git a/drivers/md/dm-vdo/config.c b/drivers/md/dm-vdo/config.c index 96e27aeecc25..cd5c36de6e2b 100644 --- a/drivers/md/dm-vdo/config.c +++ b/drivers/md/dm-vdo/config.c @@ -28,10 +28,10 @@ static bool is_version(const u8 *version, u8 *buffer) } static bool are_matching_configurations(struct configuration *saved_config, - struct geometry *saved_geometry, + struct index_geometry *saved_geometry, struct configuration *user) { - struct geometry *geometry = user->geometry; + struct index_geometry *geometry = user->geometry; bool result = true; if (saved_geometry->record_pages_per_chapter != geometry->record_pages_per_chapter) { @@ -97,7 +97,7 @@ int uds_validate_config_contents(struct buffered_reader *reader, { int result; struct configuration config; - struct geometry geometry; + struct index_geometry geometry; u8 version_buffer[INDEX_CONFIG_VERSION_LENGTH]; u32 bytes_per_page; u8 buffer[sizeof(struct uds_configuration_6_02)]; @@ -177,7 +177,7 @@ int uds_write_config_contents(struct buffered_writer *writer, struct configuration *config, u32 version) { int result; - struct geometry *geometry = config->geometry; + struct index_geometry *geometry = config->geometry; u8 buffer[sizeof(struct uds_configuration_8_02)]; size_t offset = 0; @@ -331,9 +331,9 @@ int uds_make_configuration(const struct uds_parameters *params, if (result != UDS_SUCCESS) return result; - result = uds_make_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter, - chapters_per_volume, sparse_chapters_per_volume, 0, 0, - &config->geometry); + result = uds_make_index_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter, + chapters_per_volume, sparse_chapters_per_volume, + 0, 0, &config->geometry); if (result != UDS_SUCCESS) { uds_free_configuration(config); return result; @@ -357,14 +357,14 @@ int uds_make_configuration(const struct uds_parameters *params, void uds_free_configuration(struct configuration *config) { if (config != NULL) { - uds_free_geometry(config->geometry); + uds_free_index_geometry(config->geometry); uds_free(config); } } void uds_log_configuration(struct configuration *config) { - struct geometry *geometry = config->geometry; + struct index_geometry *geometry = config->geometry; uds_log_debug("Configuration:"); uds_log_debug(" Record pages per chapter: %10u", geometry->record_pages_per_chapter); diff --git a/drivers/md/dm-vdo/config.h b/drivers/md/dm-vdo/config.h index 72430d47f983..b79fe833d825 100644 --- a/drivers/md/dm-vdo/config.h +++ b/drivers/md/dm-vdo/config.h @@ -37,7 +37,7 @@ struct configuration { /* Parameters for the volume */ /* The volume layout */ - struct geometry *geometry; + struct index_geometry *geometry; /* Index owner's nonce */ u64 nonce; diff --git a/drivers/md/dm-vdo/geometry.c b/drivers/md/dm-vdo/geometry.c index 1ebbef75f842..0e83bba4184a 100644 --- a/drivers/md/dm-vdo/geometry.c +++ b/drivers/md/dm-vdo/geometry.c @@ -52,18 +52,15 @@ * chapter it was moved to. */ -int uds_make_geometry(size_t bytes_per_page, - u32 record_pages_per_chapter, - u32 chapters_per_volume, - u32 sparse_chapters_per_volume, - u64 remapped_virtual, - u64 remapped_physical, - struct geometry **geometry_ptr) +int uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, + u32 chapters_per_volume, u32 sparse_chapters_per_volume, + u64 remapped_virtual, u64 remapped_physical, + struct index_geometry **geometry_ptr) { int result; - struct geometry *geometry; + struct index_geometry *geometry; - result = uds_allocate(1, struct geometry, "geometry", &geometry); + result = uds_allocate(1, struct index_geometry, "geometry", &geometry); if (result != UDS_SUCCESS) return result; @@ -110,27 +107,28 @@ int uds_make_geometry(size_t bytes_per_page, return UDS_SUCCESS; } -int uds_copy_geometry(struct geometry *source, struct geometry **geometry_ptr) +int uds_copy_index_geometry(struct index_geometry *source, + struct index_geometry **geometry_ptr) { - return uds_make_geometry(source->bytes_per_page, - source->record_pages_per_chapter, - source->chapters_per_volume, - source->sparse_chapters_per_volume, - source->remapped_virtual, source->remapped_physical, - geometry_ptr); + return uds_make_index_geometry(source->bytes_per_page, + source->record_pages_per_chapter, + source->chapters_per_volume, + source->sparse_chapters_per_volume, + source->remapped_virtual, source->remapped_physical, + geometry_ptr); } -void uds_free_geometry(struct geometry *geometry) +void uds_free_index_geometry(struct index_geometry *geometry) { uds_free(geometry); } -u32 __must_check uds_map_to_physical_chapter(const struct geometry *geometry, +u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry, u64 virtual_chapter) { u64 delta; - if (!uds_is_reduced_geometry(geometry)) + if (!uds_is_reduced_index_geometry(geometry)) return virtual_chapter % geometry->chapters_per_volume; if (likely(virtual_chapter > geometry->remapped_virtual)) { @@ -153,25 +151,26 @@ u32 __must_check uds_map_to_physical_chapter(const struct geometry *geometry, } /* Check whether any sparse chapters are in use. */ -bool uds_has_sparse_chapters(const struct geometry *geometry, u64 oldest_virtual_chapter, - u64 newest_virtual_chapter) +bool uds_has_sparse_chapters(const struct index_geometry *geometry, + u64 oldest_virtual_chapter, u64 newest_virtual_chapter) { - return uds_is_sparse_geometry(geometry) && - ((newest_virtual_chapter - oldest_virtual_chapter + 1) > - geometry->dense_chapters_per_volume); + return uds_is_sparse_index_geometry(geometry) && + ((newest_virtual_chapter - oldest_virtual_chapter + 1) > + geometry->dense_chapters_per_volume); } -bool uds_is_chapter_sparse(const struct geometry *geometry, u64 oldest_virtual_chapter, - u64 newest_virtual_chapter, u64 virtual_chapter_number) +bool uds_is_chapter_sparse(const struct index_geometry *geometry, + u64 oldest_virtual_chapter, u64 newest_virtual_chapter, + u64 virtual_chapter_number) { return uds_has_sparse_chapters(geometry, oldest_virtual_chapter, newest_virtual_chapter) && - ((virtual_chapter_number + geometry->dense_chapters_per_volume) <= - newest_virtual_chapter); + ((virtual_chapter_number + geometry->dense_chapters_per_volume) <= + newest_virtual_chapter); } /* Calculate how many chapters to expire after opening the newest chapter. */ -u32 uds_chapters_to_expire(const struct geometry *geometry, u64 newest_chapter) +u32 uds_chapters_to_expire(const struct index_geometry *geometry, u64 newest_chapter) { /* If the index isn't full yet, don't expire anything. */ if (newest_chapter < geometry->chapters_per_volume) diff --git a/drivers/md/dm-vdo/geometry.h b/drivers/md/dm-vdo/geometry.h index cff69087cd79..9a4a66ac2e46 100644 --- a/drivers/md/dm-vdo/geometry.h +++ b/drivers/md/dm-vdo/geometry.h @@ -3,18 +3,18 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_GEOMETRY_H -#define UDS_GEOMETRY_H +#ifndef UDS_INDEX_GEOMETRY_H +#define UDS_INDEX_GEOMETRY_H #include "uds.h" /* - * The geometry records parameters that define the layout of a UDS index volume, and the size and + * The index_geometry records parameters that define the layout of a UDS index volume, and the size and * shape of various index structures. It is created when the index is created, and is referenced by * many index sub-components. */ -struct geometry { +struct index_geometry { /* Size of a chapter page, in bytes */ size_t bytes_per_page; /* Number of record pages in a chapter */ @@ -95,44 +95,46 @@ enum { HEADER_PAGES_PER_VOLUME = 1, }; -int __must_check uds_make_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, - u32 chapters_per_volume, - u32 sparse_chapters_per_volume, u64 remapped_virtual, - u64 remapped_physical, - struct geometry **geometry_ptr); +int __must_check uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, + u32 chapters_per_volume, + u32 sparse_chapters_per_volume, u64 remapped_virtual, + u64 remapped_physical, + struct index_geometry **geometry_ptr); -int __must_check uds_copy_geometry(struct geometry *source, - struct geometry **geometry_ptr); +int __must_check uds_copy_index_geometry(struct index_geometry *source, + struct index_geometry **geometry_ptr); -void uds_free_geometry(struct geometry *geometry); +void uds_free_index_geometry(struct index_geometry *geometry); -u32 __must_check uds_map_to_physical_chapter(const struct geometry *geometry, +u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry, u64 virtual_chapter); /* * Check whether this geometry is reduced by a chapter. This will only be true if the volume was * converted from a non-lvm volume to an lvm volume. */ -static inline bool __must_check uds_is_reduced_geometry(const struct geometry *geometry) +static inline bool __must_check +uds_is_reduced_index_geometry(const struct index_geometry *geometry) { return !!(geometry->chapters_per_volume & 1); } -static inline bool __must_check uds_is_sparse_geometry(const struct geometry *geometry) +static inline bool __must_check +uds_is_sparse_index_geometry(const struct index_geometry *geometry) { return geometry->sparse_chapters_per_volume > 0; } -bool __must_check uds_has_sparse_chapters(const struct geometry *geometry, +bool __must_check uds_has_sparse_chapters(const struct index_geometry *geometry, u64 oldest_virtual_chapter, u64 newest_virtual_chapter); -bool __must_check uds_is_chapter_sparse(const struct geometry *geometry, +bool __must_check uds_is_chapter_sparse(const struct index_geometry *geometry, u64 oldest_virtual_chapter, u64 newest_virtual_chapter, u64 virtual_chapter_number); -u32 __must_check uds_chapters_to_expire(const struct geometry *geometry, +u32 __must_check uds_chapters_to_expire(const struct index_geometry *geometry, u64 newest_chapter); -#endif /* UDS_GEOMETRY_H */ +#endif /* UDS_INDEX_GEOMETRY_H */ diff --git a/drivers/md/dm-vdo/hash-utils.h b/drivers/md/dm-vdo/hash-utils.h index 9ee729c14f2e..e22be69695be 100644 --- a/drivers/md/dm-vdo/hash-utils.h +++ b/drivers/md/dm-vdo/hash-utils.h @@ -43,7 +43,7 @@ static inline u32 uds_extract_sampling_bytes(const struct uds_record_name *name) /* Compute the chapter delta list for a given name. */ static inline u32 uds_hash_to_chapter_delta_list(const struct uds_record_name *name, - const struct geometry *geometry) + const struct index_geometry *geometry) { return ((uds_extract_chapter_index_bytes(name) >> geometry->chapter_address_bits) & ((1 << geometry->chapter_delta_list_bits) - 1)); @@ -51,7 +51,7 @@ static inline u32 uds_hash_to_chapter_delta_list(const struct uds_record_name *n /* Compute the chapter delta address for a given name. */ static inline u32 uds_hash_to_chapter_delta_address(const struct uds_record_name *name, - const struct geometry *geometry) + const struct index_geometry *geometry) { return uds_extract_chapter_index_bytes(name) & ((1 << geometry->chapter_address_bits) - 1); } diff --git a/drivers/md/dm-vdo/index-layout.c b/drivers/md/dm-vdo/index-layout.c index f0511988ea07..bc67720659e7 100644 --- a/drivers/md/dm-vdo/index-layout.c +++ b/drivers/md/dm-vdo/index-layout.c @@ -226,7 +226,7 @@ static int __must_check compute_sizes(const struct configuration *config, struct save_layout_sizes *sls) { int result; - struct geometry *geometry = config->geometry; + struct index_geometry *geometry = config->geometry; memset(sls, 0, sizeof(*sls)); sls->save_count = MAX_SAVES; diff --git a/drivers/md/dm-vdo/index-page-map.c b/drivers/md/dm-vdo/index-page-map.c index acfff74f37bc..f3748a915c03 100644 --- a/drivers/md/dm-vdo/index-page-map.c +++ b/drivers/md/dm-vdo/index-page-map.c @@ -28,12 +28,12 @@ enum { PAGE_MAP_MAGIC_LENGTH = sizeof(PAGE_MAP_MAGIC) - 1, }; -static inline u32 get_entry_count(const struct geometry *geometry) +static inline u32 get_entry_count(const struct index_geometry *geometry) { return geometry->chapters_per_volume * (geometry->index_pages_per_chapter - 1); } -int uds_make_index_page_map(const struct geometry *geometry, +int uds_make_index_page_map(const struct index_geometry *geometry, struct index_page_map **map_ptr) { int result; @@ -106,7 +106,7 @@ void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_nu map->geometry->delta_lists_per_chapter - 1); } -u64 uds_compute_index_page_map_save_size(const struct geometry *geometry) +u64 uds_compute_index_page_map_save_size(const struct index_geometry *geometry) { return PAGE_MAP_MAGIC_LENGTH + sizeof(u64) + sizeof(u16) * get_entry_count(geometry); } diff --git a/drivers/md/dm-vdo/index-page-map.h b/drivers/md/dm-vdo/index-page-map.h index 51ac9b81362f..b327c0bb9656 100644 --- a/drivers/md/dm-vdo/index-page-map.h +++ b/drivers/md/dm-vdo/index-page-map.h @@ -16,13 +16,13 @@ */ struct index_page_map { - const struct geometry *geometry; + const struct index_geometry *geometry; u64 last_update; u32 entries_per_chapter; u16 *entries; }; -int __must_check uds_make_index_page_map(const struct geometry *geometry, +int __must_check uds_make_index_page_map(const struct index_geometry *geometry, struct index_page_map **map_ptr); void uds_free_index_page_map(struct index_page_map *map); @@ -45,6 +45,6 @@ void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_nu u32 index_page_number, u32 *lowest_list, u32 *highest_list); -u64 uds_compute_index_page_map_save_size(const struct geometry *geometry); +u64 uds_compute_index_page_map_save_size(const struct index_geometry *geometry); #endif /* UDS_INDEX_PAGE_MAP_H */ diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index 3d2a673b3f52..cf7cdc909a31 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -153,7 +153,7 @@ static int simulate_index_zone_barrier_message(struct index_zone *zone, u64 sparse_virtual_chapter; if ((zone->index->zone_count > 1) || - !uds_is_sparse_geometry(zone->index->volume->geometry)) + !uds_is_sparse_index_geometry(zone->index->volume->geometry)) return UDS_SUCCESS; sparse_virtual_chapter = triage_index_request(zone->index, request); @@ -472,7 +472,7 @@ static int search_index_zone(struct index_zone *zone, struct uds_request *reques found = true; } else if (request->location == UDS_LOCATION_UNAVAILABLE) { found = false; - } else if (uds_is_sparse_geometry(zone->index->volume->geometry) && + } else if (uds_is_sparse_index_geometry(zone->index->volume->geometry) && !uds_is_volume_index_sample(zone->index->volume_index, &request->record_name)) { result = search_sparse_cache_in_zone(zone, request, NO_CHAPTER, @@ -647,7 +647,7 @@ static void execute_zone_request(struct uds_request *request) } static int initialize_index_queues(struct uds_index *index, - const struct geometry *geometry) + const struct index_geometry *geometry) { int result; unsigned int i; @@ -660,7 +660,7 @@ static int initialize_index_queues(struct uds_index *index, } /* The triage queue is only needed for sparse multi-zone indexes. */ - if ((index->zone_count > 1) && uds_is_sparse_geometry(geometry)) { + if ((index->zone_count > 1) && uds_is_sparse_index_geometry(geometry)) { result = uds_make_request_queue("triageW", &triage_request, &index->triage_queue); if (result != UDS_SUCCESS) @@ -840,7 +840,7 @@ static int rebuild_index_page_map(struct uds_index *index, u64 vcn) { int result; struct delta_index_page *chapter_index_page; - struct geometry *geometry = index->volume->geometry; + struct index_geometry *geometry = index->volume->geometry; u32 chapter = uds_map_to_physical_chapter(geometry, vcn); u32 expected_list_number = 0; u32 index_page_number; @@ -987,7 +987,7 @@ static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse) int result; u32 i; u32 j; - const struct geometry *geometry; + const struct index_geometry *geometry; u32 physical_chapter; if (check_for_suspend(index)) { diff --git a/drivers/md/dm-vdo/open-chapter.c b/drivers/md/dm-vdo/open-chapter.c index 45cec2c3edb2..d9d6e5d45bfb 100644 --- a/drivers/md/dm-vdo/open-chapter.c +++ b/drivers/md/dm-vdo/open-chapter.c @@ -61,7 +61,7 @@ static inline size_t slots_size(size_t slot_count) return sizeof(struct open_chapter_zone_slot) * slot_count; } -int uds_make_open_chapter(const struct geometry *geometry, unsigned int zone_count, +int uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zone_count, struct open_chapter_zone **open_chapter_ptr) { int result; @@ -346,7 +346,7 @@ int uds_save_open_chapter(struct uds_index *index, struct buffered_writer *write return uds_flush_buffered_writer(writer); } -u64 uds_compute_saved_open_chapter_size(struct geometry *geometry) +u64 uds_compute_saved_open_chapter_size(struct index_geometry *geometry) { unsigned int records_per_chapter = geometry->records_per_chapter; diff --git a/drivers/md/dm-vdo/open-chapter.h b/drivers/md/dm-vdo/open-chapter.h index eb11d897f20a..a4250bb19525 100644 --- a/drivers/md/dm-vdo/open-chapter.h +++ b/drivers/md/dm-vdo/open-chapter.h @@ -43,7 +43,7 @@ struct open_chapter_zone { struct open_chapter_zone_slot slots[]; }; -int __must_check uds_make_open_chapter(const struct geometry *geometry, +int __must_check uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zone_count, struct open_chapter_zone **open_chapter_ptr); @@ -74,6 +74,6 @@ int __must_check uds_save_open_chapter(struct uds_index *index, int __must_check uds_load_open_chapter(struct uds_index *index, struct buffered_reader *reader); -u64 uds_compute_saved_open_chapter_size(struct geometry *geometry); +u64 uds_compute_saved_open_chapter_size(struct index_geometry *geometry); #endif /* UDS_OPEN_CHAPTER_H */ diff --git a/drivers/md/dm-vdo/sparse-cache.c b/drivers/md/dm-vdo/sparse-cache.c index f80cde3f93ee..5b41c94f53fa 100644 --- a/drivers/md/dm-vdo/sparse-cache.c +++ b/drivers/md/dm-vdo/sparse-cache.c @@ -142,7 +142,7 @@ struct search_list { }; struct sparse_cache { - const struct geometry *geometry; + const struct index_geometry *geometry; unsigned int capacity; unsigned int zone_count; @@ -157,7 +157,7 @@ struct sparse_cache { }; static int __must_check initialize_cached_chapter_index(struct cached_chapter_index *chapter, - const struct geometry *geometry) + const struct index_geometry *geometry) { int result; @@ -197,7 +197,7 @@ static int __must_check make_search_list(struct sparse_cache *cache, return UDS_SUCCESS; } -int uds_make_sparse_cache(const struct geometry *geometry, unsigned int capacity, +int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int capacity, unsigned int zone_count, struct sparse_cache **cache_ptr) { int result; @@ -512,7 +512,7 @@ static inline bool should_skip_chapter(struct cached_chapter_index *chapter, } static int __must_check search_cached_chapter_index(struct cached_chapter_index *chapter, - const struct geometry *geometry, + const struct index_geometry *geometry, const struct index_page_map *index_page_map, const struct uds_record_name *name, u16 *record_page_ptr) diff --git a/drivers/md/dm-vdo/sparse-cache.h b/drivers/md/dm-vdo/sparse-cache.h index 522027395cba..90b0be155453 100644 --- a/drivers/md/dm-vdo/sparse-cache.h +++ b/drivers/md/dm-vdo/sparse-cache.h @@ -26,7 +26,7 @@ struct index_zone; struct sparse_cache; -int __must_check uds_make_sparse_cache(const struct geometry *geometry, +int __must_check uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int capacity, unsigned int zone_count, struct sparse_cache **cache_ptr); diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c index 11dceef0ab60..0f4b9510e7ea 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/volume-index.c @@ -81,11 +81,11 @@ struct sub_index_parameters { struct split_config { /* The hook subindex configuration */ struct configuration hook_config; - struct geometry hook_geometry; + struct index_geometry hook_geometry; /* The non-hook subindex configuration */ struct configuration non_hook_config; - struct geometry non_hook_geometry; + struct index_geometry non_hook_geometry; }; struct chapter_range { @@ -204,7 +204,7 @@ static int compute_volume_sub_index_parameters(const struct configuration *confi u64 index_size_in_bits; size_t expected_index_size; u64 min_delta_lists = MAX_ZONES * MAX_ZONES; - struct geometry *geometry = config->geometry; + struct index_geometry *geometry = config->geometry; u64 records_per_chapter = geometry->records_per_chapter; params->chapter_count = geometry->chapters_per_volume; @@ -214,7 +214,7 @@ static int compute_volume_sub_index_parameters(const struct configuration *confi * index delta list. */ rounded_chapters = params->chapter_count; - if (uds_is_reduced_geometry(geometry)) + if (uds_is_reduced_index_geometry(geometry)) rounded_chapters += 1; delta_list_records = records_per_chapter * rounded_chapters; address_count = config->volume_index_mean_delta * DELTA_LIST_SIZE; @@ -353,7 +353,7 @@ static int compute_volume_index_save_bytes(const struct configuration *config, struct split_config split; int result; - if (!uds_is_sparse_geometry(config->geometry)) + if (!uds_is_sparse_index_geometry(config->geometry)) return compute_volume_sub_index_save_bytes(config, bytes); split_configuration(config, &split); @@ -1236,7 +1236,7 @@ int uds_make_volume_index(const struct configuration *config, u64 volume_nonce, volume_index->zone_count = config->zone_count; - if (!uds_is_sparse_geometry(config->geometry)) { + if (!uds_is_sparse_index_geometry(config->geometry)) { result = initialize_volume_sub_index(config, volume_nonce, 'm', &volume_index->vi_non_hook); if (result != UDS_SUCCESS) { diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 65bf6d32e5d0..0cfeae0c5d5d 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -84,22 +84,22 @@ union invalidate_counter { }; }; -static inline u32 map_to_page_number(struct geometry *geometry, u32 physical_page) +static inline u32 map_to_page_number(struct index_geometry *geometry, u32 physical_page) { return (physical_page - HEADER_PAGES_PER_VOLUME) % geometry->pages_per_chapter; } -static inline u32 map_to_chapter_number(struct geometry *geometry, u32 physical_page) +static inline u32 map_to_chapter_number(struct index_geometry *geometry, u32 physical_page) { return (physical_page - HEADER_PAGES_PER_VOLUME) / geometry->pages_per_chapter; } -static inline bool is_record_page(struct geometry *geometry, u32 physical_page) +static inline bool is_record_page(struct index_geometry *geometry, u32 physical_page) { return map_to_page_number(geometry, physical_page) >= geometry->index_pages_per_chapter; } -static u32 map_to_physical_page(const struct geometry *geometry, u32 chapter, u32 page) +static u32 map_to_physical_page(const struct index_geometry *geometry, u32 chapter, u32 page) { /* Page zero is the header page, so the first chapter index page is page one. */ return HEADER_PAGES_PER_VOLUME + (geometry->pages_per_chapter * chapter) + page; @@ -423,7 +423,7 @@ static int init_chapter_index_page(const struct volume *volume, u8 *index_page, u32 ci_chapter; u32 lowest_list; u32 highest_list; - struct geometry *geometry = volume->geometry; + struct index_geometry *geometry = volume->geometry; int result; result = uds_initialize_chapter_index_page(chapter_index_page, geometry, @@ -469,7 +469,7 @@ static int initialize_index_page(const struct volume *volume, u32 physical_page, static bool search_record_page(const u8 record_page[], const struct uds_record_name *name, - const struct geometry *geometry, + const struct index_geometry *geometry, struct uds_record_data *metadata) { /* @@ -882,7 +882,7 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req u32 chapter, u16 record_page_number, bool *found) { struct cached_page *record_page; - struct geometry *geometry = volume->geometry; + struct index_geometry *geometry = volume->geometry; int result; u32 physical_page, page_number; @@ -924,7 +924,7 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req void uds_prefetch_volume_chapter(const struct volume *volume, u32 chapter) { - const struct geometry *geometry = volume->geometry; + const struct index_geometry *geometry = volume->geometry; u32 physical_page = map_to_physical_page(geometry, chapter, 0); dm_bufio_prefetch(volume->client, physical_page, geometry->pages_per_chapter); @@ -936,7 +936,7 @@ int uds_read_chapter_index_from_volume(const struct volume *volume, u64 virtual_ { int result; u32 i; - const struct geometry *geometry = volume->geometry; + const struct index_geometry *geometry = volume->geometry; u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter); u32 physical_page = map_to_physical_page(geometry, physical_chapter, 0); @@ -995,7 +995,7 @@ int uds_search_volume_page_cache_for_rebuild(struct volume *volume, u64 virtual_chapter, bool *found) { int result; - struct geometry *geometry = volume->geometry; + struct index_geometry *geometry = volume->geometry; struct cached_page *page; u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter); u32 index_page_number; @@ -1096,7 +1096,7 @@ static int donate_index_page_locked(struct volume *volume, u32 physical_chapter, static int write_index_pages(struct volume *volume, u32 physical_chapter_number, struct open_chapter_index *chapter_index) { - struct geometry *geometry = volume->geometry; + struct index_geometry *geometry = volume->geometry; struct dm_buffer *page_buffer; u32 first_index_page = map_to_physical_page(geometry, physical_chapter_number, 0); u32 delta_list_number = 0; @@ -1207,7 +1207,7 @@ static int write_record_pages(struct volume *volume, u32 physical_chapter_number const struct uds_volume_record *records) { u32 record_page_number; - struct geometry *geometry = volume->geometry; + struct index_geometry *geometry = volume->geometry; struct dm_buffer *page_buffer; const struct uds_volume_record *next_record = records; u32 first_record_page = map_to_physical_page(geometry, physical_chapter_number, @@ -1268,7 +1268,7 @@ int uds_write_chapter(struct volume *volume, struct open_chapter_index *chapter_ static void probe_chapter(struct volume *volume, u32 chapter_number, u64 *virtual_chapter_number) { - const struct geometry *geometry = volume->geometry; + const struct index_geometry *geometry = volume->geometry; u32 expected_list_number = 0; u32 i; u64 vcn = BAD_CHAPTER; @@ -1352,7 +1352,7 @@ static void find_real_end_of_volume(struct volume *volume, u32 limit, u32 *limit static int find_chapter_limits(struct volume *volume, u32 chapter_limit, u64 *lowest_vcn, u64 *highest_vcn) { - struct geometry *geometry = volume->geometry; + struct index_geometry *geometry = volume->geometry; u64 zero_vcn; u64 lowest = BAD_CHAPTER; u64 highest = BAD_CHAPTER; @@ -1490,7 +1490,7 @@ int __must_check uds_replace_volume_storage(struct volume *volume, } static int __must_check initialize_page_cache(struct page_cache *cache, - const struct geometry *geometry, + const struct index_geometry *geometry, u32 chapters_in_cache, unsigned int zone_count) { @@ -1543,7 +1543,7 @@ int uds_make_volume(const struct configuration *config, struct index_layout *lay { unsigned int i; struct volume *volume = NULL; - struct geometry *geometry; + struct index_geometry *geometry; unsigned int reserved_buffers; int result; @@ -1553,7 +1553,7 @@ int uds_make_volume(const struct configuration *config, struct index_layout *lay volume->nonce = uds_get_volume_nonce(layout); - result = uds_copy_geometry(config->geometry, &volume->geometry); + result = uds_copy_index_geometry(config->geometry, &volume->geometry); if (result != UDS_SUCCESS) { uds_free_volume(volume); return uds_log_warning_strerror(result, @@ -1567,7 +1567,7 @@ int uds_make_volume(const struct configuration *config, struct index_layout *lay */ reserved_buffers = config->cache_chapters * geometry->record_pages_per_chapter; reserved_buffers += 1; - if (uds_is_sparse_geometry(geometry)) + if (uds_is_sparse_index_geometry(geometry)) reserved_buffers += (config->cache_chapters * geometry->index_pages_per_chapter); volume->reserved_buffers = reserved_buffers; result = uds_open_volume_bufio(layout, geometry->bytes_per_page, @@ -1592,7 +1592,7 @@ int uds_make_volume(const struct configuration *config, struct index_layout *lay return result; } - if (uds_is_sparse_geometry(geometry)) { + if (uds_is_sparse_index_geometry(geometry)) { size_t page_size = sizeof(struct delta_index_page) + geometry->bytes_per_page; result = uds_make_sparse_cache(geometry, config->cache_chapters, diff --git a/drivers/md/dm-vdo/volume.h b/drivers/md/dm-vdo/volume.h index 0a96af964887..e938dd31934f 100644 --- a/drivers/md/dm-vdo/volume.h +++ b/drivers/md/dm-vdo/volume.h @@ -96,7 +96,7 @@ struct page_cache { }; struct volume { - struct geometry *geometry; + struct index_geometry *geometry; struct dm_bufio_client *client; u64 nonce; size_t cache_size; From 952b57a58d400e5495d0f5d402c5ff9737dd5995 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 29 Jan 2024 19:51:46 -0500 Subject: [PATCH 0617/1038] dm vdo: rename struct configuration to uds_configuration Signed-off-by: Mike Snitzer Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/config.c | 20 ++++++++++---------- drivers/md/dm-vdo/config.h | 14 +++++++------- drivers/md/dm-vdo/index-layout.c | 16 ++++++++-------- drivers/md/dm-vdo/index-layout.h | 2 +- drivers/md/dm-vdo/index-session.c | 2 +- drivers/md/dm-vdo/index.c | 2 +- drivers/md/dm-vdo/index.h | 2 +- drivers/md/dm-vdo/volume-index.c | 18 +++++++++--------- drivers/md/dm-vdo/volume-index.h | 4 ++-- drivers/md/dm-vdo/volume.c | 2 +- drivers/md/dm-vdo/volume.h | 2 +- 11 files changed, 42 insertions(+), 42 deletions(-) diff --git a/drivers/md/dm-vdo/config.c b/drivers/md/dm-vdo/config.c index cd5c36de6e2b..e9c7e9bdbce0 100644 --- a/drivers/md/dm-vdo/config.c +++ b/drivers/md/dm-vdo/config.c @@ -27,9 +27,9 @@ static bool is_version(const u8 *version, u8 *buffer) return memcmp(version, buffer, INDEX_CONFIG_VERSION_LENGTH) == 0; } -static bool are_matching_configurations(struct configuration *saved_config, +static bool are_matching_configurations(struct uds_configuration *saved_config, struct index_geometry *saved_geometry, - struct configuration *user) + struct uds_configuration *user) { struct index_geometry *geometry = user->geometry; bool result = true; @@ -93,10 +93,10 @@ static bool are_matching_configurations(struct configuration *saved_config, /* Read the configuration and validate it against the provided one. */ int uds_validate_config_contents(struct buffered_reader *reader, - struct configuration *user_config) + struct uds_configuration *user_config) { int result; - struct configuration config; + struct uds_configuration config; struct index_geometry geometry; u8 version_buffer[INDEX_CONFIG_VERSION_LENGTH]; u32 bytes_per_page; @@ -174,7 +174,7 @@ int uds_validate_config_contents(struct buffered_reader *reader, * been reduced by one chapter. */ int uds_write_config_contents(struct buffered_writer *writer, - struct configuration *config, u32 version) + struct uds_configuration *config, u32 version) { int result; struct index_geometry *geometry = config->geometry; @@ -313,9 +313,9 @@ static unsigned int __must_check normalize_read_threads(unsigned int requested) } int uds_make_configuration(const struct uds_parameters *params, - struct configuration **config_ptr) + struct uds_configuration **config_ptr) { - struct configuration *config; + struct uds_configuration *config; u32 chapters_per_volume = 0; u32 record_pages_per_chapter = 0; u32 sparse_chapters_per_volume = 0; @@ -327,7 +327,7 @@ int uds_make_configuration(const struct uds_parameters *params, if (result != UDS_SUCCESS) return result; - result = uds_allocate(1, struct configuration, __func__, &config); + result = uds_allocate(1, struct uds_configuration, __func__, &config); if (result != UDS_SUCCESS) return result; @@ -354,7 +354,7 @@ int uds_make_configuration(const struct uds_parameters *params, return UDS_SUCCESS; } -void uds_free_configuration(struct configuration *config) +void uds_free_configuration(struct uds_configuration *config) { if (config != NULL) { uds_free_index_geometry(config->geometry); @@ -362,7 +362,7 @@ void uds_free_configuration(struct configuration *config) } } -void uds_log_configuration(struct configuration *config) +void uds_log_configuration(struct uds_configuration *config) { struct index_geometry *geometry = config->geometry; diff --git a/drivers/md/dm-vdo/config.h b/drivers/md/dm-vdo/config.h index b79fe833d825..7d19863800d6 100644 --- a/drivers/md/dm-vdo/config.h +++ b/drivers/md/dm-vdo/config.h @@ -11,7 +11,7 @@ #include "uds.h" /* - * The configuration records a variety of parameters used to configure a new UDS index. Some + * The uds_configuration records a variety of parameters used to configure a new UDS index. Some * parameters are provided by the client, while others are fixed or derived from user-supplied * values. It is created when an index is created, and it is recorded in the index metadata. */ @@ -24,7 +24,7 @@ enum { }; /* A set of configuration parameters for the indexer. */ -struct configuration { +struct uds_configuration { /* Storage device for the index */ struct block_device *bdev; @@ -109,16 +109,16 @@ struct uds_configuration_6_02 { } __packed; int __must_check uds_make_configuration(const struct uds_parameters *params, - struct configuration **config_ptr); + struct uds_configuration **config_ptr); -void uds_free_configuration(struct configuration *config); +void uds_free_configuration(struct uds_configuration *config); int __must_check uds_validate_config_contents(struct buffered_reader *reader, - struct configuration *config); + struct uds_configuration *config); int __must_check uds_write_config_contents(struct buffered_writer *writer, - struct configuration *config, u32 version); + struct uds_configuration *config, u32 version); -void uds_log_configuration(struct configuration *config); +void uds_log_configuration(struct uds_configuration *config); #endif /* UDS_CONFIG_H */ diff --git a/drivers/md/dm-vdo/index-layout.c b/drivers/md/dm-vdo/index-layout.c index bc67720659e7..2da507b26fd5 100644 --- a/drivers/md/dm-vdo/index-layout.c +++ b/drivers/md/dm-vdo/index-layout.c @@ -222,7 +222,7 @@ static inline bool is_converted_super_block(struct super_block_data *super) return super->version == 7; } -static int __must_check compute_sizes(const struct configuration *config, +static int __must_check compute_sizes(const struct uds_configuration *config, struct save_layout_sizes *sls) { int result; @@ -256,7 +256,7 @@ static int __must_check compute_sizes(const struct configuration *config, int uds_compute_index_size(const struct uds_parameters *parameters, u64 *index_size) { int result; - struct configuration *index_config; + struct uds_configuration *index_config; struct save_layout_sizes sizes; if (index_size == NULL) { @@ -752,7 +752,7 @@ static int __must_check write_layout_header(struct index_layout *layout, } static int __must_check write_uds_index_config(struct index_layout *layout, - struct configuration *config, + struct uds_configuration *config, off_t offset) { int result; @@ -801,7 +801,7 @@ static int __must_check save_layout(struct index_layout *layout, off_t offset) return result; } -static int create_index_layout(struct index_layout *layout, struct configuration *config) +static int create_index_layout(struct index_layout *layout, struct uds_configuration *config) { int result; struct save_layout_sizes sizes; @@ -1620,7 +1620,7 @@ static int __must_check load_sub_index_regions(struct index_layout *layout) } static int __must_check verify_uds_index_config(struct index_layout *layout, - struct configuration *config) + struct uds_configuration *config) { int result; struct buffered_reader *reader = NULL; @@ -1641,7 +1641,7 @@ static int __must_check verify_uds_index_config(struct index_layout *layout, return UDS_SUCCESS; } -static int load_index_layout(struct index_layout *layout, struct configuration *config) +static int load_index_layout(struct index_layout *layout, struct uds_configuration *config) { int result; struct buffered_reader *reader; @@ -1665,7 +1665,7 @@ static int load_index_layout(struct index_layout *layout, struct configuration * } static int create_layout_factory(struct index_layout *layout, - const struct configuration *config) + const struct uds_configuration *config) { int result; size_t writable_size; @@ -1689,7 +1689,7 @@ static int create_layout_factory(struct index_layout *layout, return UDS_SUCCESS; } -int uds_make_index_layout(struct configuration *config, bool new_layout, +int uds_make_index_layout(struct uds_configuration *config, bool new_layout, struct index_layout **layout_ptr) { int result; diff --git a/drivers/md/dm-vdo/index-layout.h b/drivers/md/dm-vdo/index-layout.h index ba5d8b004ebe..84a9eb43a49d 100644 --- a/drivers/md/dm-vdo/index-layout.h +++ b/drivers/md/dm-vdo/index-layout.h @@ -18,7 +18,7 @@ struct index_layout; -int __must_check uds_make_index_layout(struct configuration *config, bool new_layout, +int __must_check uds_make_index_layout(struct uds_configuration *config, bool new_layout, struct index_layout **layout_ptr); void uds_free_index_layout(struct index_layout *layout); diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c index c261453b9871..9b5d2cacfffd 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/index-session.c @@ -314,7 +314,7 @@ static int initialize_index_session(struct uds_index_session *index_session, enum uds_open_index_type open_type) { int result; - struct configuration *config; + struct uds_configuration *config; result = uds_make_configuration(&index_session->parameters, &config); if (result != UDS_SUCCESS) { diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index cf7cdc909a31..e343a973881d 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -1162,7 +1162,7 @@ static int make_index_zone(struct uds_index *index, unsigned int zone_number) return UDS_SUCCESS; } -int uds_make_index(struct configuration *config, enum uds_open_index_type open_type, +int uds_make_index(struct uds_configuration *config, enum uds_open_index_type open_type, struct index_load_context *load_context, index_callback_fn callback, struct uds_index **new_index) { diff --git a/drivers/md/dm-vdo/index.h b/drivers/md/dm-vdo/index.h index ace1835aecd5..edabb239548e 100644 --- a/drivers/md/dm-vdo/index.h +++ b/drivers/md/dm-vdo/index.h @@ -62,7 +62,7 @@ enum request_stage { STAGE_MESSAGE, }; -int __must_check uds_make_index(struct configuration *config, +int __must_check uds_make_index(struct uds_configuration *config, enum uds_open_index_type open_type, struct index_load_context *load_context, index_callback_fn callback, struct uds_index **new_index); diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c index 0f4b9510e7ea..8731ea1662b1 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/volume-index.c @@ -80,11 +80,11 @@ struct sub_index_parameters { struct split_config { /* The hook subindex configuration */ - struct configuration hook_config; + struct uds_configuration hook_config; struct index_geometry hook_geometry; /* The non-hook subindex configuration */ - struct configuration non_hook_config; + struct uds_configuration non_hook_config; struct index_geometry non_hook_geometry; }; @@ -192,7 +192,7 @@ unsigned int uds_get_volume_index_zone(const struct volume_index *volume_index, return get_volume_sub_index_zone(get_volume_sub_index(volume_index, name), name); } -static int compute_volume_sub_index_parameters(const struct configuration *config, +static int compute_volume_sub_index_parameters(const struct uds_configuration *config, struct sub_index_parameters *params) { enum { DELTA_LIST_SIZE = 256 }; @@ -300,7 +300,7 @@ void uds_free_volume_index(struct volume_index *volume_index) } -static int compute_volume_sub_index_save_bytes(const struct configuration *config, +static int compute_volume_sub_index_save_bytes(const struct uds_configuration *config, size_t *bytes) { struct sub_index_parameters params = { .address_bits = 0 }; @@ -317,7 +317,7 @@ static int compute_volume_sub_index_save_bytes(const struct configuration *confi } /* This function is only useful if the configuration includes sparse chapters. */ -static void split_configuration(const struct configuration *config, +static void split_configuration(const struct uds_configuration *config, struct split_config *split) { u64 sample_rate, sample_records; @@ -346,7 +346,7 @@ static void split_configuration(const struct configuration *config, split->non_hook_geometry.chapters_per_volume = dense_chapters; } -static int compute_volume_index_save_bytes(const struct configuration *config, +static int compute_volume_index_save_bytes(const struct uds_configuration *config, size_t *bytes) { size_t hook_bytes, non_hook_bytes; @@ -370,7 +370,7 @@ static int compute_volume_index_save_bytes(const struct configuration *config, return UDS_SUCCESS; } -int uds_compute_volume_index_save_blocks(const struct configuration *config, +int uds_compute_volume_index_save_blocks(const struct uds_configuration *config, size_t block_size, u64 *block_count) { size_t bytes; @@ -1172,7 +1172,7 @@ void uds_get_volume_index_stats(const struct volume_index *volume_index, stats->early_flushes += sparse_stats.early_flushes; } -static int initialize_volume_sub_index(const struct configuration *config, +static int initialize_volume_sub_index(const struct uds_configuration *config, u64 volume_nonce, u8 tag, struct volume_sub_index *sub_index) { @@ -1222,7 +1222,7 @@ static int initialize_volume_sub_index(const struct configuration *config, "volume index zones", &sub_index->zones); } -int uds_make_volume_index(const struct configuration *config, u64 volume_nonce, +int uds_make_volume_index(const struct uds_configuration *config, u64 volume_nonce, struct volume_index **volume_index_ptr) { struct split_config split; diff --git a/drivers/md/dm-vdo/volume-index.h b/drivers/md/dm-vdo/volume-index.h index 7fbe11fe3284..0f08acf29435 100644 --- a/drivers/md/dm-vdo/volume-index.h +++ b/drivers/md/dm-vdo/volume-index.h @@ -136,13 +136,13 @@ struct volume_index_record { struct delta_index_entry delta_entry; }; -int __must_check uds_make_volume_index(const struct configuration *config, +int __must_check uds_make_volume_index(const struct uds_configuration *config, u64 volume_nonce, struct volume_index **volume_index); void uds_free_volume_index(struct volume_index *volume_index); -int __must_check uds_compute_volume_index_save_blocks(const struct configuration *config, +int __must_check uds_compute_volume_index_save_blocks(const struct uds_configuration *config, size_t block_size, u64 *block_count); diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 0cfeae0c5d5d..8bd64057c2ca 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -1538,7 +1538,7 @@ static int __must_check initialize_page_cache(struct page_cache *cache, return UDS_SUCCESS; } -int uds_make_volume(const struct configuration *config, struct index_layout *layout, +int uds_make_volume(const struct uds_configuration *config, struct index_layout *layout, struct volume **new_volume) { unsigned int i; diff --git a/drivers/md/dm-vdo/volume.h b/drivers/md/dm-vdo/volume.h index e938dd31934f..066680282340 100644 --- a/drivers/md/dm-vdo/volume.h +++ b/drivers/md/dm-vdo/volume.h @@ -121,7 +121,7 @@ struct volume { unsigned int reserved_buffers; }; -int __must_check uds_make_volume(const struct configuration *config, +int __must_check uds_make_volume(const struct uds_configuration *config, struct index_layout *layout, struct volume **new_volume); From 3fa8e6ec07f7d326cbd9af2daa29b5fe3fb9eb12 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 5 Feb 2024 21:04:16 -0500 Subject: [PATCH 0618/1038] dm vdo: fix sparse warnings about missing statics Addresses various sparse warnings like: warning: symbol 'SYMBOL' was not declared. Should it be static? Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/Makefile | 1 - drivers/md/dm-vdo/block-map.c | 2 +- drivers/md/dm-vdo/constants.c | 15 --------------- drivers/md/dm-vdo/constants.h | 6 +++--- drivers/md/dm-vdo/encodings.c | 2 +- 5 files changed, 5 insertions(+), 21 deletions(-) delete mode 100644 drivers/md/dm-vdo/constants.c diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 8f8d161a6dbe..8c06c3b969e3 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -9,7 +9,6 @@ dm-vdo-objs := \ chapter-index.o \ completion.o \ config.o \ - constants.o \ data-vio.o \ dedupe.o \ delta-index.o \ diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 54ad6939cab7..1278a5791160 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -67,7 +67,7 @@ struct block_map_tree_segment { struct block_map_tree { struct block_map_tree_segment *segments; -} block_map_tree; +}; struct forest { struct block_map *map; diff --git a/drivers/md/dm-vdo/constants.c b/drivers/md/dm-vdo/constants.c deleted file mode 100644 index 14ac047101f8..000000000000 --- a/drivers/md/dm-vdo/constants.c +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -#include "types.h" - -/* The maximum logical space is 4 petabytes, which is 1 terablock. */ -const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; - -/* The maximum physical space is 256 terabytes, which is 64 gigablocks. */ -const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; - -/* unit test minimum */ -const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2; diff --git a/drivers/md/dm-vdo/constants.h b/drivers/md/dm-vdo/constants.h index 9ae3917d00de..3a997dd7d0b1 100644 --- a/drivers/md/dm-vdo/constants.h +++ b/drivers/md/dm-vdo/constants.h @@ -91,12 +91,12 @@ enum { }; /** The maximum logical space is 4 petabytes, which is 1 terablock. */ -extern const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS; +static const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; /** The maximum physical space is 256 terabytes, which is 64 gigablocks. */ -extern const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS; +static const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; /** unit test minimum */ -extern const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS; +static const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2; #endif /* VDO_CONSTANTS_H */ diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index 2d9d8645f66b..6b3e9a5b3a91 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -85,7 +85,7 @@ const struct header VDO_SLAB_DEPOT_HEADER_2_0 = { .size = sizeof(struct slab_depot_state_2_0), }; -const struct header VDO_LAYOUT_HEADER_3_0 = { +static const struct header VDO_LAYOUT_HEADER_3_0 = { .id = VDO_LAYOUT, .version = { .major_version = 3, From ff9199464886500bd0c3df4a5ebae4760607aeda Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 5 Feb 2024 21:04:17 -0500 Subject: [PATCH 0619/1038] dm vdo: fix sparse 'warning: Using plain integer as NULL pointer' Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/index.c | 8 ++++---- drivers/md/dm-vdo/vdo.c | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index e343a973881d..1596f6ba43a5 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -732,18 +732,18 @@ static void close_chapters(void *arg) static void stop_chapter_writer(struct chapter_writer *writer) { - struct thread *writer_thread = 0; + struct thread *writer_thread = NULL; uds_lock_mutex(&writer->mutex); - if (writer->thread != 0) { + if (writer->thread != NULL) { writer_thread = writer->thread; - writer->thread = 0; + writer->thread = NULL; writer->stop = true; uds_broadcast_cond(&writer->cond); } uds_unlock_mutex(&writer->mutex); - if (writer_thread != 0) + if (writer_thread != NULL) uds_join_threads(writer_thread); } diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index 1ead769f0150..e0eddd4007b8 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -889,7 +889,8 @@ int vdo_synchronous_flush(struct vdo *vdo) int result; struct bio bio; - bio_init(&bio, vdo_get_backing_device(vdo), 0, 0, REQ_OP_WRITE | REQ_PREFLUSH); + bio_init(&bio, vdo_get_backing_device(vdo), NULL, 0, + REQ_OP_WRITE | REQ_PREFLUSH); submit_bio_wait(&bio); result = blk_status_to_errno(bio.bi_status); From a6c05c981ecc841a3e95af666c2a1a6a265e7adc Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 5 Feb 2024 21:04:18 -0500 Subject: [PATCH 0620/1038] dm vdo: fix various blk_opf_t sparse warnings Use proper blk_opf_t type rather than unsigned int. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 2 +- drivers/md/dm-vdo/data-vio.c | 4 ++-- drivers/md/dm-vdo/io-submitter.c | 2 +- drivers/md/dm-vdo/io-submitter.h | 4 ++-- drivers/md/dm-vdo/vio.c | 4 ++-- drivers/md/dm-vdo/vio.h | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 1278a5791160..cc87e1ac72b6 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -1620,7 +1620,7 @@ static void write_initialized_page(struct vdo_completion *completion) struct block_map_zone *zone = pooled->context; struct tree_page *tree_page = completion->parent; struct block_map_page *page = (struct block_map_page *) vio->data; - unsigned int operation = REQ_OP_WRITE | REQ_PRIO; + blk_opf_t operation = REQ_OP_WRITE | REQ_PRIO; /* * Now that we know the page has been written at least once, mark the copy we are writing diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 328b645dee16..e0f1574ae112 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -69,7 +69,7 @@ * ASAP to service heavy load, which is the only place where REQ_BACKGROUND might aid in load * prioritization. */ -static unsigned int PASSTHROUGH_FLAGS = (REQ_PRIO | REQ_META | REQ_SYNC | REQ_RAHEAD); +static blk_opf_t PASSTHROUGH_FLAGS = (REQ_PRIO | REQ_META | REQ_SYNC | REQ_RAHEAD); /** * DOC: @@ -1570,7 +1570,7 @@ static void read_block(struct vdo_completion *completion) result = vio_reset_bio(vio, (char *) data_vio->compression.block, read_endio, REQ_OP_READ, data_vio->mapped.pbn); } else { - int opf = ((data_vio->user_bio->bi_opf & PASSTHROUGH_FLAGS) | REQ_OP_READ); + blk_opf_t opf = ((data_vio->user_bio->bi_opf & PASSTHROUGH_FLAGS) | REQ_OP_READ); if (data_vio->is_partial) { result = vio_reset_bio(vio, vio->data, read_endio, opf, diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 74f33a3ddce5..6c050f2b3b44 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -338,7 +338,7 @@ void vdo_submit_data_vio(struct data_vio *data_vio) */ void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, bio_end_io_t callback, vdo_action_fn error_handler, - unsigned int operation, char *data) + blk_opf_t operation, char *data) { int result; struct vdo_completion *completion = &vio->completion; diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h index ddd5f4b391b6..80748699496f 100644 --- a/drivers/md/dm-vdo/io-submitter.h +++ b/drivers/md/dm-vdo/io-submitter.h @@ -26,11 +26,11 @@ void vdo_submit_data_vio(struct data_vio *data_vio); void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, bio_end_io_t callback, vdo_action_fn error_handler, - unsigned int operation, char *data); + blk_opf_t operation, char *data); static inline void vdo_submit_metadata_vio(struct vio *vio, physical_block_number_t physical, bio_end_io_t callback, vdo_action_fn error_handler, - unsigned int operation) + blk_opf_t operation) { __submit_metadata_vio(vio, physical, callback, error_handler, operation, vio->data); diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index 6acaba149c75..f9c54ec56176 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -172,7 +172,7 @@ void free_vio(struct vio *vio) /* Set bio properties for a VDO read or write. */ void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callback, - unsigned int bi_opf, physical_block_number_t pbn) + blk_opf_t bi_opf, physical_block_number_t pbn) { struct vdo *vdo = vio->completion.vdo; struct device_config *config = vdo->device_config; @@ -193,7 +193,7 @@ void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callb * vio associated with the bio. */ int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, - unsigned int bi_opf, physical_block_number_t pbn) + blk_opf_t bi_opf, physical_block_number_t pbn) { int bvec_count, offset, len, i; struct bio *bio = vio->bio; diff --git a/drivers/md/dm-vdo/vio.h b/drivers/md/dm-vdo/vio.h index 71585424f85b..fbfee5e3415d 100644 --- a/drivers/md/dm-vdo/vio.h +++ b/drivers/md/dm-vdo/vio.h @@ -119,10 +119,10 @@ static inline void initialize_vio(struct vio *vio, struct bio *bio, } void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callback, - unsigned int bi_opf, physical_block_number_t pbn); + blk_opf_t bi_opf, physical_block_number_t pbn); int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, - unsigned int bi_opf, physical_block_number_t pbn); + blk_opf_t bi_opf, physical_block_number_t pbn); void update_vio_error_stats(struct vio *vio, const char *format, ...) __printf(2, 3); From 872564c501b72ae0c84af51084753e8652e4a84b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 5 Feb 2024 21:04:19 -0500 Subject: [PATCH 0621/1038] dm vdo data-vio: silence sparse warnings about locking context imbalances Factor wait_permit() out from acquire_permit() so that the latter always holds the spinlock and the former always releases it. Otherwise sparse complains about locking context imbalances due to conditional spin_unlock in acquire_permit: warning: context imbalance in 'acquire_permit' - different lock contexts for basic block warning: context imbalance in 'vdo_launch_bio' - unexpected unlock Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/data-vio.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index e0f1574ae112..d77adeb5006e 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -929,27 +929,30 @@ void free_data_vio_pool(struct data_vio_pool *pool) uds_free(pool); } -static bool acquire_permit(struct limiter *limiter, struct bio *bio) +static bool acquire_permit(struct limiter *limiter) { - if (limiter->busy >= limiter->limit) { - DEFINE_WAIT(wait); - - bio_list_add(&limiter->new_waiters, bio); - prepare_to_wait_exclusive(&limiter->blocked_threads, &wait, - TASK_UNINTERRUPTIBLE); - spin_unlock(&limiter->pool->lock); - io_schedule(); - finish_wait(&limiter->blocked_threads, &wait); + if (limiter->busy >= limiter->limit) return false; - } WRITE_ONCE(limiter->busy, limiter->busy + 1); if (limiter->max_busy < limiter->busy) WRITE_ONCE(limiter->max_busy, limiter->busy); - return true; } +static void wait_permit(struct limiter *limiter, struct bio *bio) + __releases(&limiter->pool->lock) +{ + DEFINE_WAIT(wait); + + bio_list_add(&limiter->new_waiters, bio); + prepare_to_wait_exclusive(&limiter->blocked_threads, &wait, + TASK_UNINTERRUPTIBLE); + spin_unlock(&limiter->pool->lock); + io_schedule(); + finish_wait(&limiter->blocked_threads, &wait); +} + /** * vdo_launch_bio() - Acquire a data_vio from the pool, assign the bio to it, and launch it. * @@ -965,11 +968,15 @@ void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio) bio->bi_private = (void *) jiffies; spin_lock(&pool->lock); if ((bio_op(bio) == REQ_OP_DISCARD) && - !acquire_permit(&pool->discard_limiter, bio)) + !acquire_permit(&pool->discard_limiter)) { + wait_permit(&pool->discard_limiter, bio); return; + } - if (!acquire_permit(&pool->limiter, bio)) + if (!acquire_permit(&pool->limiter)) { + wait_permit(&pool->limiter, bio); return; + } data_vio = get_available_data_vio(pool); spin_unlock(&pool->lock); From f46b1ab7e7e4dacd748ccb6e12a77a3fea229764 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 5 Feb 2024 21:04:20 -0500 Subject: [PATCH 0622/1038] dm vdo dedupe: silence sparse warnings about locking context imbalances Annotate both open_index() and close_index() with __must_hold(&zones->lock) to silence these sparse warnings: warning: context imbalance in 'close_index' - unexpected unlock warning: context imbalance in 'open_index' - unexpected unlock Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dedupe.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 8533f1669a01..4b00135511dd 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -2092,6 +2092,7 @@ static void finish_uds_queue(void *ptr __always_unused) } static void close_index(struct hash_zones *zones) + __must_hold(&zones->lock) { int result; @@ -2113,6 +2114,7 @@ static void close_index(struct hash_zones *zones) } static void open_index(struct hash_zones *zones) + __must_hold(&zones->lock) { /* ASSERTION: We enter in IS_CLOSED state. */ int result; From b863d7f7503c42bc3c7033bcf8fff70d0bf825a7 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 5 Feb 2024 21:04:21 -0500 Subject: [PATCH 0623/1038] dm vdo recovery-journal: fix sparse 'mixed bitwiseness' warning Only one user of WRITE_FLAGS so no need to factor it out in an enum (which causes sparse's 'mixed bitwiseness' warning). Just use the flags in the only consumer. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/recovery-journal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index a6981e5dd017..1e15bfe42cfc 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -34,7 +34,6 @@ enum { */ RECOVERY_JOURNAL_RESERVED_BLOCKS = (MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2, - WRITE_FLAGS = REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH | REQ_SYNC | REQ_FUA, }; /** @@ -1398,7 +1397,8 @@ static void write_block(struct vdo_waiter *waiter, void *context __always_unused * block itself is stable before allowing overwrites of the lbn's previous data. */ vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number, - complete_write_endio, handle_write_error, WRITE_FLAGS); + complete_write_endio, handle_write_error, + REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH | REQ_SYNC | REQ_FUA); } From a03652238d252e6c8a138f04eec71a90e0a99379 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 8 Feb 2024 11:39:41 -0500 Subject: [PATCH 0624/1038] dm vdo: add vdo documentation to device-mapper index Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- Documentation/admin-guide/device-mapper/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/admin-guide/device-mapper/index.rst b/Documentation/admin-guide/device-mapper/index.rst index cde52cc09645..cc5aec861576 100644 --- a/Documentation/admin-guide/device-mapper/index.rst +++ b/Documentation/admin-guide/device-mapper/index.rst @@ -34,6 +34,8 @@ Device Mapper switch thin-provisioning unstriped + vdo-design + vdo verity writecache zero From 512039b41b08177dce08f5cf324f2f57f9629639 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Fri, 2 Feb 2024 23:32:03 -0500 Subject: [PATCH 0625/1038] dm vdo: add vio life cycle details to design doc Add more documentation details for most aspects of the data_vio read and write processes. Also correct a few minor errors and rewrite some text for clarity. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- .../admin-guide/device-mapper/vdo-design.rst | 626 ++++++++++++------ 1 file changed, 422 insertions(+), 204 deletions(-) diff --git a/Documentation/admin-guide/device-mapper/vdo-design.rst b/Documentation/admin-guide/device-mapper/vdo-design.rst index c82d51071c7d..3cd59decbec0 100644 --- a/Documentation/admin-guide/device-mapper/vdo-design.rst +++ b/Documentation/admin-guide/device-mapper/vdo-design.rst @@ -38,19 +38,27 @@ structures involved in a single write operation to a vdo target is larger than most other targets. Furthermore, because vdo must operate on small block sizes in order to achieve good deduplication rates, acceptable performance can only be achieved through parallelism. Therefore, vdo's -design attempts to be lock-free. Most of a vdo's main data structures are -designed to be easily divided into "zones" such that any given bio must -only access a single zone of any zoned structure. Safety with minimal -locking is achieved by ensuring that during normal operation, each zone is -assigned to a specific thread, and only that thread will access the portion -of that data structure in that zone. Associated with each thread is a work -queue. Each bio is associated with a request object which can be added to a -work queue when the next phase of its operation requires access to the -structures in the zone associated with that queue. Although each structure -may be divided into zones, this division is not reflected in the on-disk -representation of each data structure. Therefore, the number of zones for -each structure, and hence the number of threads, is configured each time a -vdo target is started. +design attempts to be lock-free. + +Most of a vdo's main data structures are designed to be easily divided into +"zones" such that any given bio must only access a single zone of any zoned +structure. Safety with minimal locking is achieved by ensuring that during +normal operation, each zone is assigned to a specific thread, and only that +thread will access the portion of the data structure in that zone. +Associated with each thread is a work queue. Each bio is associated with a +request object (the "data_vio") which will be added to a work queue when +the next phase of its operation requires access to the structures in the +zone associated with that queue. + +Another way of thinking about this arrangement is that the work queue for +each zone has an implicit lock on the structures it manages for all its +operations, because vdo guarantees that no other thread will alter those +structures. + +Although each structure is divided into zones, this division is not +reflected in the on-disk representation of each data structure. Therefore, +the number of zones for each structure, and hence the number of threads, +can be reconfigured each time a vdo target is started. The Deduplication Index ----------------------- @@ -75,17 +83,17 @@ is sufficient to find and eliminate most of the redundancy. Each block of data is hashed to produce a 16-byte block name. An index record consists of this block name paired with the presumed location of that data on the underlying storage. However, it is not possible to -guarantee that the index is accurate. Most often, this occurs because it is -too costly to update the index when a block is over-written or discarded. -Doing so would require either storing the block name along with the blocks, -which is difficult to do efficiently in block-based storage, or reading and -rehashing each block before overwriting it. Inaccuracy can also result from -a hash collision where two different blocks have the same name. In -practice, this is extremely unlikely, but because vdo does not use a -cryptographic hash, a malicious workload can be constructed. Because of -these inaccuracies, vdo treats the locations in the index as hints, and -reads each indicated block to verify that it is indeed a duplicate before -sharing the existing block with a new one. +guarantee that the index is accurate. In the most common case, this occurs +because it is too costly to update the index when a block is over-written +or discarded. Doing so would require either storing the block name along +with the blocks, which is difficult to do efficiently in block-based +storage, or reading and rehashing each block before overwriting it. +Inaccuracy can also result from a hash collision where two different blocks +have the same name. In practice, this is extremely unlikely, but because +vdo does not use a cryptographic hash, a malicious workload could be +constructed. Because of these inaccuracies, vdo treats the locations in the +index as hints, and reads each indicated block to verify that it is indeed +a duplicate before sharing the existing block with a new one. Records are collected into groups called chapters. New records are added to the newest chapter, called the open chapter. This chapter is stored in a @@ -95,7 +103,7 @@ When the open chapter fills up, it is closed and a new open chapter is created to collect new records. Closing a chapter converts it to a different format which is optimized for -writing. The records are written to a series of record pages based on the +reading. The records are written to a series of record pages based on the order in which they were received. This means that records with temporal locality should be on a small number of pages, reducing the I/O required to retrieve them. The chapter also compiles an index that indicates which @@ -104,85 +112,110 @@ name can determine exactly which record page may contain that record, without having to load the entire chapter from storage. This index uses only a subset of the block name as its key, so it cannot guarantee that an index entry refers to the desired block name. It can only guarantee that if -there is a record for this name, it will be on the indicated page. The -contents of a closed chapter are never altered in any way; these chapters -are read-only structures. +there is a record for this name, it will be on the indicated page. Closed +chapters are read-only structures and their contents are never altered in +any way. Once enough records have been written to fill up all the available index -space, the oldest chapter gets removed to make space for new chapters. Any +space, the oldest chapter is removed to make space for new chapters. Any time a request finds a matching record in the index, that record is copied -to the open chapter. This ensures that useful block names remain available -in the index, while unreferenced block names are forgotten. +into the open chapter. This ensures that useful block names remain available +in the index, while unreferenced block names are forgotten over time. In order to find records in older chapters, the index also maintains a higher level structure called the volume index, which contains entries -mapping a block name to the chapter containing its newest record. This +mapping each block name to the chapter containing its newest record. This mapping is updated as records for the block name are copied or updated, -ensuring that only the newer record for a given block name is findable. -Older records for a block name can no longer be found even though they have -not been deleted. Like the chapter index, the volume index uses only a -subset of the block name as its key and can not definitively say that a -record exists for a name. It can only say which chapter would contain the -record if a record exists. The volume index is stored entirely in memory -and is saved to storage only when the vdo target is shut down. +ensuring that only the newest record for a given block name can be found. +An older record for a block name will no longer be found even though it has +not been deleted from its chapter. Like the chapter index, the volume index +uses only a subset of the block name as its key and can not definitively +say that a record exists for a name. It can only say which chapter would +contain the record if a record exists. The volume index is stored entirely +in memory and is saved to storage only when the vdo target is shut down. -From the viewpoint of a request for a particular block name, first it will -look up the name in the volume index which will indicate either that the -record is new, or which chapter to search. If the latter, the request looks -up its name in the chapter index to determine if the record is new, or -which record page to search. Finally, if not new, the request will look for -its record on the indicated record page. This process may require up to two -page reads per request (one for the chapter index page and one for the -request page). However, recently accessed pages are cached so that these -page reads can be amortized across many block name requests. +From the viewpoint of a request for a particular block name, it will first +look up the name in the volume index. This search will either indicate that +the name is new, or which chapter to search. If it returns a chapter, the +request looks up its name in the chapter index. This will indicate either +that the name is new, or which record page to search. Finally, if it is not +new, the request will look for its name in the indicated record page. +This process may require up to two page reads per request (one for the +chapter index page and one for the request page). However, recently +accessed pages are cached so that these page reads can be amortized across +many block name requests. The volume index and the chapter indexes are implemented using a memory-efficient structure called a delta index. Instead of storing the -entire key (the block name) for each entry, the entries are sorted by name +entire block name (the key) for each entry, the entries are sorted by name and only the difference between adjacent keys (the delta) is stored. -Because we expect the hashes to be evenly distributed, the size of the +Because we expect the hashes to be randomly distributed, the size of the deltas follows an exponential distribution. Because of this distribution, -the deltas are expressed in a Huffman code to take up even less space. The -entire sorted list of keys is called a delta list. This structure allows -the index to use many fewer bytes per entry than a traditional hash table, -but it is slightly more expensive to look up entries, because a request -must read every entry in a delta list to add up the deltas in order to find -the record it needs. The delta index reduces this lookup cost by splitting -its key space into many sub-lists, each starting at a fixed key value, so -that each individual list is short. +the deltas are expressed using a Huffman code to take up even less space. +The entire sorted list of keys is called a delta list. This structure +allows the index to use many fewer bytes per entry than a traditional hash +table, but it is slightly more expensive to look up entries, because a +request must read every entry in a delta list to add up the deltas in order +to find the record it needs. The delta index reduces this lookup cost by +splitting its key space into many sub-lists, each starting at a fixed key +value, so that each individual list is short. The default index size can hold 64 million records, corresponding to about -256GB. This means that the index can identify duplicate data if the +256GB of data. This means that the index can identify duplicate data if the original data was written within the last 256GB of writes. This range is called the deduplication window. If new writes duplicate data that is older than that, the index will not be able to find it because the records of the -older data have been removed. So when writing a 200 GB file to a vdo -target, and then immediately writing it again, the two copies will -deduplicate perfectly. Doing the same with a 500 GB file will result in no -deduplication, because the beginning of the file will no longer be in the -index by the time the second write begins (assuming there is no duplication -within the file itself). +older data have been removed. This means that if an application writes a +200 GB file to a vdo target and then immediately writes it again, the two +copies will deduplicate perfectly. Doing the same with a 500 GB file will +result in no deduplication, because the beginning of the file will no +longer be in the index by the time the second write begins (assuming there +is no duplication within the file itself). -If you anticipate a data workload that will see useful deduplication beyond -the 256GB threshold, vdo can be configured to use a larger index with a -correspondingly larger deduplication window. (This configuration can only -be set when the target is created, not altered later. It is important to -consider the expected workload for a vdo target before configuring it.) -There are two ways to do this. +If an application anticipates a data workload that will see useful +deduplication beyond the 256GB threshold, vdo can be configured to use a +larger index with a correspondingly larger deduplication window. (This +configuration can only be set when the target is created, not altered +later. It is important to consider the expected workload for a vdo target +before configuring it.) There are two ways to do this. One way is to increase the memory size of the index, which also increases the amount of backing storage required. Doubling the size of the index will double the length of the deduplication window at the expense of doubling the storage size and the memory requirements. -The other way is to enable sparse indexing. Sparse indexing increases the -deduplication window by a factor of 10, at the expense of also increasing -the storage size by a factor of 10. However with sparse indexing, the -memory requirements do not increase; the trade-off is slightly more -computation per request, and a slight decrease in the amount of -deduplication detected. (For workloads with significant amounts of +The other option is to enable sparse indexing. Sparse indexing increases +the deduplication window by a factor of 10, at the expense of also +increasing the storage size by a factor of 10. However with sparse +indexing, the memory requirements do not increase. The trade-off is +slightly more computation per request and a slight decrease in the amount +of deduplication detected. For most workloads with significant amounts of duplicate data, sparse indexing will detect 97-99% of the deduplication -that a standard, or "dense", index will detect.) +that a standard index will detect. + +The vio and data_vio Structures +------------------------------- + +A vio (short for Vdo I/O) is conceptually similar to a bio, with additional +fields and data to track vdo-specific information. A struct vio maintains a +pointer to a bio but also tracks other fields specific to the operation of +vdo. The vio is kept separate from its related bio because there are many +circumstances where vdo completes the bio but must continue to do work +related to deduplication or compression. + +Metadata reads and writes, and other writes that originate within vdo, use +a struct vio directly. Application reads and writes use a larger structure +called a data_vio to track information about their progress. A struct +data_vio contain a struct vio and also includes several other fields +related to deduplication and other vdo features. The data_vio is the +primary unit of application work in vdo. Each data_vio proceeds through a +set of steps to handle the application data, after which it is reset and +returned to a pool of data_vios for reuse. + +There is a fixed pool of 2048 data_vios. This number was chosen to bound +the amount of work that is required to recover from a crash. In addition, +benchmarks have indicated that increasing the size of the pool does not +significantly improve performance. The Data Store -------------- @@ -199,13 +232,18 @@ three sections. Most of a slab consists of a linear sequence of 4K blocks. These blocks are used either to store data, or to hold portions of the block map (see below). In addition to the data blocks, each slab has a set of reference counters, using 1 byte for each data block. Finally each slab -has a journal. Reference updates are written to the slab journal, which is -written out one block at a time as each block fills. A copy of the -reference counters are kept in memory, and are written out a block at a -time, in oldest-dirtied-order whenever there is a need to reclaim slab -journal space. The journal is used both to ensure that the main recovery -journal (see below) can regularly free up space, and also to amortize the -cost of updating individual reference blocks. +has a journal. + +Reference updates are written to the slab journal. Slab journal blocks are +written out either when they are full, or when the recovery journal +requests they do so in order to allow the main recovery journal (see below) +to free up space. The slab journal is used both to ensure that the main +recovery journal can regularly free up space, and also to amortize the cost +of updating individual reference blocks. The reference counters are kept in +memory and are written out, a block at a time in oldest-dirtied-order, only +when there is a need to reclaim slab journal space. The write operations +are performed in the background as needed so they do not add latency to +particular I/O operations. Each slab is independent of every other. They are assigned to "physical zones" in round-robin fashion. If there are P physical zones, then slab n @@ -214,14 +252,14 @@ is assigned to zone n mod P. The slab depot maintains an additional small data structure, the "slab summary," which is used to reduce the amount of work needed to come back online after a crash. The slab summary maintains an entry for each slab -indicating whether or not the slab has ever been used, whether it is clean -(i.e. all of its reference count updates have been persisted to storage), -and approximately how full it is. During recovery, each physical zone will -attempt to recover at least one slab, stopping whenever it has recovered a -slab which has some free blocks. Once each zone has some space (or has -determined that none is available), the target can resume normal operation -in a degraded mode. Read and write requests can be serviced, perhaps with -degraded performance, while the remainder of the dirty slabs are recovered. +indicating whether or not the slab has ever been used, whether all of its +reference count updates have been persisted to storage, and approximately +how full it is. During recovery, each physical zone will attempt to recover +at least one slab, stopping whenever it has recovered a slab which has some +free blocks. Once each zone has some space, or has determined that none is +available, the target can resume normal operation in a degraded mode. Read +and write requests can be serviced, perhaps with degraded performance, +while the remainder of the dirty slabs are recovered. *The Block Map* @@ -233,32 +271,30 @@ of the mapping. Of the 16 possible states, one represents a logical address which is unmapped (i.e. it has never been written, or has been discarded), one represents an uncompressed block, and the other 14 states are used to indicate that the mapped data is compressed, and which of the compression -slots in the compressed block this logical address maps to (see below). +slots in the compressed block contains the data for this logical address. In practice, the array of mapping entries is divided into "block map pages," each of which fits in a single 4K block. Each block map page -consists of a header, and 812 mapping entries (812 being the number that -fit). Each mapping page is actually a leaf of a radix tree which consists -of block map pages at each level. There are 60 radix trees which are -assigned to "logical zones" in round robin fashion (if there are L logical -zones, tree n will belong to zone n mod L). At each level, the trees are -interleaved, so logical addresses 0-811 belong to tree 0, logical addresses -812-1623 belong to tree 1, and so on. The interleaving is maintained all -the way up the forest. 60 was chosen as the number of trees because it is -highly composite and hence results in an evenly distributed number of trees -per zone for a large number of possible logical zone counts. The storage -for the 60 tree roots is allocated at format time. All other block map -pages are allocated out of the slabs as needed. This flexible allocation -avoids the need to pre-allocate space for the entire set of logical -mappings and also makes growing the logical size of a vdo easy to -implement. +consists of a header and 812 mapping entries. Each mapping page is actually +a leaf of a radix tree which consists of block map pages at each level. +There are 60 radix trees which are assigned to "logical zones" in round +robin fashion. (If there are L logical zones, tree n will belong to zone n +mod L.) At each level, the trees are interleaved, so logical addresses +0-811 belong to tree 0, logical addresses 812-1623 belong to tree 1, and so +on. The interleaving is maintained all the way up to the 60 root nodes. +Choosing 60 trees results in an evenly distributed number of trees per zone +for a large number of possible logical zone counts. The storage for the 60 +tree roots is allocated at format time. All other block map pages are +allocated out of the slabs as needed. This flexible allocation avoids the +need to pre-allocate space for the entire set of logical mappings and also +makes growing the logical size of a vdo relatively easy. In operation, the block map maintains two caches. It is prohibitive to keep the entire leaf level of the trees in memory, so each logical zone maintains its own cache of leaf pages. The size of this cache is configurable at target start time. The second cache is allocated at start time, and is large enough to hold all the non-leaf pages of the entire -block map. This cache is populated as needed. +block map. This cache is populated as pages are needed. *The Recovery Journal* @@ -267,127 +303,307 @@ slab depot. Each write request causes an entry to be made in the journal. Entries are either "data remappings" or "block map remappings." For a data remapping, the journal records the logical address affected and its old and new physical mappings. For a block map remapping, the journal records the -block map page number and the physical block allocated for it (block map -pages are never reclaimed, so the old mapping is always 0). Each journal -entry and the data write it represents must be stable on disk before the -other metadata structures may be updated to reflect the operation. +block map page number and the physical block allocated for it. Block map +pages are never reclaimed or repurposed, so the old mapping is always 0. + +Each journal entry is an intent record summarizing the metadata updates +that are required for a data_vio. The recovery journal issues a flush +before each journal block write to ensure that the physical data for the +new block mappings in that block are stable on storage, and journal block +writes are all issued with the FUA bit set to ensure the recovery journal +entries themselves are stable. The journal entry and the data write it +represents must be stable on disk before the other metadata structures may +be updated to reflect the operation. These entries allow the vdo device to +reconstruct the logical to physical mappings after an unexpected +interruption such as a loss of power. *Write Path* -A write bio is first assigned a "data_vio," the request object which will -operate on behalf of the bio. (A "vio," from Vdo I/O, is vdo's wrapper for -bios; metadata operations use a vio, whereas submitted bios require the -much larger data_vio.) There is a fixed pool of 2048 data_vios. This number -was chosen both to bound the amount of work that is required to recover -from a crash, and because measurements indicate that increasing it consumes -more resources, but does not improve performance. These measurements have -been, and should continue to be, revisited over time. +All write I/O to vdo is asynchronous. Each bio will be acknowledged as soon +as vdo has done enough work to guarantee that it can complete the write +eventually. Generally, the data for acknowledged but unflushed write I/O +can be treated as though it is cached in memory. If an application +requires data to be stable on storage, it must issue a flush or write the +data with the FUA bit set like any other asynchronous I/O. Shutting down +the vdo target will also flush any remaining I/O. -Once a data_vio is assigned, the following steps are performed: +Application write bios follow the steps outlined below. -1. The bio's data is checked to see if it is all zeros, and copied if not. +1. A data_vio is obtained from the data_vio pool and associated with the + application bio. If there are no data_vios available, the incoming bio + will block until a data_vio is available. This provides back pressure + to the application. The data_vio pool is protected by a spin lock. -2. A lock is obtained on the logical address of the bio. Because - deduplication involves sharing blocks, it is vital to prevent - simultaneous modifications of the same block. + The newly acquired data_vio is reset and the bio's data is copied into + the data_vio if it is a write and the data is not all zeroes. The data + must be copied because the application bio can be acknowledged before + the data_vio processing is complete, which means later processing steps + will no longer have access to the application bio. The application bio + may also be smaller than 4K, in which case the data_vio will have + already read the underlying block and the data is instead copied over + the relevant portion of the larger block. -3. The block map tree is traversed, loading any non-leaf pages which cover - the logical address and are not already in memory. If any of these - pages, or the leaf page which covers the logical address have not been - allocated, and the block is not all zeros, they are allocated at this - time. +2. The data_vio places a claim (the "logical lock") on the logical address + of the bio. It is vital to prevent simultaneous modifications of the + same logical address, because deduplication involves sharing blocks. + This claim is implemented as an entry in a hashtable where the key is + the logical address and the value is a pointer to the data_vio + currently handling that address. + + If a data_vio looks in the hashtable and finds that another data_vio is + already operating on that logical address, it waits until the previous + operation finishes. It also sends a message to inform the current + lock holder that it is waiting. Most notably, a new data_vio waiting + for a logical lock will flush the previous lock holder out of the + compression packer (step 8d) rather than allowing it to continue + waiting to be packed. + + This stage requires the data_vio to get an implicit lock on the + appropriate logical zone to prevent concurrent modifications of the + hashtable. This implicit locking is handled by the zone divisions + described above. + +3. The data_vio traverses the block map tree to ensure that all the + necessary internal tree nodes have been allocated, by trying to find + the leaf page for its logical address. If any interior tree page is + missing, it is allocated at this time out of the same physical storage + pool used to store application data. + + a. If any page-node in the tree has not yet been allocated, it must be + allocated before the write can continue. This step requires the + data_vio to lock the page-node that needs to be allocated. This + lock, like the logical block lock in step 2, is a hashtable entry + that causes other data_vios to wait for the allocation process to + complete. + + The implicit logical zone lock is released while the allocation is + happening, in order to allow other operations in the same logical + zone to proceed. The details of allocation are the same as in + step 4. Once a new node has been allocated, that node is added to + the tree using a similar process to adding a new data block mapping. + The data_vio journals the intent to add the new node to the block + map tree (step 10), updates the reference count of the new block + (step 11), and reacquires the implicit logical zone lock to add the + new mapping to the parent tree node (step 12). Once the tree is + updated, the data_vio proceeds down the tree. Any other data_vios + waiting on this allocation also proceed. + + b. In the steady-state case, the block map tree nodes will already be + allocated, so the data_vio just traverses the tree until it finds + the required leaf node. The location of the mapping (the "block map + slot") is recorded in the data_vio so that later steps do not need + to traverse the tree again. The data_vio then releases the implicit + logical zone lock. 4. If the block is a zero block, skip to step 9. Otherwise, an attempt is - made to allocate a free data block. + made to allocate a free data block. This allocation ensures that the + data_vio can write its data somewhere even if deduplication and + compression are not possible. This stage gets an implicit lock on a + physical zone to search for free space within that zone. -5. If an allocation was obtained, the bio is acknowledged. + The data_vio will search each slab in a zone until it finds a free + block or decides there are none. If the first zone has no free space, + it will proceed to search the next physical zone by taking the implicit + lock for that zone and releasing the previous one until it finds a + free block or runs out of zones to search. The data_vio will acquire a + struct pbn_lock (the "physical block lock") on the free block. The + struct pbn_lock also has several fields to record the various kinds of + claims that data_vios can have on physical blocks. The pbn_lock is + added to a hashtable like the logical block locks in step 2. This + hashtable is also covered by the implicit physical zone lock. The + reference count of the free block is updated to prevent any other + data_vio from considering it free. The reference counters are a + sub-component of the slab and are thus also covered by the implicit + physical zone lock. -6. The bio's data is hashed. +5. If an allocation was obtained, the data_vio has all the resources it + needs to complete the write. The application bio can safely be + acknowledged at this point. The acknowledgment happens on a separate + thread to prevent the application callback from blocking other data_vio + operations. -7. The data_vio obtains or joins a "hash lock," which represents all of - the bios currently writing the same data. + If an allocation could not be obtained, the data_vio continues to + attempt to deduplicate or compress the data, but the bio is not + acknowledged because the vdo device may be out of space. -8. If the hash lock does not already have a data_vio acting as its agent, - the current one assumes that role. As the agent: +6. At this point vdo must determine where to store the application data. + The data_vio's data is hashed and the hash (the "record name") is + recorded in the data_vio. - a) The index is queried. +7. The data_vio reserves or joins a struct hash_lock, which manages all of + the data_vios currently writing the same data. Active hash locks are + tracked in a hashtable similar to the way logical block locks are + tracked in step 2. This hashtable is covered by the implicit lock on + the hash zone. - b) If an entry is found, the indicated block is read and compared - to the data being written. + If there is no existing hash lock for this data_vio's record_name, the + data_vio obtains a hash lock from the pool, adds it to the hashtable, + and sets itself as the new hash lock's "agent." The hash_lock pool is + also covered by the implicit hash zone lock. The hash lock agent will + do all the work to decide where the application data will be + written. If a hash lock for the data_vio's record_name already exists, + and the data_vio's data is the same as the agent's data, the new + data_vio will wait for the agent to complete its work and then share + its result. - c) If the data matches, we have identified duplicate data. As many - of the data_vios as there are references available for that - block (including the agent) are shared. If there are more - data_vios in the hash lock than there are references available, - one of them becomes the new agent and continues as if there was - no duplicate found. + In the rare case that a hash lock exists for the data_vio's hash but + the data does not match the hash lock's agent, the data_vio skips to + step 8h and attempts to write its data directly. This can happen if two + different data blocks produce the same hash, for example. - d) If no duplicate was found, and the agent in the hash lock does - not have an allocation (fron step 3), another data_vio in the - hash lock will become the agent and write the data. If no - data_vio in the hash lock has an allocation, the data_vios will - be marked out of space and go to step 13 for cleanup. +8. The hash lock agent attempts to deduplicate or compress its data with + the following steps. - If there is an allocation, the data being written will be - compressed. If the compressed size is sufficiently small, the - data_vio will go to the packer where it may be placed in a bin - along with other data_vios. + a. The agent initializes and sends its embedded deduplication request + (struct uds_request) to the deduplication index. This does not + require the data_vio to get any locks because the index components + manage their own locking. The data_vio waits until it either gets a + response from the index or times out. - e) Once a bin is full, either because it is out of space, or - because all 14 of its slots are in use, it is written out. + b. If the deduplication index returns advice, the data_vio attempts to + obtain a physical block lock on the indicated physical address, in + order to read the data and verify that it is the same as the + data_vio's data, and that it can accept more references. If the + physical address is already locked by another data_vio, the data at + that address may soon be overwritten so it is not safe to use the + address for deduplication. - f) Each data_vio from the bin just written is the agent of some - hash lock, it will now proceed to treat the just written - compressed block as if it were a duplicate and share it with as - many other data_vios in its hash lock as possible. + c. If the data matches and the physical block can add references, the + agent and any other data_vios waiting on it will record this + physical block as their new physical address and proceed to step 9 + to record their new mapping. If there are more data_vios in the hash + lock than there are references available, one of the remaining + data_vios becomes the new agent and continues to step 8d as if no + valid advice was returned. - g) If the agent's data is not compressed, it will attempt to write - its data to the block it has allocated. + d. If no usable duplicate block was found, the agent first checks that + it has an allocated physical block (from step 3) that it can write + to. If the agent does not have an allocation, some other data_vio in + the hash lock that does have an allocation takes over as agent. If + none of the data_vios have an allocated physical block, these writes + are out of space, so they proceed to step 13 for cleanup. - h) If the data was written, this new block is treated as a - duplicate and shared as much as possible with any other - data_vios in the hash lock. + e. The agent attempts to compress its data. If the data does not + compress, the data_vio will continue to step 8h to write its data + directly. - i) If the agent wrote new data (whether compressed or not), the - index is updated to reflect the new entry. + If the compressed size is small enough, the agent will release the + implicit hash zone lock and go to the packer (struct packer) where + it will be placed in a bin (struct packer_bin) along with other + data_vios. All compression operations require the implicit lock on + the packer zone. -9. The block map is queried to determine the previous mapping of the - logical address. + The packer can combine up to 14 compressed blocks in a single 4k + data block. Compression is only helpful if vdo can pack at least 2 + data_vios into a single data block. This means that a data_vio may + wait in the packer for an arbitrarily long time for other data_vios + to fill out the compressed block. There is a mechanism for vdo to + evict waiting data_vios when continuing to wait would cause + problems. Circumstances causing an eviction include an application + flush, device shutdown, or a subsequent data_vio trying to overwrite + the same logical block address. A data_vio may also be evicted from + the packer if it cannot be paired with any other compressed block + before more compressible blocks need to use its bin. An evicted + data_vio will proceed to step 8h to write its data directly. -10. An entry is made in the recovery journal. The data_vio will block in - the journal until a flush has completed to ensure the data it may have - written is stable. It must also wait until its journal entry is stable - on disk. (Journal writes are all issued with the FUA bit set.) + f. If the agent fills a packer bin, either because all 14 of its slots + are used or because it has no remaining space, it is written out + using the allocated physical block from one of its data_vios. Step + 8d has already ensured that an allocation is available. + + g. Each data_vio sets the compressed block as its new physical address. + The data_vio obtains an implicit lock on the physical zone and + acquires the struct pbn_lock for the compressed block, which is + modified to be a shared lock. Then it releases the implicit physical + zone lock and proceeds to step 8i. + + h. Any data_vio evicted from the packer will have an allocation from + step 3. It will write its data to that allocated physical block. + + i. After the data is written, if the data_vio is the agent of a hash + lock, it will reacquire the implicit hash zone lock and share its + physical address with as many other data_vios in the hash lock as + possible. Each data_vio will then proceed to step 9 to record its + new mapping. + + j. If the agent actually wrote new data (whether compressed or not), + the deduplication index is updated to reflect the location of the + new data. The agent then releases the implicit hash zone lock. + +9. The data_vio determines the previous mapping of the logical address. + There is a cache for block map leaf pages (the "block map cache"), + because there are usually too many block map leaf nodes to store + entirely in memory. If the desired leaf page is not in the cache, the + data_vio will reserve a slot in the cache and load the desired page + into it, possibly evicting an older cached page. The data_vio then + finds the current physical address for this logical address (the "old + physical mapping"), if any, and records it. This step requires a lock + on the block map cache structures, covered by the implicit logical zone + lock. + +10. The data_vio makes an entry in the recovery journal containing the + logical block address, the old physical mapping, and the new physical + mapping. Making this journal entry requires holding the implicit + recovery journal lock. The data_vio will wait in the journal until all + recovery blocks up to the one containing its entry have been written + and flushed to ensure the transaction is stable on storage. 11. Once the recovery journal entry is stable, the data_vio makes two slab journal entries: an increment entry for the new mapping, and a - decrement entry for the old mapping, if that mapping was non-zero. For - correctness during recovery, the slab journal entries in any given slab - journal must be in the same order as the corresponding recovery journal - entries. Therefore, if the two entries are in different zones, they are - made concurrently, and if they are in the same zone, the increment is - always made before the decrement in order to avoid underflow. After - each slab journal entry is made in memory, the associated reference - count is also updated in memory. Each of these updates will get written - out as needed. (Slab journal blocks are written out either when they - are full, or when the recovery journal requests they do so in order to - allow the recovery journal to free up space; reference count blocks are - written out whenever the associated slab journal requests they do so in - order to free up slab journal space.) + decrement entry for the old mapping. These two operations each require + holding a lock on the affected physical slab, covered by its implicit + physical zone lock. For correctness during recovery, the slab journal + entries in any given slab journal must be in the same order as the + corresponding recovery journal entries. Therefore, if the two entries + are in different zones, they are made concurrently, and if they are in + the same zone, the increment is always made before the decrement in + order to avoid underflow. After each slab journal entry is made in + memory, the associated reference count is also updated in memory. -12. Once all the reference count updates are done, the block map is updated - and the write is complete. +12. Once both of the reference count updates are done, the data_vio + acquires the implicit logical zone lock and updates the + logical-to-physical mapping in the block map to point to the new + physical block. At this point the write operation is complete. -13. If the data_vio did not use its allocation, it releases the allocated - block, the hash lock (if it has one), and its logical lock. The - data_vio then returns to the pool. +13. If the data_vio has a hash lock, it acquires the implicit hash zone + lock and releases its hash lock to the pool. + + The data_vio then acquires the implicit physical zone lock and releases + the struct pbn_lock it holds for its allocated block. If it had an + allocation that it did not use, it also sets the reference count for + that block back to zero to free it for use by subsequent data_vios. + + The data_vio then acquires the implicit logical zone lock and releases + the logical block lock acquired in step 2. + + The application bio is then acknowledged if it has not previously been + acknowledged, and the data_vio is returned to the pool. *Read Path* -Reads are much simpler than writes. After a data_vio is assigned to the -bio, and the logical lock is obtained, the block map is queried. If the -block is mapped, the appropriate physical block is read, and if necessary, -decompressed. +An application read bio follows a much simpler set of steps. It does steps +1 and 2 in the write path to obtain a data_vio and lock its logical +address. If there is already a write data_vio in progress for that logical +address that is guaranteed to complete, the read data_vio will copy the +data from the write data_vio and return it. Otherwise, it will look up the +logical-to-physical mapping by traversing the block map tree as in step 3, +and then read and possibly decompress the indicated data at the indicated +physical block address. A read data_vio will not allocate block map tree +nodes if they are missing. If the interior block map nodes do not exist +yet, the logical block map address must still be unmapped and the read +data_vio will return all zeroes. A read data_vio handles cleanup and +acknowledgment as in step 13, although it only needs to release the logical +lock and return itself to the pool. + +*Small Writes* + +All storage within vdo is managed as 4KB blocks, but it can accept writes +as small as 512 bytes. Processing a write that is smaller than 4K requires +a read-modify-write operation that reads the relevant 4K block, copies the +new data over the approriate sectors of the block, and then launches a +write operation for the modified data block. The read and write stages of +this operation are nearly identical to the normal read and write +operations, and a single data_vio is used throughout this operation. *Recovery* @@ -399,7 +615,7 @@ into the slab journals. Finally, each physical zone attempts to replay at least one slab journal to reconstruct the reference counts of one slab. Once each zone has some free space (or has determined that it has none), the vdo comes back online, while the remainder of the slab journals are -used to reconstruct the rest of the reference counts. +used to reconstruct the rest of the reference counts in the background. *Read-only Rebuild* @@ -407,9 +623,11 @@ If a vdo encounters an unrecoverable error, it will enter read-only mode. This mode indicates that some previously acknowledged data may have been lost. The vdo may be instructed to rebuild as best it can in order to return to a writable state. However, this is never done automatically due -to the likelihood that data has been lost. During a read-only rebuild, the +to the possibility that data has been lost. During a read-only rebuild, the block map is recovered from the recovery journal as before. However, the -reference counts are not rebuilt from the slab journals. Rather, the -reference counts are zeroed, and then the entire block map is traversed, -and the reference counts are updated from it. While this may lose some -data, it ensures that the block map and reference counts are consistent. +reference counts are not rebuilt from the slab journals. Instead, the +reference counts are zeroed, the entire block map is traversed, and the +reference counts are updated from the block mappings. While this may lose +some data, it ensures that the block map and reference counts are +consistent with each other. This allows vdo to resume normal operation and +accept further writes. From ea9ca07affd80668b207703919eaba849654e11f Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Tue, 6 Feb 2024 22:00:42 -0500 Subject: [PATCH 0626/1038] dm vdo: add documentation details on zones and locking Add details describing the vdo zone and thread model to the documentation comments for major vdo components. Also added some high-level description of the block map structure. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.h | 15 +++++++++++++++ drivers/md/dm-vdo/dedupe.c | 5 +++++ drivers/md/dm-vdo/recovery-journal.h | 4 ++++ drivers/md/dm-vdo/slab-depot.h | 16 +++++++++++----- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h index c574bd524bc2..b662c318c2ea 100644 --- a/drivers/md/dm-vdo/block-map.h +++ b/drivers/md/dm-vdo/block-map.h @@ -19,6 +19,21 @@ #include "vio.h" #include "wait-queue.h" +/* + * The block map is responsible for tracking all the logical to physical mappings of a VDO. It + * consists of a collection of 60 radix trees gradually allocated as logical addresses are used. + * Each tree is assigned to a logical zone such that it is easy to compute which zone must handle + * each logical address. Each logical zone also has a dedicated portion of the leaf page cache. + * + * Each logical zone has a single dedicated queue and thread for performing all updates to the + * radix trees assigned to that zone. The concurrency guarantees of this single-threaded model + * allow the code to omit more fine-grained locking for the block map structures. + * + * Load operations must be performed on the admin thread. Normal operations, such as reading and + * updating mappings, must be performed on the appropriate logical zone thread. Save operations + * must be launched from the same admin thread as the original load operation. + */ + enum { BLOCK_MAP_VIO_POOL_SIZE = 64, }; diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 4b00135511dd..d81065a0951c 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -14,6 +14,11 @@ * deduplicate against a single block instead of being serialized through a PBN read lock. Only one * index query is needed for each hash_lock, instead of one for every data_vio. * + * Hash_locks are assigned to hash_zones by computing a modulus on the hash itself. Each hash_zone + * has a single dedicated queue and thread for performing all operations on the hash_locks assigned + * to that zone. The concurrency guarantees of this single-threaded model allow the code to omit + * more fine-grained locking for the hash_lock structures. + * * A hash_lock acts like a state machine perhaps more than as a lock. Other than the starting and * ending states INITIALIZING and BYPASSING, every state represents and is held for the duration of * an asynchronous operation. All state transitions are performed on the thread of the hash_zone diff --git a/drivers/md/dm-vdo/recovery-journal.h b/drivers/md/dm-vdo/recovery-journal.h index 19fa7ed9648a..d78c6c7da4ea 100644 --- a/drivers/md/dm-vdo/recovery-journal.h +++ b/drivers/md/dm-vdo/recovery-journal.h @@ -26,6 +26,10 @@ * write amplification of writes by providing amortization of slab journal and block map page * updates. * + * The recovery journal has a single dedicated queue and thread for performing all journal updates. + * The concurrency guarantees of this single-threaded model allow the code to omit more + * fine-grained locking for recovery journal structures. + * * The journal consists of a set of on-disk blocks arranged as a circular log with monotonically * increasing sequence numbers. Three sequence numbers serve to define the active extent of the * journal. The 'head' is the oldest active block in the journal. The 'tail' is the end of the diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h index efdef566709a..fba293f9713e 100644 --- a/drivers/md/dm-vdo/slab-depot.h +++ b/drivers/md/dm-vdo/slab-depot.h @@ -29,11 +29,17 @@ * a single array of slabs in order to eliminate the need for additional math in order to compute * which physical zone a PBN is in. It also has a block_allocator per zone. * - * Load operations are required to be performed on a single thread. Normal operations are assumed - * to be performed in the appropriate zone. Allocations and reference count updates must be done - * from the thread of their physical zone. Requests to commit slab journal tail blocks from the - * recovery journal must be done on the journal zone thread. Save operations are required to be - * launched from the same thread as the original load operation. + * Each physical zone has a single dedicated queue and thread for performing all updates to the + * slabs assigned to that zone. The concurrency guarantees of this single-threaded model allow the + * code to omit more fine-grained locking for the various slab structures. Each physical zone + * maintains a separate copy of the slab summary to remove the need for explicit locking on that + * structure as well. + * + * Load operations must be performed on the admin thread. Normal operations, such as allocations + * and reference count updates, must be performed on the appropriate physical zone thread. Requests + * from the recovery journal to commit slab journal tail blocks must be scheduled from the recovery + * journal thread to run on the appropriate physical zone thread. Save operations must be launched + * from the same admin thread as the original load operation. */ enum { From b196d6bd30c39548436cbc7cfd40efcf4f006dfa Mon Sep 17 00:00:00 2001 From: Susan LeGendre-McGhee Date: Mon, 5 Feb 2024 21:04:22 -0500 Subject: [PATCH 0627/1038] dm vdo: move encoding constants to encodings.c Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/constants.h | 12 +++--------- drivers/md/dm-vdo/encodings.c | 6 ++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/md/dm-vdo/constants.h b/drivers/md/dm-vdo/constants.h index 3a997dd7d0b1..24237bc29f87 100644 --- a/drivers/md/dm-vdo/constants.h +++ b/drivers/md/dm-vdo/constants.h @@ -44,6 +44,9 @@ enum { /** The default size of each slab journal, in blocks */ DEFAULT_VDO_SLAB_JOURNAL_SIZE = 224, + /** Unit test minimum */ + MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2, + /* * The initial size of lbn_operations and pbn_operations, which is based upon the expected * maximum number of outstanding VIOs. This value was chosen to make it highly unlikely @@ -90,13 +93,4 @@ enum { VDO_ZERO_BLOCK = 0, }; -/** The maximum logical space is 4 petabytes, which is 1 terablock. */ -static const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; - -/** The maximum physical space is 256 terabytes, which is 64 gigablocks. */ -static const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; - -/** unit test minimum */ -static const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2; - #endif /* VDO_CONSTANTS_H */ diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index 6b3e9a5b3a91..9e45411fe816 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -15,6 +15,12 @@ #include "status-codes.h" #include "types.h" +/** The maximum logical space is 4 petabytes, which is 1 terablock. */ +static const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; + +/** The maximum physical space is 256 terabytes, which is 64 gigablocks. */ +static const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; + struct geometry_block { char magic_number[VDO_GEOMETRY_MAGIC_NUMBER_SIZE]; struct packed_header header; From fbbd7a25e8b568259e2b4125714308d765e70baf Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 7 Feb 2024 11:12:44 -0500 Subject: [PATCH 0628/1038] dm vdo: use #define for NO_CHAPTER and NO_CHAPTER_INDEX_ENTRY Avoids unconventional use of 'static const' and enum in headers. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/chapter-index.h | 6 ++---- drivers/md/dm-vdo/volume-index.h | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/chapter-index.h b/drivers/md/dm-vdo/chapter-index.h index e1bc7e31a2a5..be8bf2b675b1 100644 --- a/drivers/md/dm-vdo/chapter-index.h +++ b/drivers/md/dm-vdo/chapter-index.h @@ -18,10 +18,8 @@ * is more efficient. Both types of chapter index are implemented with a delta index. */ -enum { - /* The value returned when no entry is found in the chapter index. */ - NO_CHAPTER_INDEX_ENTRY = U16_MAX, -}; +/* The value returned when no entry is found in the chapter index. */ +#define NO_CHAPTER_INDEX_ENTRY U16_MAX struct open_chapter_index { const struct index_geometry *geometry; diff --git a/drivers/md/dm-vdo/volume-index.h b/drivers/md/dm-vdo/volume-index.h index 0f08acf29435..537e9947cf4a 100644 --- a/drivers/md/dm-vdo/volume-index.h +++ b/drivers/md/dm-vdo/volume-index.h @@ -26,7 +26,7 @@ * managed with volume_index_record structures. */ -static const u64 NO_CHAPTER = U64_MAX; +#define NO_CHAPTER U64_MAX struct volume_index_stats { /* Nanoseconds spent rebalancing */ From 5f770bd1f250217846e8c055ec1858a93bffd56b Mon Sep 17 00:00:00 2001 From: Ken Raeburn Date: Tue, 13 Feb 2024 20:14:56 -0500 Subject: [PATCH 0629/1038] dm vdo message-stats: reformat to remove excessive newlines Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/message-stats.c | 791 +++++++----------------------- 1 file changed, 174 insertions(+), 617 deletions(-) diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index 5be8503ed96e..af964e55b98c 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -11,16 +11,11 @@ #include "thread-device.h" #include "vdo.h" -static int write_u64(char *prefix, - u64 value, - char *suffix, - char **buf, +static int write_u64(char *prefix, u64 value, char *suffix, char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%llu%s", - prefix == NULL ? "" : prefix, - value, - suffix == NULL ? "" : suffix); + int count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; if (count >= *maxlen) @@ -28,16 +23,11 @@ static int write_u64(char *prefix, return VDO_SUCCESS; } -static int write_u32(char *prefix, - u32 value, - char *suffix, - char **buf, +static int write_u32(char *prefix, u32 value, char *suffix, char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%u%s", - prefix == NULL ? "" : prefix, - value, - suffix == NULL ? "" : suffix); + int count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; if (count >= *maxlen) @@ -45,16 +35,11 @@ static int write_u32(char *prefix, return VDO_SUCCESS; } -static int write_block_count_t(char *prefix, - block_count_t value, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_block_count_t(char *prefix, block_count_t value, char *suffix, + char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%llu%s", - prefix == NULL ? "" : prefix, - value, - suffix == NULL ? "" : suffix); + int count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; if (count >= *maxlen) @@ -62,16 +47,11 @@ static int write_block_count_t(char *prefix, return VDO_SUCCESS; } -static int write_string(char *prefix, - char *value, - char *suffix, - char **buf, +static int write_string(char *prefix, char *value, char *suffix, char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%s%s", - prefix == NULL ? "" : prefix, - value, - suffix == NULL ? "" : suffix); + int count = scnprintf(*buf, *maxlen, "%s%s%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; if (count >= *maxlen) @@ -79,16 +59,11 @@ static int write_string(char *prefix, return VDO_SUCCESS; } -static int write_bool(char *prefix, - bool value, - char *suffix, - char **buf, +static int write_bool(char *prefix, bool value, char *suffix, char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%d%s", - prefix == NULL ? "" : prefix, - value, - suffix == NULL ? "" : suffix); + int count = scnprintf(*buf, *maxlen, "%s%d%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; if (count >= *maxlen) @@ -96,16 +71,11 @@ static int write_bool(char *prefix, return VDO_SUCCESS; } -static int write_u8(char *prefix, - u8 value, - char *suffix, - char **buf, +static int write_u8(char *prefix, u8 value, char *suffix, char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%u%s", - prefix == NULL ? "" : prefix, - value, - suffix == NULL ? "" : suffix); + int count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; if (count >= *maxlen) @@ -115,8 +85,7 @@ static int write_u8(char *prefix, static int write_block_allocator_statistics(char *prefix, struct block_allocator_statistics *stats, - char *suffix, - char **buf, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -125,27 +94,15 @@ static int write_block_allocator_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* The total number of slabs from which blocks may be allocated */ - result = write_u64("slabCount : ", - stats->slab_count, - ", ", - buf, - maxlen); + result = write_u64("slabCount : ", stats->slab_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The total number of slabs from which blocks have ever been allocated */ - result = write_u64("slabsOpened : ", - stats->slabs_opened, - ", ", - buf, - maxlen); + result = write_u64("slabsOpened : ", stats->slabs_opened, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The number of times since loading that a slab has been re-opened */ - result = write_u64("slabsReopened : ", - stats->slabs_reopened, - ", ", - buf, - maxlen); + result = write_u64("slabsReopened : ", stats->slabs_reopened, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -154,11 +111,8 @@ static int write_block_allocator_statistics(char *prefix, return VDO_SUCCESS; } -static int write_commit_statistics(char *prefix, - struct commit_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_commit_statistics(char *prefix, struct commit_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -166,27 +120,15 @@ static int write_commit_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* The total number of items on which processing has started */ - result = write_u64("started : ", - stats->started, - ", ", - buf, - maxlen); + result = write_u64("started : ", stats->started, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The total number of items for which a write operation has been issued */ - result = write_u64("written : ", - stats->written, - ", ", - buf, - maxlen); + result = write_u64("written : ", stats->written, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The total number of items for which a write operation has completed */ - result = write_u64("committed : ", - stats->committed, - ", ", - buf, - maxlen); + result = write_u64("committed : ", stats->committed, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -197,8 +139,7 @@ static int write_commit_statistics(char *prefix, static int write_recovery_journal_statistics(char *prefix, struct recovery_journal_statistics *stats, - char *suffix, - char **buf, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -207,35 +148,21 @@ static int write_recovery_journal_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of times the on-disk journal was full */ - result = write_u64("diskFull : ", - stats->disk_full, - ", ", - buf, - maxlen); + result = write_u64("diskFull : ", stats->disk_full, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times the recovery journal requested slab journal commits. */ result = write_u64("slabJournalCommitsRequested : ", - stats->slab_journal_commits_requested, - ", ", - buf, - maxlen); + stats->slab_journal_commits_requested, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Write/Commit totals for individual journal entries */ - result = write_commit_statistics("entries : ", - &stats->entries, - ", ", - buf, + result = write_commit_statistics("entries : ", &stats->entries, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Write/Commit totals for journal blocks */ - result = write_commit_statistics("blocks : ", - &stats->blocks, - ", ", - buf, - maxlen); + result = write_commit_statistics("blocks : ", &stats->blocks, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -244,11 +171,8 @@ static int write_recovery_journal_statistics(char *prefix, return VDO_SUCCESS; } -static int write_packer_statistics(char *prefix, - struct packer_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_packer_statistics(char *prefix, struct packer_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -257,26 +181,17 @@ static int write_packer_statistics(char *prefix, return result; /* Number of compressed data items written since startup */ result = write_u64("compressedFragmentsWritten : ", - stats->compressed_fragments_written, - ", ", - buf, - maxlen); + stats->compressed_fragments_written, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of blocks containing compressed items written since startup */ result = write_u64("compressedBlocksWritten : ", - stats->compressed_blocks_written, - ", ", - buf, - maxlen); + stats->compressed_blocks_written, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of VIOs that are pending in the packer */ result = write_u64("compressedFragmentsInPacker : ", - stats->compressed_fragments_in_packer, - ", ", - buf, - maxlen); + stats->compressed_fragments_in_packer, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -287,9 +202,7 @@ static int write_packer_statistics(char *prefix, static int write_slab_journal_statistics(char *prefix, struct slab_journal_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -297,42 +210,24 @@ static int write_slab_journal_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of times the on-disk journal was full */ - result = write_u64("diskFullCount : ", - stats->disk_full_count, - ", ", - buf, + result = write_u64("diskFullCount : ", stats->disk_full_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times an entry was added over the flush threshold */ - result = write_u64("flushCount : ", - stats->flush_count, - ", ", - buf, - maxlen); + result = write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times an entry was added over the block threshold */ - result = write_u64("blockedCount : ", - stats->blocked_count, - ", ", - buf, - maxlen); + result = write_u64("blockedCount : ", stats->blocked_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times a tail block was written */ - result = write_u64("blocksWritten : ", - stats->blocks_written, - ", ", - buf, - maxlen); + result = write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times we had to wait for the tail to write */ - result = write_u64("tailBusyCount : ", - stats->tail_busy_count, - ", ", - buf, + result = write_u64("tailBusyCount : ", stats->tail_busy_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; @@ -344,9 +239,7 @@ static int write_slab_journal_statistics(char *prefix, static int write_slab_summary_statistics(char *prefix, struct slab_summary_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -354,11 +247,7 @@ static int write_slab_summary_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of blocks written */ - result = write_u64("blocksWritten : ", - stats->blocks_written, - ", ", - buf, - maxlen); + result = write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -367,11 +256,8 @@ static int write_slab_summary_statistics(char *prefix, return VDO_SUCCESS; } -static int write_ref_counts_statistics(char *prefix, - struct ref_counts_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_ref_counts_statistics(char *prefix, struct ref_counts_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -379,11 +265,7 @@ static int write_ref_counts_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of reference blocks written */ - result = write_u64("blocksWritten : ", - stats->blocks_written, - ", ", - buf, - maxlen); + result = write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -392,11 +274,8 @@ static int write_ref_counts_statistics(char *prefix, return VDO_SUCCESS; } -static int write_block_map_statistics(char *prefix, - struct block_map_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_block_map_statistics(char *prefix, struct block_map_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -404,163 +283,84 @@ static int write_block_map_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* number of dirty (resident) pages */ - result = write_u32("dirtyPages : ", - stats->dirty_pages, - ", ", - buf, - maxlen); + result = write_u32("dirtyPages : ", stats->dirty_pages, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of clean (resident) pages */ - result = write_u32("cleanPages : ", - stats->clean_pages, - ", ", - buf, - maxlen); + result = write_u32("cleanPages : ", stats->clean_pages, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of free pages */ - result = write_u32("freePages : ", - stats->free_pages, - ", ", - buf, - maxlen); + result = write_u32("freePages : ", stats->free_pages, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of pages in failed state */ - result = write_u32("failedPages : ", - stats->failed_pages, - ", ", - buf, - maxlen); + result = write_u32("failedPages : ", stats->failed_pages, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of pages incoming */ - result = write_u32("incomingPages : ", - stats->incoming_pages, - ", ", - buf, - maxlen); + result = write_u32("incomingPages : ", stats->incoming_pages, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of pages outgoing */ - result = write_u32("outgoingPages : ", - stats->outgoing_pages, - ", ", - buf, - maxlen); + result = write_u32("outgoingPages : ", stats->outgoing_pages, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* how many times free page not avail */ - result = write_u32("cachePressure : ", - stats->cache_pressure, - ", ", - buf, - maxlen); + result = write_u32("cachePressure : ", stats->cache_pressure, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of get_vdo_page() calls for read */ - result = write_u64("readCount : ", - stats->read_count, - ", ", - buf, - maxlen); + result = write_u64("readCount : ", stats->read_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of get_vdo_page() calls for write */ - result = write_u64("writeCount : ", - stats->write_count, - ", ", - buf, - maxlen); + result = write_u64("writeCount : ", stats->write_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of times pages failed to read */ - result = write_u64("failedReads : ", - stats->failed_reads, - ", ", - buf, - maxlen); + result = write_u64("failedReads : ", stats->failed_reads, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of times pages failed to write */ - result = write_u64("failedWrites : ", - stats->failed_writes, - ", ", - buf, - maxlen); + result = write_u64("failedWrites : ", stats->failed_writes, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of gets that are reclaimed */ - result = write_u64("reclaimed : ", - stats->reclaimed, - ", ", - buf, - maxlen); + result = write_u64("reclaimed : ", stats->reclaimed, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of gets for outgoing pages */ - result = write_u64("readOutgoing : ", - stats->read_outgoing, - ", ", - buf, - maxlen); + result = write_u64("readOutgoing : ", stats->read_outgoing, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of gets that were already there */ - result = write_u64("foundInCache : ", - stats->found_in_cache, - ", ", - buf, - maxlen); + result = write_u64("foundInCache : ", stats->found_in_cache, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of gets requiring discard */ - result = write_u64("discardRequired : ", - stats->discard_required, - ", ", - buf, + result = write_u64("discardRequired : ", stats->discard_required, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of gets enqueued for their page */ - result = write_u64("waitForPage : ", - stats->wait_for_page, - ", ", - buf, - maxlen); + result = write_u64("waitForPage : ", stats->wait_for_page, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of gets that have to fetch */ - result = write_u64("fetchRequired : ", - stats->fetch_required, - ", ", - buf, - maxlen); + result = write_u64("fetchRequired : ", stats->fetch_required, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of page fetches */ - result = write_u64("pagesLoaded : ", - stats->pages_loaded, - ", ", - buf, - maxlen); + result = write_u64("pagesLoaded : ", stats->pages_loaded, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of page saves */ - result = write_u64("pagesSaved : ", - stats->pages_saved, - ", ", - buf, - maxlen); + result = write_u64("pagesSaved : ", stats->pages_saved, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* the number of flushes issued */ - result = write_u64("flushCount : ", - stats->flush_count, - ", ", - buf, - maxlen); + result = write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -569,11 +369,8 @@ static int write_block_map_statistics(char *prefix, return VDO_SUCCESS; } -static int write_hash_lock_statistics(char *prefix, - struct hash_lock_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_hash_lock_statistics(char *prefix, struct hash_lock_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -581,42 +378,27 @@ static int write_hash_lock_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of times the UDS advice proved correct */ - result = write_u64("dedupeAdviceValid : ", - stats->dedupe_advice_valid, - ", ", - buf, + result = write_u64("dedupeAdviceValid : ", stats->dedupe_advice_valid, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times the UDS advice proved incorrect */ - result = write_u64("dedupeAdviceStale : ", - stats->dedupe_advice_stale, - ", ", - buf, + result = write_u64("dedupeAdviceStale : ", stats->dedupe_advice_stale, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of writes with the same data as another in-flight write */ - result = write_u64("concurrentDataMatches : ", - stats->concurrent_data_matches, - ", ", - buf, - maxlen); + result = write_u64("concurrentDataMatches : ", stats->concurrent_data_matches, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of writes whose hash collided with an in-flight write */ result = write_u64("concurrentHashCollisions : ", - stats->concurrent_hash_collisions, - ", ", - buf, - maxlen); + stats->concurrent_hash_collisions, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Current number of dedupe queries that are in flight */ - result = write_u32("currDedupeQueries : ", - stats->curr_dedupe_queries, - ", ", - buf, + result = write_u32("currDedupeQueries : ", stats->curr_dedupe_queries, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; @@ -626,11 +408,8 @@ static int write_hash_lock_statistics(char *prefix, return VDO_SUCCESS; } -static int write_error_statistics(char *prefix, - struct error_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_error_statistics(char *prefix, struct error_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -638,27 +417,18 @@ static int write_error_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* number of times VDO got an invalid dedupe advice PBN from UDS */ - result = write_u64("invalidAdvicePBNCount : ", - stats->invalid_advice_pbn_count, - ", ", - buf, - maxlen); + result = write_u64("invalidAdvicePBNCount : ", stats->invalid_advice_pbn_count, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of times a VIO completed with a VDO_NO_SPACE error */ - result = write_u64("noSpaceErrorCount : ", - stats->no_space_error_count, - ", ", - buf, - maxlen); + result = write_u64("noSpaceErrorCount : ", stats->no_space_error_count, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of times a VIO completed with a VDO_READ_ONLY error */ - result = write_u64("readOnlyErrorCount : ", - stats->read_only_error_count, - ", ", - buf, - maxlen); + result = write_u64("readOnlyErrorCount : ", stats->read_only_error_count, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -667,11 +437,8 @@ static int write_error_statistics(char *prefix, return VDO_SUCCESS; } -static int write_bio_stats(char *prefix, - struct bio_stats *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_bio_stats(char *prefix, struct bio_stats *stats, char *suffix, + char **buf, unsigned int *maxlen) { int result; @@ -679,51 +446,27 @@ static int write_bio_stats(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of REQ_OP_READ bios */ - result = write_u64("read : ", - stats->read, - ", ", - buf, - maxlen); + result = write_u64("read : ", stats->read, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of REQ_OP_WRITE bios with data */ - result = write_u64("write : ", - stats->write, - ", ", - buf, - maxlen); + result = write_u64("write : ", stats->write, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of bios tagged with REQ_PREFLUSH and containing no data */ - result = write_u64("emptyFlush : ", - stats->empty_flush, - ", ", - buf, - maxlen); + result = write_u64("emptyFlush : ", stats->empty_flush, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of REQ_OP_DISCARD bios */ - result = write_u64("discard : ", - stats->discard, - ", ", - buf, - maxlen); + result = write_u64("discard : ", stats->discard, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of bios tagged with REQ_PREFLUSH */ - result = write_u64("flush : ", - stats->flush, - ", ", - buf, - maxlen); + result = write_u64("flush : ", stats->flush, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of bios tagged with REQ_FUA */ - result = write_u64("fua : ", - stats->fua, - ", ", - buf, - maxlen); + result = write_u64("fua : ", stats->fua, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -732,11 +475,8 @@ static int write_bio_stats(char *prefix, return VDO_SUCCESS; } -static int write_memory_usage(char *prefix, - struct memory_usage *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_memory_usage(char *prefix, struct memory_usage *stats, char *suffix, + char **buf, unsigned int *maxlen) { int result; @@ -744,18 +484,11 @@ static int write_memory_usage(char *prefix, if (result != VDO_SUCCESS) return result; /* Tracked bytes currently allocated. */ - result = write_u64("bytesUsed : ", - stats->bytes_used, - ", ", - buf, - maxlen); + result = write_u64("bytesUsed : ", stats->bytes_used, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Maximum tracked bytes allocated. */ - result = write_u64("peakBytesUsed : ", - stats->peak_bytes_used, - ", ", - buf, + result = write_u64("peakBytesUsed : ", stats->peak_bytes_used, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; @@ -765,11 +498,8 @@ static int write_memory_usage(char *prefix, return VDO_SUCCESS; } -static int write_index_statistics(char *prefix, - struct index_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_index_statistics(char *prefix, struct index_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { int result; @@ -777,66 +507,39 @@ static int write_index_statistics(char *prefix, if (result != VDO_SUCCESS) return result; /* Number of records stored in the index */ - result = write_u64("entriesIndexed : ", - stats->entries_indexed, - ", ", - buf, + result = write_u64("entriesIndexed : ", stats->entries_indexed, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of post calls that found an existing entry */ - result = write_u64("postsFound : ", - stats->posts_found, - ", ", - buf, - maxlen); + result = write_u64("postsFound : ", stats->posts_found, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of post calls that added a new entry */ - result = write_u64("postsNotFound : ", - stats->posts_not_found, - ", ", - buf, + result = write_u64("postsNotFound : ", stats->posts_not_found, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of query calls that found an existing entry */ - result = write_u64("queriesFound : ", - stats->queries_found, - ", ", - buf, - maxlen); + result = write_u64("queriesFound : ", stats->queries_found, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of query calls that added a new entry */ - result = write_u64("queriesNotFound : ", - stats->queries_not_found, - ", ", - buf, + result = write_u64("queriesNotFound : ", stats->queries_not_found, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of update calls that found an existing entry */ - result = write_u64("updatesFound : ", - stats->updates_found, - ", ", - buf, - maxlen); + result = write_u64("updatesFound : ", stats->updates_found, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of update calls that added a new entry */ - result = write_u64("updatesNotFound : ", - stats->updates_not_found, - ", ", - buf, + result = write_u64("updatesNotFound : ", stats->updates_not_found, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of entries discarded */ - result = write_u64("entriesDiscarded : ", - stats->entries_discarded, - ", ", - buf, + result = write_u64("entriesDiscarded : ", stats->entries_discarded, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; @@ -846,349 +549,205 @@ static int write_index_statistics(char *prefix, return VDO_SUCCESS; } -static int write_vdo_statistics(char *prefix, - struct vdo_statistics *stats, - char *suffix, - char **buf, - unsigned int *maxlen) +static int write_vdo_statistics(char *prefix, struct vdo_statistics *stats, char *suffix, + char **buf, unsigned int *maxlen) { int result; result = write_string(prefix, "{ ", NULL, buf, maxlen); if (result != VDO_SUCCESS) return result; - result = write_u32("version : ", - stats->version, - ", ", - buf, - maxlen); + result = write_u32("version : ", stats->version, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of blocks used for data */ - result = write_u64("dataBlocksUsed : ", - stats->data_blocks_used, - ", ", - buf, + result = write_u64("dataBlocksUsed : ", stats->data_blocks_used, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of blocks used for VDO metadata */ - result = write_u64("overheadBlocksUsed : ", - stats->overhead_blocks_used, - ", ", - buf, - maxlen); + result = write_u64("overheadBlocksUsed : ", stats->overhead_blocks_used, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of logical blocks that are currently mapped to physical blocks */ - result = write_u64("logicalBlocksUsed : ", - stats->logical_blocks_used, - ", ", - buf, + result = write_u64("logicalBlocksUsed : ", stats->logical_blocks_used, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of physical blocks */ - result = write_block_count_t("physicalBlocks : ", - stats->physical_blocks, - ", ", - buf, - maxlen); + result = write_block_count_t("physicalBlocks : ", stats->physical_blocks, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* number of logical blocks */ - result = write_block_count_t("logicalBlocks : ", - stats->logical_blocks, - ", ", - buf, - maxlen); + result = write_block_count_t("logicalBlocks : ", stats->logical_blocks, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* Size of the block map page cache, in bytes */ - result = write_u64("blockMapCacheSize : ", - stats->block_map_cache_size, - ", ", - buf, - maxlen); + result = write_u64("blockMapCacheSize : ", stats->block_map_cache_size, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* The physical block size */ - result = write_u64("blockSize : ", - stats->block_size, - ", ", - buf, - maxlen); + result = write_u64("blockSize : ", stats->block_size, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times the VDO has successfully recovered */ - result = write_u64("completeRecoveries : ", - stats->complete_recoveries, - ", ", - buf, - maxlen); + result = write_u64("completeRecoveries : ", stats->complete_recoveries, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times the VDO has recovered from read-only mode */ - result = write_u64("readOnlyRecoveries : ", - stats->read_only_recoveries, - ", ", - buf, - maxlen); + result = write_u64("readOnlyRecoveries : ", stats->read_only_recoveries, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* String describing the operating mode of the VDO */ - result = write_string("mode : ", - stats->mode, - ", ", - buf, - maxlen); + result = write_string("mode : ", stats->mode, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Whether the VDO is in recovery mode */ - result = write_bool("inRecoveryMode : ", - stats->in_recovery_mode, - ", ", - buf, + result = write_bool("inRecoveryMode : ", stats->in_recovery_mode, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* What percentage of recovery mode work has been completed */ - result = write_u8("recoveryPercentage : ", - stats->recovery_percentage, - ", ", - buf, + result = write_u8("recoveryPercentage : ", stats->recovery_percentage, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The statistics for the compressed block packer */ - result = write_packer_statistics("packer : ", - &stats->packer, - ", ", - buf, - maxlen); + result = write_packer_statistics("packer : ", &stats->packer, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Counters for events in the block allocator */ - result = write_block_allocator_statistics("allocator : ", - &stats->allocator, - ", ", - buf, - maxlen); + result = write_block_allocator_statistics("allocator : ", &stats->allocator, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Counters for events in the recovery journal */ - result = write_recovery_journal_statistics("journal : ", - &stats->journal, - ", ", - buf, - maxlen); + result = write_recovery_journal_statistics("journal : ", &stats->journal, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* The statistics for the slab journals */ - result = write_slab_journal_statistics("slabJournal : ", - &stats->slab_journal, - ", ", - buf, - maxlen); + result = write_slab_journal_statistics("slabJournal : ", &stats->slab_journal, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The statistics for the slab summary */ - result = write_slab_summary_statistics("slabSummary : ", - &stats->slab_summary, - ", ", - buf, - maxlen); + result = write_slab_summary_statistics("slabSummary : ", &stats->slab_summary, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The statistics for the reference counts */ - result = write_ref_counts_statistics("refCounts : ", - &stats->ref_counts, - ", ", - buf, - maxlen); + result = write_ref_counts_statistics("refCounts : ", &stats->ref_counts, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* The statistics for the block map */ - result = write_block_map_statistics("blockMap : ", - &stats->block_map, - ", ", - buf, + result = write_block_map_statistics("blockMap : ", &stats->block_map, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The dedupe statistics from hash locks */ - result = write_hash_lock_statistics("hashLock : ", - &stats->hash_lock, - ", ", - buf, + result = write_hash_lock_statistics("hashLock : ", &stats->hash_lock, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Counts of error conditions */ - result = write_error_statistics("errors : ", - &stats->errors, - ", ", - buf, - maxlen); + result = write_error_statistics("errors : ", &stats->errors, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The VDO instance */ - result = write_u32("instance : ", - stats->instance, - ", ", - buf, - maxlen); + result = write_u32("instance : ", stats->instance, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Current number of active VIOs */ - result = write_u32("currentVIOsInProgress : ", - stats->current_vios_in_progress, - ", ", - buf, - maxlen); + result = write_u32("currentVIOsInProgress : ", stats->current_vios_in_progress, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Maximum number of active VIOs */ - result = write_u32("maxVIOs : ", - stats->max_vios, - ", ", - buf, - maxlen); + result = write_u32("maxVIOs : ", stats->max_vios, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of times the UDS index was too slow in responding */ - result = write_u64("dedupeAdviceTimeouts : ", - stats->dedupe_advice_timeouts, - ", ", - buf, - maxlen); + result = write_u64("dedupeAdviceTimeouts : ", stats->dedupe_advice_timeouts, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Number of flush requests submitted to the storage device */ - result = write_u64("flushOut : ", - stats->flush_out, - ", ", - buf, - maxlen); + result = write_u64("flushOut : ", stats->flush_out, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Logical block size */ - result = write_u64("logicalBlockSize : ", - stats->logical_block_size, - ", ", - buf, + result = write_u64("logicalBlockSize : ", stats->logical_block_size, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Bios submitted into VDO from above */ - result = write_bio_stats("biosIn : ", - &stats->bios_in, - ", ", - buf, - maxlen); + result = write_bio_stats("biosIn : ", &stats->bios_in, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; - result = write_bio_stats("biosInPartial : ", - &stats->bios_in_partial, - ", ", - buf, + result = write_bio_stats("biosInPartial : ", &stats->bios_in_partial, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Bios submitted onward for user data */ - result = write_bio_stats("biosOut : ", - &stats->bios_out, - ", ", - buf, - maxlen); + result = write_bio_stats("biosOut : ", &stats->bios_out, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Bios submitted onward for metadata */ - result = write_bio_stats("biosMeta : ", - &stats->bios_meta, - ", ", - buf, + result = write_bio_stats("biosMeta : ", &stats->bios_meta, ", ", buf, maxlen); + if (result != VDO_SUCCESS) + return result; + result = write_bio_stats("biosJournal : ", &stats->bios_journal, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; - result = write_bio_stats("biosJournal : ", - &stats->bios_journal, - ", ", - buf, + result = write_bio_stats("biosPageCache : ", &stats->bios_page_cache, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; - result = write_bio_stats("biosPageCache : ", - &stats->bios_page_cache, - ", ", - buf, - maxlen); + result = write_bio_stats("biosOutCompleted : ", &stats->bios_out_completed, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; - result = write_bio_stats("biosOutCompleted : ", - &stats->bios_out_completed, - ", ", - buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosMetaCompleted : ", - &stats->bios_meta_completed, - ", ", - buf, - maxlen); + result = write_bio_stats("biosMetaCompleted : ", &stats->bios_meta_completed, + ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_bio_stats("biosJournalCompleted : ", - &stats->bios_journal_completed, - ", ", - buf, - maxlen); + &stats->bios_journal_completed, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_bio_stats("biosPageCacheCompleted : ", - &stats->bios_page_cache_completed, - ", ", - buf, - maxlen); + &stats->bios_page_cache_completed, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; - result = write_bio_stats("biosAcknowledged : ", - &stats->bios_acknowledged, - ", ", - buf, - maxlen); + result = write_bio_stats("biosAcknowledged : ", &stats->bios_acknowledged, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_bio_stats("biosAcknowledgedPartial : ", - &stats->bios_acknowledged_partial, - ", ", - buf, - maxlen); + &stats->bios_acknowledged_partial, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* Current number of bios in progress */ - result = write_bio_stats("biosInProgress : ", - &stats->bios_in_progress, - ", ", - buf, - maxlen); + result = write_bio_stats("biosInProgress : ", &stats->bios_in_progress, ", ", + buf, maxlen); if (result != VDO_SUCCESS) return result; /* Memory usage stats. */ - result = write_memory_usage("memoryUsage : ", - &stats->memory_usage, - ", ", - buf, + result = write_memory_usage("memoryUsage : ", &stats->memory_usage, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; /* The statistics for the UDS index */ - result = write_index_statistics("index : ", - &stats->index, - ", ", - buf, - maxlen); + result = write_index_statistics("index : ", &stats->index, ", ", buf, maxlen); if (result != VDO_SUCCESS) return result; result = write_string(NULL, "}", suffix, buf, maxlen); @@ -1197,9 +756,7 @@ static int write_vdo_statistics(char *prefix, return VDO_SUCCESS; } -int vdo_write_stats(struct vdo *vdo, - char *buf, - unsigned int maxlen) +int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) { struct vdo_statistics *stats; int result; From 181547bbb81e8ed91e1df7eaa0303fcbc44caeff Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 23:57:08 -0500 Subject: [PATCH 0630/1038] dm vdo string-utils: remove unnecessary includes Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/string-utils.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/md/dm-vdo/string-utils.c b/drivers/md/dm-vdo/string-utils.c index a584b37bb70c..6cdf018cdaf0 100644 --- a/drivers/md/dm-vdo/string-utils.c +++ b/drivers/md/dm-vdo/string-utils.c @@ -5,12 +5,6 @@ #include "string-utils.h" -#include "errors.h" -#include "logger.h" -#include "memory-alloc.h" -#include "permassert.h" -#include "uds.h" - char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) { va_list args; From d008f6eeab44798943a86f7a0b50f1b1a6192f38 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 23:57:09 -0500 Subject: [PATCH 0631/1038] dm vdo dedupe: fix various small nits Add a __must_hold sparse annotation to launch_dedupe_state_change that reflects its ASSERTION code comments about locking requirements, add some extra braces and fix a couple typos. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dedupe.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index d81065a0951c..2a1902c4423c 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -81,7 +81,7 @@ * * A query to the UDS index is handled asynchronously by the index's threads. When the query is * complete, a callback supplied with the query will be called from one of the those threads. Under - * heavy system load, the index may be slower to respond then is desirable for reasonable I/O + * heavy system load, the index may be slower to respond than is desirable for reasonable I/O * throughput. Since deduplication of writes is not necessary for correct operation of a VDO * device, it is acceptable to timeout out slow index queries and proceed to fulfill a write * request without deduplicating. However, because the uds_request struct itself is supplied by the @@ -1311,7 +1311,7 @@ static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_loc * behalf of its hash lock. * * If the PBN is already locked for writing, the lock attempt is abandoned and is_duplicate will be - * cleared before calling back. this continuation is launched from start_locking(), and calls back + * cleared before calling back. This continuation is launched from start_locking(), and calls back * to finish_locking() on the hash zone thread. */ static void lock_duplicate_pbn(struct vdo_completion *completion) @@ -2300,9 +2300,10 @@ static void finish_index_operation(struct uds_request *request) * data_vio has already moved on. */ if (!change_context_state(context, DEDUPE_CONTEXT_TIMED_OUT, - DEDUPE_CONTEXT_TIMED_OUT_COMPLETE)) + DEDUPE_CONTEXT_TIMED_OUT_COMPLETE)) { ASSERT_LOG_ONLY(false, "uds request was timed out (state %d)", atomic_read(&context->state)); + } uds_funnel_queue_put(context->zone->timed_out_complete, &context->queue_entry); } @@ -2616,6 +2617,7 @@ void vdo_drain_hash_zones(struct hash_zones *zones, struct vdo_completion *paren } static void launch_dedupe_state_change(struct hash_zones *zones) + __must_hold(&zones->lock) { /* ASSERTION: We enter with the lock held. */ if (zones->changing || !vdo_is_state_normal(&zones->state)) @@ -3056,9 +3058,10 @@ int vdo_add_dedupe_index_sysfs(struct hash_zones *zones) int result = kobject_add(&zones->dedupe_directory, &zones->completion.vdo->vdo_directory, "dedupe"); - if (result == 0) + if (result == 0) { vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + } return result; } From 571eff3969a58cce63b02acda4462aa0ca12e116 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 23:57:10 -0500 Subject: [PATCH 0632/1038] dm vdo: cleanup style for comments in structs Use /* ... */ rather than /** ... */ if for no other reason than syntax highlighting is improved (at least for me, in emacs: comments are now red, code is yellow. Previously comments were also yellow). Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.h | 12 +- drivers/md/dm-vdo/constants.h | 38 +++--- drivers/md/dm-vdo/encodings.h | 8 +- drivers/md/dm-vdo/flush.c | 28 ++-- drivers/md/dm-vdo/recovery-journal.h | 22 +-- drivers/md/dm-vdo/statistics.h | 194 +++++++++++++-------------- drivers/md/dm-vdo/vdo.h | 8 +- drivers/md/dm-vdo/vio.c | 16 +-- 8 files changed, 163 insertions(+), 163 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h index b662c318c2ea..39a13039e4a3 100644 --- a/drivers/md/dm-vdo/block-map.h +++ b/drivers/md/dm-vdo/block-map.h @@ -216,17 +216,17 @@ enum block_map_page_type { typedef struct list_head dirty_era_t[2]; struct dirty_lists { - /** The number of periods after which an element will be expired */ + /* The number of periods after which an element will be expired */ block_count_t maximum_age; - /** The oldest period which has unexpired elements */ + /* The oldest period which has unexpired elements */ sequence_number_t oldest_period; - /** One more than the current period */ + /* One more than the current period */ sequence_number_t next_period; - /** The offset in the array of lists of the oldest period */ + /* The offset in the array of lists of the oldest period */ block_count_t offset; - /** Expired pages */ + /* Expired pages */ dirty_era_t expired; - /** The lists of dirty pages */ + /* The lists of dirty pages */ dirty_era_t eras[]; }; diff --git a/drivers/md/dm-vdo/constants.h b/drivers/md/dm-vdo/constants.h index 24237bc29f87..a8c4d6e24b38 100644 --- a/drivers/md/dm-vdo/constants.h +++ b/drivers/md/dm-vdo/constants.h @@ -17,10 +17,10 @@ enum { */ VDO_BIO_ROTATION_INTERVAL_LIMIT = 1024, - /** The number of entries on a block map page */ + /* The number of entries on a block map page */ VDO_BLOCK_MAP_ENTRIES_PER_PAGE = 812, - /** The origin of the flat portion of the block map */ + /* The origin of the flat portion of the block map */ VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN = 1, /* @@ -29,22 +29,22 @@ enum { */ VDO_BLOCK_MAP_TREE_HEIGHT = 5, - /** The default number of bio submission queues. */ + /* The default number of bio submission queues. */ DEFAULT_VDO_BIO_SUBMIT_QUEUE_COUNT = 4, - /** The number of contiguous PBNs to be submitted to a single bio queue. */ + /* The number of contiguous PBNs to be submitted to a single bio queue. */ DEFAULT_VDO_BIO_SUBMIT_QUEUE_ROTATE_INTERVAL = 64, - /** The number of trees in the arboreal block map */ + /* The number of trees in the arboreal block map */ DEFAULT_VDO_BLOCK_MAP_TREE_ROOT_COUNT = 60, - /** The default size of the recovery journal, in blocks */ + /* The default size of the recovery journal, in blocks */ DEFAULT_VDO_RECOVERY_JOURNAL_SIZE = 32 * 1024, - /** The default size of each slab journal, in blocks */ + /* The default size of each slab journal, in blocks */ DEFAULT_VDO_SLAB_JOURNAL_SIZE = 224, - /** Unit test minimum */ + /* Unit test minimum */ MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2, /* @@ -54,16 +54,16 @@ enum { */ VDO_LOCK_MAP_CAPACITY = 10000, - /** The maximum number of logical zones */ + /* The maximum number of logical zones */ MAX_VDO_LOGICAL_ZONES = 60, - /** The maximum number of physical zones */ + /* The maximum number of physical zones */ MAX_VDO_PHYSICAL_ZONES = 16, - /** The base-2 logarithm of the maximum blocks in one slab */ + /* The base-2 logarithm of the maximum blocks in one slab */ MAX_VDO_SLAB_BITS = 23, - /** The maximum number of slabs the slab depot supports */ + /* The maximum number of slabs the slab depot supports */ MAX_VDO_SLABS = 8192, /* @@ -71,25 +71,25 @@ enum { */ MAXIMUM_SIMULTANEOUS_VDO_BLOCK_MAP_RESTORATION_READS = 1024, - /** The maximum number of entries in the slab summary */ + /* The maximum number of entries in the slab summary */ MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES = MAX_VDO_SLABS * MAX_VDO_PHYSICAL_ZONES, - /** The maximum number of total threads in a VDO thread configuration. */ + /* The maximum number of total threads in a VDO thread configuration. */ MAXIMUM_VDO_THREADS = 100, - /** The maximum number of VIOs in the system at once */ + /* The maximum number of VIOs in the system at once */ MAXIMUM_VDO_USER_VIOS = 2048, - /** The only physical block size supported by VDO */ + /* The only physical block size supported by VDO */ VDO_BLOCK_SIZE = 4096, - /** The number of sectors per block */ + /* The number of sectors per block */ VDO_SECTORS_PER_BLOCK = (VDO_BLOCK_SIZE >> SECTOR_SHIFT), - /** The size of a sector that will not be torn */ + /* The size of a sector that will not be torn */ VDO_SECTOR_SIZE = 512, - /** The physical block number reserved for storing the zero block */ + /* The physical block number reserved for storing the zero block */ VDO_ZERO_BLOCK = 0, }; diff --git a/drivers/md/dm-vdo/encodings.h b/drivers/md/dm-vdo/encodings.h index 260822a3958a..ba3db9867f4a 100644 --- a/drivers/md/dm-vdo/encodings.h +++ b/drivers/md/dm-vdo/encodings.h @@ -155,7 +155,7 @@ struct block_map_page_header { __le64 nonce; __le64 pbn; - /** May be non-zero on disk */ + /* May be non-zero on disk */ u8 unused_long_word[8]; /* Whether this page has been written twice to disk */ @@ -197,11 +197,11 @@ extern const struct header VDO_BLOCK_MAP_HEADER_2_0; /* The state of the recovery journal as encoded in the VDO super block. */ struct recovery_journal_state_7_0 { - /** Sequence number to start the journal */ + /* Sequence number to start the journal */ sequence_number_t journal_start; - /** Number of logical blocks used by VDO */ + /* Number of logical blocks used by VDO */ block_count_t logical_blocks_used; - /** Number of block map pages allocated */ + /* Number of block map pages allocated */ block_count_t block_map_data_blocks; } __packed; diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index a6eeb425d721..330b18715027 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -22,33 +22,33 @@ struct flusher { struct vdo_completion completion; - /** The vdo to which this flusher belongs */ + /* The vdo to which this flusher belongs */ struct vdo *vdo; - /** The administrative state of the flusher */ + /* The administrative state of the flusher */ struct admin_state state; - /** The current flush generation of the vdo */ + /* The current flush generation of the vdo */ sequence_number_t flush_generation; - /** The first unacknowledged flush generation */ + /* The first unacknowledged flush generation */ sequence_number_t first_unacknowledged_generation; - /** The queue of flush requests waiting to notify other threads */ + /* The queue of flush requests waiting to notify other threads */ struct vdo_wait_queue notifiers; - /** The queue of flush requests waiting for VIOs to complete */ + /* The queue of flush requests waiting for VIOs to complete */ struct vdo_wait_queue pending_flushes; - /** The flush generation for which notifications are being sent */ + /* The flush generation for which notifications are being sent */ sequence_number_t notify_generation; - /** The logical zone to notify next */ + /* The logical zone to notify next */ struct logical_zone *logical_zone_to_notify; - /** The ID of the thread on which flush requests should be made */ + /* The ID of the thread on which flush requests should be made */ thread_id_t thread_id; - /** The pool of flush requests */ + /* The pool of flush requests */ mempool_t *flush_pool; - /** Bios waiting for a flush request to become available */ + /* Bios waiting for a flush request to become available */ struct bio_list waiting_flush_bios; - /** The lock to protect the previous fields */ + /* The lock to protect the previous fields */ spinlock_t lock; - /** The rotor for selecting the bio queue for submitting flush bios */ + /* The rotor for selecting the bio queue for submitting flush bios */ zone_count_t bio_queue_rotor; - /** The number of flushes submitted to the current bio queue */ + /* The number of flushes submitted to the current bio queue */ int flush_count; }; diff --git a/drivers/md/dm-vdo/recovery-journal.h b/drivers/md/dm-vdo/recovery-journal.h index d78c6c7da4ea..899071173015 100644 --- a/drivers/md/dm-vdo/recovery-journal.h +++ b/drivers/md/dm-vdo/recovery-journal.h @@ -89,27 +89,27 @@ enum vdo_zone_type { }; struct lock_counter { - /** The completion for notifying the owner of a lock release */ + /* The completion for notifying the owner of a lock release */ struct vdo_completion completion; - /** The number of logical zones which may hold locks */ + /* The number of logical zones which may hold locks */ zone_count_t logical_zones; - /** The number of physical zones which may hold locks */ + /* The number of physical zones which may hold locks */ zone_count_t physical_zones; - /** The number of locks */ + /* The number of locks */ block_count_t locks; - /** Whether the lock release notification is in flight */ + /* Whether the lock release notification is in flight */ atomic_t state; - /** The number of logical zones which hold each lock */ + /* The number of logical zones which hold each lock */ atomic_t *logical_zone_counts; - /** The number of physical zones which hold each lock */ + /* The number of physical zones which hold each lock */ atomic_t *physical_zone_counts; - /** The per-lock counts for the journal zone */ + /* The per-lock counts for the journal zone */ u16 *journal_counters; - /** The per-lock decrement counts for the journal zone */ + /* The per-lock decrement counts for the journal zone */ atomic_t *journal_decrement_counts; - /** The per-zone, per-lock reference counts for logical zones */ + /* The per-zone, per-lock reference counts for logical zones */ u16 *logical_counters; - /** The per-zone, per-lock reference counts for physical zones */ + /* The per-zone, per-lock reference counts for physical zones */ u16 *physical_counters; }; diff --git a/drivers/md/dm-vdo/statistics.h b/drivers/md/dm-vdo/statistics.h index da8b6a7fe24f..c88a75dffba3 100644 --- a/drivers/md/dm-vdo/statistics.h +++ b/drivers/md/dm-vdo/statistics.h @@ -13,11 +13,11 @@ enum { }; struct block_allocator_statistics { - /** The total number of slabs from which blocks may be allocated */ + /* The total number of slabs from which blocks may be allocated */ u64 slab_count; - /** The total number of slabs from which blocks have ever been allocated */ + /* The total number of slabs from which blocks have ever been allocated */ u64 slabs_opened; - /** The number of times since loading that a slab has been re-opened */ + /* The number of times since loading that a slab has been re-opened */ u64 slabs_reopened; }; @@ -29,235 +29,235 @@ struct block_allocator_statistics { * reporting purposes */ struct commit_statistics { - /** The total number of items on which processing has started */ + /* The total number of items on which processing has started */ u64 started; - /** The total number of items for which a write operation has been issued */ + /* The total number of items for which a write operation has been issued */ u64 written; - /** The total number of items for which a write operation has completed */ + /* The total number of items for which a write operation has completed */ u64 committed; }; /** Counters for events in the recovery journal */ struct recovery_journal_statistics { - /** Number of times the on-disk journal was full */ + /* Number of times the on-disk journal was full */ u64 disk_full; - /** Number of times the recovery journal requested slab journal commits. */ + /* Number of times the recovery journal requested slab journal commits. */ u64 slab_journal_commits_requested; - /** Write/Commit totals for individual journal entries */ + /* Write/Commit totals for individual journal entries */ struct commit_statistics entries; - /** Write/Commit totals for journal blocks */ + /* Write/Commit totals for journal blocks */ struct commit_statistics blocks; }; /** The statistics for the compressed block packer. */ struct packer_statistics { - /** Number of compressed data items written since startup */ + /* Number of compressed data items written since startup */ u64 compressed_fragments_written; - /** Number of blocks containing compressed items written since startup */ + /* Number of blocks containing compressed items written since startup */ u64 compressed_blocks_written; - /** Number of VIOs that are pending in the packer */ + /* Number of VIOs that are pending in the packer */ u64 compressed_fragments_in_packer; }; /** The statistics for the slab journals. */ struct slab_journal_statistics { - /** Number of times the on-disk journal was full */ + /* Number of times the on-disk journal was full */ u64 disk_full_count; - /** Number of times an entry was added over the flush threshold */ + /* Number of times an entry was added over the flush threshold */ u64 flush_count; - /** Number of times an entry was added over the block threshold */ + /* Number of times an entry was added over the block threshold */ u64 blocked_count; - /** Number of times a tail block was written */ + /* Number of times a tail block was written */ u64 blocks_written; - /** Number of times we had to wait for the tail to write */ + /* Number of times we had to wait for the tail to write */ u64 tail_busy_count; }; /** The statistics for the slab summary. */ struct slab_summary_statistics { - /** Number of blocks written */ + /* Number of blocks written */ u64 blocks_written; }; /** The statistics for the reference counts. */ struct ref_counts_statistics { - /** Number of reference blocks written */ + /* Number of reference blocks written */ u64 blocks_written; }; /** The statistics for the block map. */ struct block_map_statistics { - /** number of dirty (resident) pages */ + /* number of dirty (resident) pages */ u32 dirty_pages; - /** number of clean (resident) pages */ + /* number of clean (resident) pages */ u32 clean_pages; - /** number of free pages */ + /* number of free pages */ u32 free_pages; - /** number of pages in failed state */ + /* number of pages in failed state */ u32 failed_pages; - /** number of pages incoming */ + /* number of pages incoming */ u32 incoming_pages; - /** number of pages outgoing */ + /* number of pages outgoing */ u32 outgoing_pages; - /** how many times free page not avail */ + /* how many times free page not avail */ u32 cache_pressure; - /** number of get_vdo_page() calls for read */ + /* number of get_vdo_page() calls for read */ u64 read_count; - /** number of get_vdo_page() calls for write */ + /* number of get_vdo_page() calls for write */ u64 write_count; - /** number of times pages failed to read */ + /* number of times pages failed to read */ u64 failed_reads; - /** number of times pages failed to write */ + /* number of times pages failed to write */ u64 failed_writes; - /** number of gets that are reclaimed */ + /* number of gets that are reclaimed */ u64 reclaimed; - /** number of gets for outgoing pages */ + /* number of gets for outgoing pages */ u64 read_outgoing; - /** number of gets that were already there */ + /* number of gets that were already there */ u64 found_in_cache; - /** number of gets requiring discard */ + /* number of gets requiring discard */ u64 discard_required; - /** number of gets enqueued for their page */ + /* number of gets enqueued for their page */ u64 wait_for_page; - /** number of gets that have to fetch */ + /* number of gets that have to fetch */ u64 fetch_required; - /** number of page fetches */ + /* number of page fetches */ u64 pages_loaded; - /** number of page saves */ + /* number of page saves */ u64 pages_saved; - /** the number of flushes issued */ + /* the number of flushes issued */ u64 flush_count; }; /** The dedupe statistics from hash locks */ struct hash_lock_statistics { - /** Number of times the UDS advice proved correct */ + /* Number of times the UDS advice proved correct */ u64 dedupe_advice_valid; - /** Number of times the UDS advice proved incorrect */ + /* Number of times the UDS advice proved incorrect */ u64 dedupe_advice_stale; - /** Number of writes with the same data as another in-flight write */ + /* Number of writes with the same data as another in-flight write */ u64 concurrent_data_matches; - /** Number of writes whose hash collided with an in-flight write */ + /* Number of writes whose hash collided with an in-flight write */ u64 concurrent_hash_collisions; - /** Current number of dedupe queries that are in flight */ + /* Current number of dedupe queries that are in flight */ u32 curr_dedupe_queries; }; /** Counts of error conditions in VDO. */ struct error_statistics { - /** number of times VDO got an invalid dedupe advice PBN from UDS */ + /* number of times VDO got an invalid dedupe advice PBN from UDS */ u64 invalid_advice_pbn_count; - /** number of times a VIO completed with a VDO_NO_SPACE error */ + /* number of times a VIO completed with a VDO_NO_SPACE error */ u64 no_space_error_count; - /** number of times a VIO completed with a VDO_READ_ONLY error */ + /* number of times a VIO completed with a VDO_READ_ONLY error */ u64 read_only_error_count; }; struct bio_stats { - /** Number of REQ_OP_READ bios */ + /* Number of REQ_OP_READ bios */ u64 read; - /** Number of REQ_OP_WRITE bios with data */ + /* Number of REQ_OP_WRITE bios with data */ u64 write; - /** Number of bios tagged with REQ_PREFLUSH and containing no data */ + /* Number of bios tagged with REQ_PREFLUSH and containing no data */ u64 empty_flush; - /** Number of REQ_OP_DISCARD bios */ + /* Number of REQ_OP_DISCARD bios */ u64 discard; - /** Number of bios tagged with REQ_PREFLUSH */ + /* Number of bios tagged with REQ_PREFLUSH */ u64 flush; - /** Number of bios tagged with REQ_FUA */ + /* Number of bios tagged with REQ_FUA */ u64 fua; }; struct memory_usage { - /** Tracked bytes currently allocated. */ + /* Tracked bytes currently allocated. */ u64 bytes_used; - /** Maximum tracked bytes allocated. */ + /* Maximum tracked bytes allocated. */ u64 peak_bytes_used; }; /** UDS index statistics */ struct index_statistics { - /** Number of records stored in the index */ + /* Number of records stored in the index */ u64 entries_indexed; - /** Number of post calls that found an existing entry */ + /* Number of post calls that found an existing entry */ u64 posts_found; - /** Number of post calls that added a new entry */ + /* Number of post calls that added a new entry */ u64 posts_not_found; - /** Number of query calls that found an existing entry */ + /* Number of query calls that found an existing entry */ u64 queries_found; - /** Number of query calls that added a new entry */ + /* Number of query calls that added a new entry */ u64 queries_not_found; - /** Number of update calls that found an existing entry */ + /* Number of update calls that found an existing entry */ u64 updates_found; - /** Number of update calls that added a new entry */ + /* Number of update calls that added a new entry */ u64 updates_not_found; - /** Number of entries discarded */ + /* Number of entries discarded */ u64 entries_discarded; }; /** The statistics of the vdo service. */ struct vdo_statistics { u32 version; - /** Number of blocks used for data */ + /* Number of blocks used for data */ u64 data_blocks_used; - /** Number of blocks used for VDO metadata */ + /* Number of blocks used for VDO metadata */ u64 overhead_blocks_used; - /** Number of logical blocks that are currently mapped to physical blocks */ + /* Number of logical blocks that are currently mapped to physical blocks */ u64 logical_blocks_used; - /** number of physical blocks */ + /* number of physical blocks */ block_count_t physical_blocks; - /** number of logical blocks */ + /* number of logical blocks */ block_count_t logical_blocks; - /** Size of the block map page cache, in bytes */ + /* Size of the block map page cache, in bytes */ u64 block_map_cache_size; - /** The physical block size */ + /* The physical block size */ u64 block_size; - /** Number of times the VDO has successfully recovered */ + /* Number of times the VDO has successfully recovered */ u64 complete_recoveries; - /** Number of times the VDO has recovered from read-only mode */ + /* Number of times the VDO has recovered from read-only mode */ u64 read_only_recoveries; - /** String describing the operating mode of the VDO */ + /* String describing the operating mode of the VDO */ char mode[15]; - /** Whether the VDO is in recovery mode */ + /* Whether the VDO is in recovery mode */ bool in_recovery_mode; - /** What percentage of recovery mode work has been completed */ + /* What percentage of recovery mode work has been completed */ u8 recovery_percentage; - /** The statistics for the compressed block packer */ + /* The statistics for the compressed block packer */ struct packer_statistics packer; - /** Counters for events in the block allocator */ + /* Counters for events in the block allocator */ struct block_allocator_statistics allocator; - /** Counters for events in the recovery journal */ + /* Counters for events in the recovery journal */ struct recovery_journal_statistics journal; - /** The statistics for the slab journals */ + /* The statistics for the slab journals */ struct slab_journal_statistics slab_journal; - /** The statistics for the slab summary */ + /* The statistics for the slab summary */ struct slab_summary_statistics slab_summary; - /** The statistics for the reference counts */ + /* The statistics for the reference counts */ struct ref_counts_statistics ref_counts; - /** The statistics for the block map */ + /* The statistics for the block map */ struct block_map_statistics block_map; - /** The dedupe statistics from hash locks */ + /* The dedupe statistics from hash locks */ struct hash_lock_statistics hash_lock; - /** Counts of error conditions */ + /* Counts of error conditions */ struct error_statistics errors; - /** The VDO instance */ + /* The VDO instance */ u32 instance; - /** Current number of active VIOs */ + /* Current number of active VIOs */ u32 current_vios_in_progress; - /** Maximum number of active VIOs */ + /* Maximum number of active VIOs */ u32 max_vios; - /** Number of times the UDS index was too slow in responding */ + /* Number of times the UDS index was too slow in responding */ u64 dedupe_advice_timeouts; - /** Number of flush requests submitted to the storage device */ + /* Number of flush requests submitted to the storage device */ u64 flush_out; - /** Logical block size */ + /* Logical block size */ u64 logical_block_size; - /** Bios submitted into VDO from above */ + /* Bios submitted into VDO from above */ struct bio_stats bios_in; struct bio_stats bios_in_partial; - /** Bios submitted onward for user data */ + /* Bios submitted onward for user data */ struct bio_stats bios_out; - /** Bios submitted onward for metadata */ + /* Bios submitted onward for metadata */ struct bio_stats bios_meta; struct bio_stats bios_journal; struct bio_stats bios_page_cache; @@ -267,11 +267,11 @@ struct vdo_statistics { struct bio_stats bios_page_cache_completed; struct bio_stats bios_acknowledged; struct bio_stats bios_acknowledged_partial; - /** Current number of bios in progress */ + /* Current number of bios in progress */ struct bio_stats bios_in_progress; - /** Memory usage stats. */ + /* Memory usage stats. */ struct memory_usage memory_usage; - /** The statistics for the UDS index */ + /* The statistics for the UDS index */ struct index_statistics index; }; diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h index 35d603066529..772317e6db52 100644 --- a/drivers/md/dm-vdo/vdo.h +++ b/drivers/md/dm-vdo/vdo.h @@ -25,13 +25,13 @@ #include "uds.h" enum notifier_state { - /** Notifications are allowed but not in progress */ + /* Notifications are allowed but not in progress */ MAY_NOTIFY, - /** A notification is in progress */ + /* A notification is in progress */ NOTIFYING, - /** Notifications are not allowed */ + /* Notifications are not allowed */ MAY_NOT_NOTIFY, - /** A notification has completed */ + /* A notification has completed */ NOTIFIED, }; diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index f9c54ec56176..eb6838ddabbb 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -20,21 +20,21 @@ /* A vio_pool is a collection of preallocated vios. */ struct vio_pool { - /** The number of objects managed by the pool */ + /* The number of objects managed by the pool */ size_t size; - /** The list of objects which are available */ + /* The list of objects which are available */ struct list_head available; - /** The queue of requestors waiting for objects from the pool */ + /* The queue of requestors waiting for objects from the pool */ struct vdo_wait_queue waiting; - /** The number of objects currently in use */ + /* The number of objects currently in use */ size_t busy_count; - /** The list of objects which are in use */ + /* The list of objects which are in use */ struct list_head busy; - /** The ID of the thread on which this pool may be used */ + /* The ID of the thread on which this pool may be used */ thread_id_t thread_id; - /** The buffer backing the pool's vios */ + /* The buffer backing the pool's vios */ char *buffer; - /** The pool entries */ + /* The pool entries */ struct pooled_vio vios[]; }; From dea93aab18341d6d96a49e840e5605cebca7e8ea Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 23:57:11 -0500 Subject: [PATCH 0633/1038] dm vdo chapter_index: fix a few small nits Add missing braces and raise one function arg up a line to eliminate line wrap. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/chapter-index.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo/chapter-index.c b/drivers/md/dm-vdo/chapter-index.c index 7285a842e888..363991d56218 100644 --- a/drivers/md/dm-vdo/chapter-index.c +++ b/drivers/md/dm-vdo/chapter-index.c @@ -19,8 +19,7 @@ int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, size_t memory_size; struct open_chapter_index *index; - result = uds_allocate(1, struct open_chapter_index, "open chapter index", - &index); + result = uds_allocate(1, struct open_chapter_index, "open chapter index", &index); if (result != UDS_SUCCESS) return result; @@ -197,9 +196,10 @@ int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, } while (!entry.at_end); } - if (removals > 0) + if (removals > 0) { uds_log_warning("To avoid chapter index page overflow in chapter %llu, %u entries were removed from the chapter index", (unsigned long long) chapter_number, removals); + } return UDS_SUCCESS; } @@ -249,13 +249,14 @@ int uds_validate_chapter_index_page(const struct delta_index_page *index_page, /* Also make sure that the record page field contains a plausible value. */ if (uds_get_delta_entry_value(&entry) >= - geometry->record_pages_per_chapter) + geometry->record_pages_per_chapter) { /* * Do not log this as an error. It happens in normal operation when * we are doing a rebuild but haven't written the entire volume * once. */ return UDS_CORRUPT_DATA; + } } } return UDS_SUCCESS; From 5581a43d302fb860f88f649c1a48a1c65c566eb0 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 23:57:12 -0500 Subject: [PATCH 0634/1038] dm vdo delta-index: fix various small nits Fix some needless line wrapping (given surrounding context), missing braces and some stale or incorrect references to data structure or function name. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/delta-index.c | 23 ++++++++++------------- drivers/md/dm-vdo/delta-index.h | 2 +- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/drivers/md/dm-vdo/delta-index.c b/drivers/md/dm-vdo/delta-index.c index 0322552cdbd8..6306777bb202 100644 --- a/drivers/md/dm-vdo/delta-index.c +++ b/drivers/md/dm-vdo/delta-index.c @@ -47,7 +47,7 @@ * * The delta in each entry is encoded with a variable-length Huffman code to minimize the memory * used by small deltas. The Huffman code is specified by three parameters, which can be computed - * from the desired mean delta when the index is full. (See compute_coding constants() for + * from the desired mean delta when the index is full. (See compute_coding_constants() for * details.) * * The bit field utilities used to read and write delta entries assume that it is possible to read @@ -294,8 +294,7 @@ void uds_reset_delta_index(const struct delta_index *delta_index) * - The next INCR values code using MINBITS+2 bits. * - (and so on). */ -static void compute_coding_constants(u32 mean_delta, u16 *min_bits, u32 *min_keys, - u32 *incr_keys) +static void compute_coding_constants(u32 mean_delta, u16 *min_bits, u32 *min_keys, u32 *incr_keys) { /* * We want to compute the rounded value of log(2) * mean_delta. Since we cannot always use @@ -617,8 +616,7 @@ static inline void set_zero(u8 *memory, u64 offset, u32 size) * Move several bits from a higher to a lower address, moving the lower addressed bits first. The * size and memory offsets are measured in bits. */ -static void move_bits_down(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, - u32 size) +static void move_bits_down(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, u32 size) { const u8 *source; u8 *destination; @@ -658,8 +656,7 @@ static void move_bits_down(const u8 *from, u64 from_offset, u8 *to, u64 to_offse * Move several bits from a lower to a higher address, moving the higher addressed bits first. The * size and memory offsets are measured in bits. */ -static void move_bits_up(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, - u32 size) +static void move_bits_up(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, u32 size) { const u8 *source; u8 *destination; @@ -1006,8 +1003,7 @@ static int restore_delta_list_to_zone(struct delta_zone *delta_zone, return UDS_SUCCESS; } -static int restore_delta_list_data(struct delta_index *delta_index, - unsigned int load_zone, +static int restore_delta_list_data(struct delta_index *delta_index, unsigned int load_zone, struct buffered_reader *buffered_reader, u8 *data) { int result; @@ -1016,9 +1012,10 @@ static int restore_delta_list_data(struct delta_index *delta_index, unsigned int new_zone; result = uds_read_from_buffered_reader(buffered_reader, buffer, sizeof(buffer)); - if (result != UDS_SUCCESS) + if (result != UDS_SUCCESS) { return uds_log_warning_strerror(result, "failed to read delta list data"); + } save_info = (struct delta_list_save_info) { .tag = buffer[0], @@ -1028,9 +1025,10 @@ static int restore_delta_list_data(struct delta_index *delta_index, }; if ((save_info.bit_offset >= BITS_PER_BYTE) || - (save_info.byte_count > DELTA_LIST_MAX_BYTE_COUNT)) + (save_info.byte_count > DELTA_LIST_MAX_BYTE_COUNT)) { return uds_log_warning_strerror(UDS_CORRUPT_DATA, "corrupt delta list data"); + } /* Make sure the data is intended for this delta index. */ if (save_info.tag != delta_index->tag) @@ -1422,8 +1420,7 @@ static void set_delta(struct delta_index_entry *delta_entry, u32 delta) { const struct delta_zone *delta_zone = delta_entry->delta_zone; u32 key_bits = (delta_zone->min_bits + - ((delta_zone->incr_keys - - delta_zone->min_keys + delta) / + ((delta_zone->incr_keys - delta_zone->min_keys + delta) / delta_zone->incr_keys)); delta_entry->delta = delta; diff --git a/drivers/md/dm-vdo/delta-index.h b/drivers/md/dm-vdo/delta-index.h index 046290b7ebe7..b3b38fb440bf 100644 --- a/drivers/md/dm-vdo/delta-index.h +++ b/drivers/md/dm-vdo/delta-index.h @@ -113,7 +113,7 @@ struct delta_index { */ struct delta_index_page { struct delta_index delta_index; - /* These values are loaded from the DeltaPageHeader */ + /* These values are loaded from the delta_page_header */ u32 lowest_list_number; u32 highest_list_number; u64 virtual_chapter_number; From cae3816d9933dc8041e1f3e14f10df34707f0b06 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 14 Feb 2024 10:25:39 -0500 Subject: [PATCH 0635/1038] dm vdo: tweak wait_for_completion_interruptible callers Update uds_join_threads to delay in wait_for_completion_interruptible loop. And cleanup style nits in perform_admin_operation(). Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dm-vdo-target.c | 5 +++-- drivers/md/dm-vdo/uds-threads.c | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index e2e60a29e873..e754b9e30cab 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -1235,9 +1235,10 @@ static int perform_admin_operation(struct vdo *vdo, u32 starting_phase, * Using the "interruptible" interface means that Linux will not log a message when we wait * for more than 120 seconds. */ - while (wait_for_completion_interruptible(&admin->callback_sync) != 0) - /* * However, if we get a signal in a user-mode process, we could spin... */ + while (wait_for_completion_interruptible(&admin->callback_sync)) { + /* However, if we get a signal in a user-mode process, we could spin... */ fsleep(1000); + } result = admin->completion.result; /* pairs with implicit barrier in cmpxchg above */ diff --git a/drivers/md/dm-vdo/uds-threads.c b/drivers/md/dm-vdo/uds-threads.c index e7524617f9f8..769c783e342a 100644 --- a/drivers/md/dm-vdo/uds-threads.c +++ b/drivers/md/dm-vdo/uds-threads.c @@ -6,6 +6,7 @@ #include "uds-threads.h" #include +#include #include #include #include @@ -125,9 +126,8 @@ int uds_create_thread(void (*thread_function)(void *), void *thread_data, int uds_join_threads(struct thread *thread) { - while (wait_for_completion_interruptible(&thread->thread_done) != 0) - /* empty loop */ - ; + while (wait_for_completion_interruptible(&thread->thread_done)) + fsleep(1000); mutex_lock(&thread_mutex); hlist_del(&thread->thread_links); From 318a9ce59bfaf47d23b7e80eb8c2697281108a5f Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 14 Feb 2024 18:25:13 -0500 Subject: [PATCH 0636/1038] dm vdo logger: switch UDS_LOG_NOTICE to be alias for UDS_LOG_INFO Prepare to bring VDO's logging closer to DM's logging by eliminating support for KERN_NOTICE log level (DM hasn't ever had a need for it). Only one message in index-session.c used UDS_LOG_NOTICE, convert it to log with uds_log_info(). Signed-off-by: Mike Snitzer Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/index-session.c | 2 +- drivers/md/dm-vdo/logger.c | 2 -- drivers/md/dm-vdo/logger.h | 5 ----- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c index 9b5d2cacfffd..7afc19748712 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/index-session.c @@ -378,7 +378,7 @@ int uds_open_index(enum uds_open_index_type open_type, session->parameters = *parameters; format_dev_t(name, parameters->bdev->bd_dev); - uds_log_notice("%s: %s", get_open_type_string(open_type), name); + uds_log_info("%s: %s", get_open_type_string(open_type), name); result = initialize_index_session(session, open_type); if (result != UDS_SUCCESS) diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index fe632f10b5d9..322224a07cf5 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -124,8 +124,6 @@ static void emit_log_message_to_kernel(int priority, const char *fmt, ...) printk(KERN_WARNING "%pV", &vaf); break; case UDS_LOG_NOTICE: - printk(KERN_NOTICE "%pV", &vaf); - break; case UDS_LOG_INFO: printk(KERN_INFO "%pV", &vaf); break; diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h index 246aa43db9ee..09d448ff9ca9 100644 --- a/drivers/md/dm-vdo/logger.h +++ b/drivers/md/dm-vdo/logger.h @@ -74,9 +74,6 @@ int uds_vlog_strerror(int priority, int errnum, const char *module, const char * #define uds_log_info_strerror(errnum, ...) \ uds_log_strerror(UDS_LOG_INFO, errnum, __VA_ARGS__) -#define uds_log_notice_strerror(errnum, ...) \ - uds_log_strerror(UDS_LOG_NOTICE, errnum, __VA_ARGS__) - #define uds_log_warning_strerror(errnum, ...) \ uds_log_strerror(UDS_LOG_WARNING, errnum, __VA_ARGS__) @@ -93,8 +90,6 @@ void __uds_log_message(int priority, const char *module, const char *format, ... #define uds_log_info(...) uds_log_message(UDS_LOG_INFO, __VA_ARGS__) -#define uds_log_notice(...) uds_log_message(UDS_LOG_NOTICE, __VA_ARGS__) - #define uds_log_warning(...) uds_log_message(UDS_LOG_WARNING, __VA_ARGS__) #define uds_log_error(...) uds_log_message(UDS_LOG_ERR, __VA_ARGS__) From 1e00d57694bc2f050f73b632ce3822fe28af63d2 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 14 Feb 2024 18:25:14 -0500 Subject: [PATCH 0637/1038] dm vdo logger: update logging to start with "device-mapper: vdo" Stops short of actually using DM's various logging macros (e.g. DMERR, DMINFO, etc) because VDO's logger isn't quite compatible with them. Also switch emit_log_message_to_kernel() from open-coding printk with log-level to using corresponding pr_ macro. Signed-off-by: Mike Snitzer Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/logger.c | 10 +++++----- drivers/md/dm-vdo/logger.h | 8 +++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 322224a07cf5..1efbf8d52f2c 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -115,20 +115,20 @@ static void emit_log_message_to_kernel(int priority, const char *fmt, ...) case UDS_LOG_EMERG: case UDS_LOG_ALERT: case UDS_LOG_CRIT: - printk(KERN_CRIT "%pV", &vaf); + pr_crit("%pV", &vaf); break; case UDS_LOG_ERR: - printk(KERN_ERR "%pV", &vaf); + pr_err("%pV", &vaf); break; case UDS_LOG_WARNING: - printk(KERN_WARNING "%pV", &vaf); + pr_warn("%pV", &vaf); break; case UDS_LOG_NOTICE: case UDS_LOG_INFO: - printk(KERN_INFO "%pV", &vaf); + pr_info("%pV", &vaf); break; case UDS_LOG_DEBUG: - printk(KERN_DEBUG "%pV", &vaf); + pr_debug("%pV", &vaf); break; default: printk(KERN_DEFAULT "%pV", &vaf); diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h index 09d448ff9ca9..4e2f18042ba7 100644 --- a/drivers/md/dm-vdo/logger.h +++ b/drivers/md/dm-vdo/logger.h @@ -8,6 +8,7 @@ #include #include +#include /* Custom logging utilities for UDS */ @@ -20,11 +21,8 @@ #define UDS_LOG_INFO 6 #define UDS_LOG_DEBUG 7 -#if defined(MODULE) -#define UDS_LOGGING_MODULE_NAME THIS_MODULE->name -#else /* compiled into the kernel */ -#define UDS_LOGGING_MODULE_NAME "vdo" -#endif +#define DM_MSG_PREFIX "vdo" +#define UDS_LOGGING_MODULE_NAME DM_NAME ": " DM_MSG_PREFIX /* Apply a rate limiter to a log method call. */ #define uds_log_ratelimit(log_fn, ...) \ From 6e5f0f6383b4896c7e9b943d84b136149d0f45e9 Mon Sep 17 00:00:00 2001 From: Hongyu Jin Date: Wed, 24 Jan 2024 13:35:53 +0800 Subject: [PATCH 0638/1038] dm io: Support IO priority Some IO will dispatch from kworker with different io_context settings than the submitting task, we may need to specify a priority to avoid losing priority. Add IO priority parameter to dm_io() and update all callers. Co-developed-by: Yibin Ding Signed-off-by: Yibin Ding Signed-off-by: Hongyu Jin Reviewed-by: Eric Biggers Reviewed-by: Mikulas Patocka Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 6 +++--- drivers/md/dm-integrity.c | 12 ++++++------ drivers/md/dm-io.c | 23 +++++++++++++---------- drivers/md/dm-kcopyd.c | 4 ++-- drivers/md/dm-log.c | 4 ++-- drivers/md/dm-raid1.c | 6 +++--- drivers/md/dm-snap-persistent.c | 4 ++-- drivers/md/dm-verity-target.c | 2 +- drivers/md/dm-writecache.c | 8 ++++---- include/linux/dm-io.h | 3 ++- 10 files changed, 38 insertions(+), 34 deletions(-) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 13c65b7e1ed6..f5541b8f6320 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1315,7 +1315,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL); + r = dm_io(&io_req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -2167,7 +2167,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c) if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; - return dm_io(&io_req, 1, &io_reg, NULL); + return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); @@ -2191,7 +2191,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; /* discards are optional */ - return dm_io(&io_req, 1, &io_reg, NULL); + return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 44a1dcc06dd7..c3c666937112 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -555,7 +555,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) } } - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) return r; @@ -1073,7 +1073,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? "reading journal" : "writing journal", r); @@ -1190,7 +1190,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u io_loc.sector = target; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); fn(-1UL, data); @@ -1519,7 +1519,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat fr.io_reg.count = 0, fr.ic = ic; init_completion(&fr.comp); - r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL); + r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); BUG_ON(r); } @@ -1728,7 +1728,7 @@ static void integrity_recheck(struct dm_integrity_io *dio) io_loc.sector = sector; io_loc.count = ic->sectors_per_block; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dio->bi_status = errno_to_blk_status(r); goto free_ret; @@ -2808,7 +2808,7 @@ next_chunk: io_loc.sector = get_data_sector(ic, area, offset); io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, "reading data", r); goto err; diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index f053ce245814..7409490259d1 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -305,7 +305,7 @@ static void km_dp_init(struct dpages *dp, void *data) */ static void do_region(const blk_opf_t opf, unsigned int region, struct dm_io_region *where, struct dpages *dp, - struct io *io) + struct io *io, unsigned short ioprio) { struct bio *bio; struct page *page; @@ -354,6 +354,7 @@ static void do_region(const blk_opf_t opf, unsigned int region, &io->client->bios); bio->bi_iter.bi_sector = where->sector + (where->count - remaining); bio->bi_end_io = endio; + bio->bi_ioprio = ioprio; store_io_and_region_in_bio(bio, io, region); if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) { @@ -383,7 +384,7 @@ static void do_region(const blk_opf_t opf, unsigned int region, static void dispatch_io(blk_opf_t opf, unsigned int num_regions, struct dm_io_region *where, struct dpages *dp, - struct io *io, int sync) + struct io *io, int sync, unsigned short ioprio) { int i; struct dpages old_pages = *dp; @@ -400,7 +401,7 @@ static void dispatch_io(blk_opf_t opf, unsigned int num_regions, for (i = 0; i < num_regions; i++) { *dp = old_pages; if (where[i].count || (opf & REQ_PREFLUSH)) - do_region(opf, i, where + i, dp, io); + do_region(opf, i, where + i, dp, io, ioprio); } /* @@ -425,7 +426,7 @@ static void sync_io_complete(unsigned long error, void *context) static int sync_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, struct dpages *dp, - unsigned long *error_bits) + unsigned long *error_bits, unsigned short ioprio) { struct io *io; struct sync_io sio; @@ -447,7 +448,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(opf, num_regions, where, dp, io, 1); + dispatch_io(opf, num_regions, where, dp, io, 1, ioprio); wait_for_completion_io(&sio.wait); @@ -459,7 +460,8 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, static int async_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, - struct dpages *dp, io_notify_fn fn, void *context) + struct dpages *dp, io_notify_fn fn, void *context, + unsigned short ioprio) { struct io *io; @@ -479,7 +481,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(opf, num_regions, where, dp, io, 0); + dispatch_io(opf, num_regions, where, dp, io, 0, ioprio); return 0; } @@ -521,7 +523,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, } int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *where, unsigned long *sync_error_bits) + struct dm_io_region *where, unsigned long *sync_error_bits, + unsigned short ioprio) { int r; struct dpages dp; @@ -532,11 +535,11 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_opf, &dp, sync_error_bits); + io_req->bi_opf, &dp, sync_error_bits, ioprio); return async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp, io_req->notify.fn, - io_req->notify.context); + io_req->notify.context, ioprio); } EXPORT_SYMBOL(dm_io); diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 36bcfdccae04..6ea75436a433 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job) io_job_start(job->kc->throttle); if (job->op == REQ_OP_READ) - r = dm_io(&io_req, 1, &job->source, NULL); + r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT); else - r = dm_io(&io_req, job->num_dests, job->dests, NULL); + r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT); return r; } diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index f9f84236dfcd..f7f9c2100937 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op) { lc->io_req.bi_opf = op; - return dm_io(&lc->io_req, 1, &lc->header_location, NULL); + return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT); } static int flush_header(struct log_c *lc) @@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc) lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - return dm_io(&lc->io_req, 1, &null_location, NULL); + return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT); } static int read_header(struct log_c *log) diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index ddcb2bc4a617..9511dae5b556 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -278,7 +278,7 @@ static int mirror_flush(struct dm_target *ti) } error_bits = -1; - dm_io(&io_req, ms->nr_mirrors, io, &error_bits); + dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT); if (unlikely(error_bits != 0)) { for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error_bits)) @@ -554,7 +554,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) map_region(&io, m, bio); bio_set_m(bio, m); - BUG_ON(dm_io(&io_req, 1, &io, NULL)); + BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT)); } static inline int region_in_sync(struct mirror_set *ms, region_t region, @@ -681,7 +681,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) */ bio_set_m(bio, get_default_mirror(ms)); - BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL)); + BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT)); } static void do_writes(struct mirror_set *ms, struct bio_list *writes) diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index 15649921f2a9..568d10842b1f 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work) { struct mdata_req *req = container_of(work, struct mdata_req, work); - req->result = dm_io(req->io_req, 1, req->where, NULL); + req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT); } /* @@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf, struct mdata_req req; if (!metadata) - return dm_io(&io_req, 1, &where, NULL); + return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT); req.where = &where; req.io_req = &io_req; diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 06c8b637849c..f55fbbe5d22f 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -511,7 +511,7 @@ static int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, io_loc.bdev = v->data_dev->bdev; io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) goto free_ret; diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index b463c28c39ad..01ab141bc51e 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -531,7 +531,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) req.notify.context = &endio; /* writing via async dm-io (implied by notify.fn above) won't return an error */ - (void) dm_io(&req, 1, ®ion, NULL); + (void) dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); i = j; } @@ -568,7 +568,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) req.notify.fn = NULL; req.notify.context = NULL; - r = dm_io(&req, 1, ®ion, NULL); + r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error writing superblock"); } @@ -596,7 +596,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) req.client = wc->dm_io; req.notify.fn = NULL; - r = dm_io(&req, 1, ®ion, NULL); + r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error flushing metadata: %d", r); } @@ -990,7 +990,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors req.client = wc->dm_io; req.notify.fn = NULL; - return dm_io(&req, 1, ®ion, NULL); + return dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); } static void writecache_resume(struct dm_target *ti) diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index 7595142f3fc5..7b2968612b7e 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -80,7 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client); * error occurred doing io to the corresponding region. */ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *region, unsigned int long *sync_error_bits); + struct dm_io_region *region, unsigned int long *sync_error_bits, + unsigned short ioprio); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_IO_H */ From e9b2238e47cb66521381c0bf9234e979afa19d77 Mon Sep 17 00:00:00 2001 From: Hongyu Jin Date: Wed, 24 Jan 2024 13:35:54 +0800 Subject: [PATCH 0639/1038] dm bufio: Support IO priority Some IO will dispatch from kworker with different io_context settings than the submitting task, we may need to specify a priority to avoid losing priority. Add dm_bufio_read_with_ioprio() and dm_bufio_prefetch_with_ioprio() for use by bufio users to pass an ioprio other than IOPRIO_DEFAULT. Co-developed-by: Yibin Ding Signed-off-by: Yibin Ding Signed-off-by: Hongyu Jin Reviewed-by: Eric Biggers Reviewed-by: Mikulas Patocka [snitzer: introduced _with_ioprio() wrappers to reduce churn] Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 68 +++++++++++++++++++++++++++++----------- include/linux/dm-bufio.h | 7 +++++ 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index f5541b8f6320..dad75b8d0b55 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1292,7 +1292,8 @@ static void dmio_complete(unsigned long error, void *context) } static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, - unsigned int n_sectors, unsigned int offset) + unsigned int n_sectors, unsigned int offset, + unsigned short ioprio) { int r; struct dm_io_request io_req = { @@ -1315,7 +1316,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, ®ion, NULL, ioprio); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -1331,7 +1332,8 @@ static void bio_complete(struct bio *bio) } static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, - unsigned int n_sectors, unsigned int offset) + unsigned int n_sectors, unsigned int offset, + unsigned short ioprio) { struct bio *bio; char *ptr; @@ -1339,13 +1341,14 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN); if (!bio) { - use_dmio(b, op, sector, n_sectors, offset); + use_dmio(b, op, sector, n_sectors, offset, ioprio); return; } bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); bio->bi_iter.bi_sector = sector; bio->bi_end_io = bio_complete; bio->bi_private = b; + bio->bi_ioprio = ioprio; ptr = (char *)b->data + offset; len = n_sectors << SECTOR_SHIFT; @@ -1368,7 +1371,7 @@ static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block return sector; } -static void submit_io(struct dm_buffer *b, enum req_op op, +static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio, void (*end_io)(struct dm_buffer *, blk_status_t)) { unsigned int n_sectors; @@ -1398,9 +1401,9 @@ static void submit_io(struct dm_buffer *b, enum req_op op, } if (b->data_mode != DATA_MODE_VMALLOC) - use_bio(b, op, sector, n_sectors, offset); + use_bio(b, op, sector, n_sectors, offset, ioprio); else - use_dmio(b, op, sector, n_sectors, offset); + use_dmio(b, op, sector, n_sectors, offset, ioprio); } /* @@ -1456,7 +1459,7 @@ static void __write_dirty_buffer(struct dm_buffer *b, b->write_end = b->dirty_end; if (!write_list) - submit_io(b, REQ_OP_WRITE, write_endio); + submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); else list_add_tail(&b->write_list, write_list); } @@ -1470,7 +1473,7 @@ static void __flush_write_list(struct list_head *write_list) struct dm_buffer *b = list_entry(write_list->next, struct dm_buffer, write_list); list_del(&b->write_list); - submit_io(b, REQ_OP_WRITE, write_endio); + submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); cond_resched(); } blk_finish_plug(&plug); @@ -1852,7 +1855,8 @@ static void read_endio(struct dm_buffer *b, blk_status_t status) * and uses dm_bufio_mark_buffer_dirty to write new data back). */ static void *new_read(struct dm_bufio_client *c, sector_t block, - enum new_flag nf, struct dm_buffer **bp) + enum new_flag nf, struct dm_buffer **bp, + unsigned short ioprio) { int need_submit = 0; struct dm_buffer *b; @@ -1905,7 +1909,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, return NULL; if (need_submit) - submit_io(b, REQ_OP_READ, read_endio); + submit_io(b, REQ_OP_READ, ioprio, read_endio); if (nf != NF_GET) /* we already tested this condition above */ wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); @@ -1926,32 +1930,46 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp) { - return new_read(c, block, NF_GET, bp); + return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_get); -void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, - struct dm_buffer **bp) +static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio) { if (WARN_ON_ONCE(dm_bufio_in_request())) return ERR_PTR(-EINVAL); - return new_read(c, block, NF_READ, bp); + return new_read(c, block, NF_READ, bp, ioprio); +} + +void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp) +{ + return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_read); +void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio) +{ + return __dm_bufio_read(c, block, bp, ioprio); +} +EXPORT_SYMBOL_GPL(dm_bufio_read_with_ioprio); + void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp) { if (WARN_ON_ONCE(dm_bufio_in_request())) return ERR_PTR(-EINVAL); - return new_read(c, block, NF_FRESH, bp); + return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_new); -void dm_bufio_prefetch(struct dm_bufio_client *c, - sector_t block, unsigned int n_blocks) +static void __dm_bufio_prefetch(struct dm_bufio_client *c, + sector_t block, unsigned int n_blocks, + unsigned short ioprio) { struct blk_plug plug; @@ -1987,7 +2005,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, dm_bufio_unlock(c); if (need_submit) - submit_io(b, REQ_OP_READ, read_endio); + submit_io(b, REQ_OP_READ, ioprio, read_endio); dm_bufio_release(b); cond_resched(); @@ -2002,8 +2020,20 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, flush_plug: blk_finish_plug(&plug); } + +void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks) +{ + return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT); +} EXPORT_SYMBOL_GPL(dm_bufio_prefetch); +void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block, + unsigned int n_blocks, unsigned short ioprio) +{ + return __dm_bufio_prefetch(c, block, n_blocks, ioprio); +} +EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio); + void dm_bufio_release(struct dm_buffer *b) { struct dm_bufio_client *c = b->c; diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h index 75e7d8cbb532..d1503b815a78 100644 --- a/include/linux/dm-bufio.h +++ b/include/linux/dm-bufio.h @@ -64,6 +64,9 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start); void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp); +void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio); + /* * Like dm_bufio_read, but return buffer from cache, don't read * it. If the buffer is not in the cache, return NULL. @@ -86,6 +89,10 @@ void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks); +void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, + sector_t block, unsigned int n_blocks, + unsigned short ioprio); + /* * Release a reference obtained with dm_bufio_{read,get,new}. The data * pointer and dm_buffer pointer is no longer valid after this call. From d95e2c34a3ca76861014cde4205934e434c27c45 Mon Sep 17 00:00:00 2001 From: Hongyu Jin Date: Wed, 24 Jan 2024 13:35:55 +0800 Subject: [PATCH 0640/1038] dm verity: Fix IO priority lost when reading FEC and hash After obtaining the data, verification or error correction process may trigger a new IO that loses the priority of the original IO, that is, the verification of the higher priority IO may be blocked by the lower priority IO. Make the IO used for verification and error correction follow the priority of the original IO. Co-developed-by: Yibin Ding Signed-off-by: Yibin Ding Signed-off-by: Hongyu Jin Reviewed-by: Eric Biggers Reviewed-by: Mikulas Patocka Signed-off-by: Mike Snitzer --- drivers/md/dm-verity-fec.c | 21 ++++++++++++--------- drivers/md/dm-verity-target.c | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index b475200d8586..e46aee6f932e 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -60,7 +60,8 @@ static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, * to the data block. Caller is responsible for releasing buf. */ static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, - unsigned int *offset, struct dm_buffer **buf) + unsigned int *offset, struct dm_buffer **buf, + unsigned short ioprio) { u64 position, block, rem; u8 *res; @@ -69,7 +70,7 @@ static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, block = div64_u64_rem(position, v->fec->io_size, &rem); *offset = (unsigned int)rem; - res = dm_bufio_read(v->fec->bufio, block, buf); + res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); if (IS_ERR(res)) { DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", v->data_dev->name, (unsigned long long)rsb, @@ -121,16 +122,17 @@ static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j) * Decode all RS blocks from buffers and copy corrected bytes into fio->output * starting from block_offset. */ -static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, - u64 rsb, int byte_index, unsigned int block_offset, - int neras) +static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, + struct dm_verity_fec_io *fio, u64 rsb, int byte_index, + unsigned int block_offset, int neras) { int r, corrected = 0, res; struct dm_buffer *buf; unsigned int n, i, offset; u8 *par, *block; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); - par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + par = fec_read_parity(v, rsb, block_offset, &offset, &buf, bio_prio(bio)); if (IS_ERR(par)) return PTR_ERR(par); @@ -158,7 +160,7 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, if (offset >= v->fec->io_size) { dm_bufio_release(buf); - par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + par = fec_read_parity(v, rsb, block_offset, &offset, &buf, bio_prio(bio)); if (IS_ERR(par)) return PTR_ERR(par); } @@ -210,6 +212,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, u8 *bbuf, *rs_block; u8 want_digest[HASH_MAX_DIGESTSIZE]; unsigned int n, k; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); if (neras) *neras = 0; @@ -248,7 +251,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, bufio = v->bufio; } - bbuf = dm_bufio_read(bufio, block, &buf); + bbuf = dm_bufio_read_with_ioprio(bufio, block, &buf, bio_prio(bio)); if (IS_ERR(bbuf)) { DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", v->data_dev->name, @@ -377,7 +380,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, if (unlikely(r < 0)) return r; - r = fec_decode_bufs(v, fio, rsb, r, pos, neras); + r = fec_decode_bufs(v, io, fio, rsb, r, pos, neras); if (r < 0) return r; diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index f55fbbe5d22f..01ec71dcbc7d 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -51,6 +51,7 @@ static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); struct dm_verity_prefetch_work { struct work_struct work; struct dm_verity *v; + unsigned short ioprio; sector_t block; unsigned int n_blocks; }; @@ -294,6 +295,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, int r; sector_t hash_block; unsigned int offset; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); verity_hash_at_level(v, block, level, &hash_block, &offset); @@ -307,8 +309,10 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, */ return -EAGAIN; } - } else - data = dm_bufio_read(v->bufio, hash_block, &buf); + } else { + data = dm_bufio_read_with_ioprio(v->bufio, hash_block, + &buf, bio_prio(bio)); + } if (IS_ERR(data)) return PTR_ERR(data); @@ -752,14 +756,16 @@ static void verity_prefetch_io(struct work_struct *work) hash_block_end = v->hash_blocks - 1; } no_prefetch_cluster: - dm_bufio_prefetch(v->bufio, hash_block_start, - hash_block_end - hash_block_start + 1); + dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start, + hash_block_end - hash_block_start + 1, + pw->ioprio); } kfree(pw); } -static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) +static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io, + unsigned short ioprio) { sector_t block = io->block; unsigned int n_blocks = io->n_blocks; @@ -787,6 +793,7 @@ static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) pw->v = v; pw->block = block; pw->n_blocks = n_blocks; + pw->ioprio = ioprio; queue_work(v->verify_wq, &pw->work); } @@ -829,7 +836,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio) verity_fec_init_io(io); - verity_submit_prefetch(v, io); + verity_submit_prefetch(v, io, bio_prio(bio)); submit_bio_noacct(bio); From 5d8d40815314684af2593c075e6eee1afb42bb92 Mon Sep 17 00:00:00 2001 From: Hongyu Jin Date: Wed, 24 Jan 2024 13:35:56 +0800 Subject: [PATCH 0641/1038] dm crypt: Fix IO priority lost when queuing write bios Since dm-crypt queues writes to a different kernel thread (workqueue), the bios will dispatch from tasks with different io_context->ioprio settings and blkcg than the submitting task, thus giving incorrect ioprio to the io scheduler. Get the original IO priority setting via struct dm_crypt_io::base_bio and set this priority in the bio for write. Link: https://lore.kernel.org/dm-devel/alpine.LRH.2.11.1612141049250.13402@mail.ewheeler.net Signed-off-by: Hongyu Jin Reviewed-by: Eric Biggers Reviewed-by: Mikulas Patocka Signed-off-by: Mike Snitzer --- drivers/md/dm-crypt.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 7388f45ada07..50467f005177 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1688,6 +1688,7 @@ retry: GFP_NOIO, &cc->bs); clone->bi_private = io; clone->bi_end_io = crypt_endio; + clone->bi_ioprio = io->base_bio->bi_ioprio; remaining_size = size; From 9356fcfe0ac4a8545f9fc32f2e404524e1115ee6 Mon Sep 17 00:00:00 2001 From: Fan Wu Date: Tue, 30 Jan 2024 14:37:00 -0800 Subject: [PATCH 0642/1038] dm verity: set DM_TARGET_SINGLETON feature flag The device-mapper has a flag to mark targets as singleton, which is a required flag for immutable targets. Without this flag, multiple dm-verity targets can be added to a mapped device, which has no practical use cases and will let dm_table_get_immutable_target return NULL. This patch adds the missing flag, restricting only one dm-verity target per mapped device. Signed-off-by: Fan Wu Signed-off-by: Mike Snitzer --- drivers/md/dm-verity-target.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 01ec71dcbc7d..6b8f9698488e 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -1566,7 +1566,7 @@ int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned i static struct target_type verity_target = { .name = "verity", - .features = DM_TARGET_IMMUTABLE, + .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, .version = {1, 10, 0}, .module = THIS_MODULE, .ctr = verity_ctr, From 86ab1b84b257bce8110947f563e168498d2d3860 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 7 Feb 2024 11:42:53 -0500 Subject: [PATCH 0643/1038] dm ioctl: update DM_DRIVER_EMAIL to new dm-devel mailing list Fixes: 3da5d2de9238 ("MAINTAINERS: update the dm-devel mailing list") Signed-off-by: Mike Snitzer --- drivers/md/dm-ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 3b1ad7127cb8..c2c07bfa6471 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -25,7 +25,7 @@ #include #define DM_MSG_PREFIX "ioctl" -#define DM_DRIVER_EMAIL "dm-devel@redhat.com" +#define DM_DRIVER_EMAIL "dm-devel@lists.linux.dev" struct dm_file { /* From fa34e5893ff2d5b0174c124a29e1be6d0426a169 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 7 Feb 2024 15:51:24 -0500 Subject: [PATCH 0644/1038] dm: update relevant MODULE_AUTHOR entries to latest dm-devel mailing list Signed-off-by: Mike Snitzer --- drivers/md/dm-bio-prison-v1.c | 2 +- drivers/md/dm-bufio.c | 2 +- drivers/md/dm-cache-policy-smq.c | 2 +- drivers/md/dm-dust.c | 2 +- drivers/md/dm-ebs-target.c | 2 +- drivers/md/dm-flakey.c | 2 +- drivers/md/dm-log-userspace-base.c | 2 +- drivers/md/dm-log.c | 2 +- drivers/md/dm-mpath.c | 2 +- drivers/md/dm-ps-round-robin.c | 2 +- drivers/md/dm-raid.c | 4 ++-- drivers/md/dm-region-hash.c | 2 +- drivers/md/dm-thin.c | 2 +- drivers/md/dm-writecache.c | 2 +- drivers/md/dm.c | 2 +- drivers/md/persistent-data/dm-block-manager.c | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/drivers/md/dm-bio-prison-v1.c b/drivers/md/dm-bio-prison-v1.c index 9ab32abe5ed4..bca0f39e15b8 100644 --- a/drivers/md/dm-bio-prison-v1.c +++ b/drivers/md/dm-bio-prison-v1.c @@ -489,5 +489,5 @@ module_init(dm_bio_prison_init); module_exit(dm_bio_prison_exit); MODULE_DESCRIPTION(DM_NAME " bio prison"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index dad75b8d0b55..098bf526136c 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -2998,6 +2998,6 @@ MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc"); module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, 0444); MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache"); -MODULE_AUTHOR("Mikulas Patocka "); +MODULE_AUTHOR("Mikulas Patocka "); MODULE_DESCRIPTION(DM_NAME " buffered I/O library"); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c index 8bd2ad743d9a..2ed894155cab 100644 --- a/drivers/md/dm-cache-policy-smq.c +++ b/drivers/md/dm-cache-policy-smq.c @@ -1947,7 +1947,7 @@ static void __exit smq_exit(void) module_init(smq_init); module_exit(smq_exit); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("smq cache policy"); diff --git a/drivers/md/dm-dust.c b/drivers/md/dm-dust.c index 12a377e06d02..1a33820c9f46 100644 --- a/drivers/md/dm-dust.c +++ b/drivers/md/dm-dust.c @@ -573,5 +573,5 @@ static struct target_type dust_target = { module_dm(dust); MODULE_DESCRIPTION(DM_NAME " dust test target"); -MODULE_AUTHOR("Bryan Gurney "); +MODULE_AUTHOR("Bryan Gurney "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-ebs-target.c b/drivers/md/dm-ebs-target.c index 435b45201f4d..b70d4016c2ac 100644 --- a/drivers/md/dm-ebs-target.c +++ b/drivers/md/dm-ebs-target.c @@ -454,6 +454,6 @@ static struct target_type ebs_target = { }; module_dm(ebs); -MODULE_AUTHOR("Heinz Mauelshagen "); +MODULE_AUTHOR("Heinz Mauelshagen "); MODULE_DESCRIPTION(DM_NAME " emulated block size target"); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index 7916ed9f10e8..731467d4ed10 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -690,5 +690,5 @@ static struct target_type flakey_target = { module_dm(flakey); MODULE_DESCRIPTION(DM_NAME " flakey target"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c index 7e4f27e86150..9fbb4b48fb2b 100644 --- a/drivers/md/dm-log-userspace-base.c +++ b/drivers/md/dm-log-userspace-base.c @@ -926,5 +926,5 @@ module_init(userspace_dirty_log_init); module_exit(userspace_dirty_log_exit); MODULE_DESCRIPTION(DM_NAME " userspace dirty log link"); -MODULE_AUTHOR("Jonathan Brassow "); +MODULE_AUTHOR("Jonathan Brassow "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index f7f9c2100937..9d85d045f9d9 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -908,5 +908,5 @@ module_init(dm_dirty_log_init); module_exit(dm_dirty_log_exit); MODULE_DESCRIPTION(DM_NAME " dirty region log"); -MODULE_AUTHOR("Joe Thornber, Heinz Mauelshagen "); +MODULE_AUTHOR("Joe Thornber, Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index bea3cda9938e..05d1328d1811 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -2266,5 +2266,5 @@ module_param_named(queue_if_no_path_timeout_secs, queue_if_no_path_timeout_secs, MODULE_PARM_DESC(queue_if_no_path_timeout_secs, "No available paths queue IO timeout in seconds"); MODULE_DESCRIPTION(DM_NAME " multipath target"); -MODULE_AUTHOR("Sistina Software "); +MODULE_AUTHOR("Sistina Software "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-ps-round-robin.c b/drivers/md/dm-ps-round-robin.c index 0f04b673597a..d1745b123dc1 100644 --- a/drivers/md/dm-ps-round-robin.c +++ b/drivers/md/dm-ps-round-robin.c @@ -240,5 +240,5 @@ module_init(dm_rr_init); module_exit(dm_rr_exit); MODULE_DESCRIPTION(DM_NAME " round-robin multipath path selector"); -MODULE_AUTHOR("Sistina Software "); +MODULE_AUTHOR("Sistina Software "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index eb009d6bb03a..6bb1765be1e5 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -4091,6 +4091,6 @@ MODULE_ALIAS("dm-raid10"); MODULE_ALIAS("dm-raid4"); MODULE_ALIAS("dm-raid5"); MODULE_ALIAS("dm-raid6"); -MODULE_AUTHOR("Neil Brown "); -MODULE_AUTHOR("Heinz Mauelshagen "); +MODULE_AUTHOR("Neil Brown "); +MODULE_AUTHOR("Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-region-hash.c b/drivers/md/dm-region-hash.c index 852cfa37d48a..a4550975c27d 100644 --- a/drivers/md/dm-region-hash.c +++ b/drivers/md/dm-region-hash.c @@ -723,5 +723,5 @@ void dm_rh_start_recovery(struct dm_region_hash *rh) EXPORT_SYMBOL_GPL(dm_rh_start_recovery); MODULE_DESCRIPTION(DM_NAME " region hash"); -MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen "); +MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index f88646f9f81f..4793ad2aa1f7 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -4564,5 +4564,5 @@ module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644); MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds"); MODULE_DESCRIPTION(DM_NAME " thin provisioning target"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index 01ab141bc51e..7ce8847b3404 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -2776,5 +2776,5 @@ static struct target_type writecache_target = { module_dm(writecache); MODULE_DESCRIPTION(DM_NAME " writecache target"); -MODULE_AUTHOR("Mikulas Patocka "); +MODULE_AUTHOR("Mikulas Patocka "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 8dcabf84d866..1bed115a7e8e 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -3512,5 +3512,5 @@ module_param(swap_bios, int, 0644); MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); MODULE_DESCRIPTION(DM_NAME " driver"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/persistent-data/dm-block-manager.c b/drivers/md/persistent-data/dm-block-manager.c index 0e010e1204aa..b17b54df673b 100644 --- a/drivers/md/persistent-data/dm-block-manager.c +++ b/drivers/md/persistent-data/dm-block-manager.c @@ -656,7 +656,7 @@ EXPORT_SYMBOL_GPL(dm_bm_checksum); /*----------------------------------------------------------------*/ MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_DESCRIPTION("Immutable metadata library for dm"); /*----------------------------------------------------------------*/ From 132a85f1c28b41ef4950c64fbf2d3a0ee23e79dd Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 21 Feb 2024 01:58:50 +0200 Subject: [PATCH 0645/1038] regulator: dt-bindings: qcom,usb-vbus-regulator: add support for PM4125 The VBUS register block on the PM4125 PMIC shares the design with the PM8150B one. Define corresponding compatible string, having the qcom,pm8150b-vbus-reg as a fallback. Acked-by: Krzysztof Kozlowski Signed-off-by: Dmitry Baryshkov Link: https://msgid.link/r/20240221-pm4125-typec-v3-1-fdd0ee0465b8@linaro.org Signed-off-by: Mark Brown --- .../devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml index bf6336850be6..8afb40c67af3 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml @@ -24,6 +24,7 @@ properties: - qcom,pm8150b-vbus-reg - items: - enum: + - qcom,pm4125-vbus-reg - qcom,pm6150-vbus-reg - const: qcom,pm8150b-vbus-reg From b9262cc1b988cdaf9bb5c2a4411d4ad4e7128e8d Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 21 Feb 2024 01:58:50 +0200 Subject: [PATCH 0646/1038] regulator: dt-bindings: qcom,usb-vbus-regulator: add support for PM4125 The VBUS register block on the PM4125 PMIC shares the design with the PM8150B one. Define corresponding compatible string, having the qcom,pm8150b-vbus-reg as a fallback. Acked-by: Krzysztof Kozlowski Signed-off-by: Dmitry Baryshkov Link: https://msgid.link/r/20240221-pm4125-typec-v3-1-fdd0ee0465b8@linaro.org Signed-off-by: Mark Brown --- .../devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml index bf6336850be6..8afb40c67af3 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml @@ -24,6 +24,7 @@ properties: - qcom,pm8150b-vbus-reg - items: - enum: + - qcom,pm4125-vbus-reg - qcom,pm6150-vbus-reg - const: qcom,pm8150b-vbus-reg From 303cd69394bb01f7ce0ce7509b3bd02f34795c05 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Wed, 21 Feb 2024 12:12:53 +0000 Subject: [PATCH 0647/1038] mm, slab: remove unused object_size parameter in kmem_cache_flags() We don't use the object_size parameter in kmem_cache_flags(), so just remove it. Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slab.h | 3 +-- mm/slab_common.c | 2 +- mm/slub.c | 9 +++------ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index 9abec38be1d0..6b879d8e8a7a 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -433,8 +433,7 @@ struct kmem_cache * __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, slab_flags_t flags, void (*ctor)(void *)); -slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name); +slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name); static inline bool is_kmalloc_cache(struct kmem_cache *s) { diff --git a/mm/slab_common.c b/mm/slab_common.c index 1910252d7e89..e19544043fdf 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -172,7 +172,7 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); size = ALIGN(size, align); - flags = kmem_cache_flags(size, flags, name); + flags = kmem_cache_flags(flags, name); if (flags & SLAB_NEVER_MERGE) return NULL; diff --git a/mm/slub.c b/mm/slub.c index 4ebc0df410ff..d8d8dd8e9803 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1765,7 +1765,6 @@ __setup_param("slub_debug", slub_debug, setup_slub_debug, 0); /* * kmem_cache_flags - apply debugging options to the cache - * @object_size: the size of an object without meta data * @flags: flags to set * @name: name of the cache * @@ -1774,8 +1773,7 @@ __setup_param("slub_debug", slub_debug, setup_slub_debug, 0); * slab_debug=,, ... * then only the select slabs will receive the debug option(s). */ -slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name) +slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) { char *iter; size_t len; @@ -1851,8 +1849,7 @@ static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) {} static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) {} -slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name) +slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) { return flags; } @@ -5105,7 +5102,7 @@ static int calculate_sizes(struct kmem_cache *s) static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) { - s->flags = kmem_cache_flags(s->size, flags, s->name); + s->flags = kmem_cache_flags(flags, s->name); #ifdef CONFIG_SLAB_FREELIST_HARDENED s->random = get_random_long(); #endif From c94d222445c168788a8849413e676fcc256e9479 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Wed, 21 Feb 2024 12:12:54 +0000 Subject: [PATCH 0648/1038] mm, slab: fix the comment of cpu partial list The partial slabs on cpu partial list are not frozen after the commit 8cd3fa428b56 ("slub: Delay freezing of partial slabs") merged. So fix the comment. Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index d8d8dd8e9803..e28929e6e252 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -391,7 +391,7 @@ struct kmem_cache_cpu { }; struct slab *slab; /* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL - struct slab *partial; /* Partially allocated frozen slabs */ + struct slab *partial; /* Partially allocated slabs */ #endif local_lock_t lock; /* Protects the fields above */ #ifdef CONFIG_SLUB_STATS From 0d62c64a8e48438545dcef7e5d2f4839ff5cfe4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Fri, 9 Feb 2024 14:45:30 +0100 Subject: [PATCH 0649/1038] spi: cadence-qspi: assert each subnode flash CS is valid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check each flash CS against the num-cs property from devicetree. Fallback to the driver max supported value (CQSPI_MAX_CHIPSELECT) if num-cs isn't present. cqspi->num_chipselect is set in cqspi_of_get_pdata() to the num-cs devicetree property, or to CQSPI_MAX_CHIPSELECT if num-cs is not set. Signed-off-by: Théo Lebrun Reviewed-by: Dhruva Gole Link: https://msgid.link/r/20240209-cdns-qspi-cs-v1-1-a4f9dfed9ab4@bootlin.com Signed-off-by: Mark Brown --- drivers/spi/spi-cadence-quadspi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index a5e2f7786b76..a397f2c2b5fc 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1635,7 +1635,7 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi) return ret; } - if (cs >= CQSPI_MAX_CHIPSELECT) { + if (cs >= cqspi->num_chipselect) { dev_err(dev, "Chip select %d out of range.\n", cs); of_node_put(np); return -EINVAL; From 7cc3522aedb5f4360c4502b2e89b279b7aa94ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Fri, 9 Feb 2024 14:45:31 +0100 Subject: [PATCH 0650/1038] spi: cadence-qspi: set maximum chip-select to 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the maximum chip-select count in cadence-qspi to 4 instead of 16. The value gets used as default ->num_chipselect when the num-cs DT property isn't received from devicetree. It also determines the cqspi->f_pdata array size. Hardware only supports values up to 4; see cqspi_chipselect() that sets CS using a one-bit-per-CS 4-bit register field. Add a static_assert() call as a defensive measure to ensure we stay under the SPI subsystem limit. It got set to 4 when introduced in 4d8ff6b0991d ("spi: Add multi-cs memories support in SPI core") and later increased to 16 in 2f8c7c3715f2 ("spi: Raise limit on number of chip selects"). Signed-off-by: Théo Lebrun Link: https://msgid.link/r/20240209-cdns-qspi-cs-v1-2-a4f9dfed9ab4@bootlin.com Signed-off-by: Mark Brown --- drivers/spi/spi-cadence-quadspi.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index a397f2c2b5fc..fd34b48dfb4f 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -31,7 +31,9 @@ #include #define CQSPI_NAME "cadence-qspi" -#define CQSPI_MAX_CHIPSELECT 16 +#define CQSPI_MAX_CHIPSELECT 4 + +static_assert(CQSPI_MAX_CHIPSELECT <= SPI_CS_CNT_MAX); /* Quirks */ #define CQSPI_NEEDS_WR_DELAY BIT(0) From 0f3841a5e1152eca1a58cfbd9ceb6d311aa7e647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Fri, 9 Feb 2024 14:45:32 +0100 Subject: [PATCH 0651/1038] spi: cadence-qspi: report correct number of chip-select MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set the ->num_chipselect field in struct cqspi_st and struct spi_controller to the current number of chip-select. The value is dependent on declared flashes in devicetree. Previously, the num-cs property from devicetree or the maximum value was being reported. Signed-off-by: Théo Lebrun Link: https://msgid.link/r/20240209-cdns-qspi-cs-v1-3-a4f9dfed9ab4@bootlin.com Signed-off-by: Mark Brown --- drivers/spi/spi-cadence-quadspi.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index fd34b48dfb4f..0df732b03a2d 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1621,6 +1621,7 @@ static const struct spi_controller_mem_caps cqspi_mem_caps = { static int cqspi_setup_flash(struct cqspi_st *cqspi) { + unsigned int max_cs = cqspi->num_chipselect - 1; struct platform_device *pdev = cqspi->pdev; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; @@ -1641,6 +1642,8 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi) dev_err(dev, "Chip select %d out of range.\n", cs); of_node_put(np); return -EINVAL; + } else if (cs < max_cs) { + max_cs = cs; } f_pdata = &cqspi->f_pdata[cs]; @@ -1654,6 +1657,7 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi) } } + cqspi->num_chipselect = max_cs + 1; return 0; } @@ -1864,14 +1868,14 @@ static int cqspi_probe(struct platform_device *pdev) cqspi->current_cs = -1; cqspi->sclk = 0; - host->num_chipselect = cqspi->num_chipselect; - ret = cqspi_setup_flash(cqspi); if (ret) { dev_err(dev, "failed to setup flash parameters %d\n", ret); goto probe_setup_failed; } + host->num_chipselect = cqspi->num_chipselect; + if (cqspi->use_direct_mode) { ret = cqspi_request_mmap_dma(cqspi); if (ret == -EPROBE_DEFER) From 13ec4098d8032b0e8d2b2548803002df80d7f9c6 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Mon, 19 Feb 2024 21:47:43 +0100 Subject: [PATCH 0652/1038] ahci: print the number of implemented ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are currently printing the CAP.NP field. CAP.NP is a 0's based value indicating the maximum number of ports supported by the HBA silicon. Note that the number of ports indicated in this field may be more than the number of ports indicated in the PI (ports implemented) register. (See AHCI 1.3.1, section 3.1.1 - Offset 00h: CAP – HBA Capabilities.) PI (ports implemented) register is a field that has a bit set to '1' if that specific port is implemented. This register is allowed to have zeroes mixed with ones, i.e. a port in the middle is allowed to be unimplemented. (See AHCI 1.3.1, section 3.1.4 - Offset 0Ch: PI – Ports Implemented.) Since the number of ports implemented might be smaller than the maximum number of ports supported by the HBA silicon, print the number of implemented ports as well. While at it, clarify the properties being printed, and add a separator (,), as that is currently missing, making it very easy to get confused if the number before or after belongs to . before: ahci 0000:00:03.0: masking port_map 0x3f -> 0x2f ahci 0000:00:03.0: AHCI 0001.0000 32 slots 6 ports 1.5 Gbps 0x2f impl SATA mode ahci 0000:00:03.0: flags: 64bit ncq only after: ahci 0000:00:03.0: masking port_map 0x3f -> 0x2f ahci 0000:00:03.0: AHCI vers 0001.0000, 32 command slots, 1.5 Gbps, SATA mode ahci 0000:00:03.0: 5/6 ports implemented (port mask 0x2f) ahci 0000:00:03.0: flags: 64bit ncq only Suggested-by: Damien Le Moal Reviewed-by: Damien Le Moal Signed-off-by: Niklas Cassel --- drivers/ata/libahci.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index fca376f03c9e..83431aae74d8 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -2620,8 +2620,8 @@ void ahci_print_info(struct ata_host *host, const char *scc_s) speed_s = "?"; dev_info(host->dev, - "AHCI %02x%02x.%02x%02x " - "%u slots %u ports %s Gbps 0x%x impl %s mode\n" + "AHCI vers %02x%02x.%02x%02x, " + "%u command slots, %s Gbps, %s mode\n" , (vers >> 24) & 0xff, @@ -2630,11 +2630,17 @@ void ahci_print_info(struct ata_host *host, const char *scc_s) vers & 0xff, ((cap >> 8) & 0x1f) + 1, - (cap & 0x1f) + 1, speed_s, - impl, scc_s); + dev_info(host->dev, + "%u/%u ports implemented (port mask 0x%x)\n" + , + + hweight32(impl), + (cap & 0x1f) + 1, + impl); + dev_info(host->dev, "flags: " "%s%s%s%s%s%s%s" From 936615f63768482be942644dd3b1b1dca862f25b Mon Sep 17 00:00:00 2001 From: Pairman Guo Date: Sat, 17 Feb 2024 21:35:04 +0800 Subject: [PATCH 0653/1038] lsm: fix typos in security/security.c comment headers This commit fixes several typos in comment headers in security/security.c where "Check is" should be "Check if". Signed-off-by: Pairman Guo [PM: subject line tweak] Signed-off-by: Paul Moore --- security/security.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/security/security.c b/security/security.c index cbdc9bebe802..c31e45da672b 100644 --- a/security/security.c +++ b/security/security.c @@ -2955,7 +2955,7 @@ int security_file_send_sigiotask(struct task_struct *tsk, } /** - * security_file_receive() - Check is receiving a file via IPC is allowed + * security_file_receive() - Check if receiving a file via IPC is allowed * @file: file being received * * This hook allows security modules to control the ability of a process to @@ -3182,7 +3182,7 @@ int security_kernel_create_files_as(struct cred *new, struct inode *inode) } /** - * security_kernel_module_request() - Check is loading a module is allowed + * security_kernel_module_request() - Check if loading a module is allowed * @kmod_name: module name * * Ability to trigger the kernel to automatically upcall to userspace for @@ -4155,7 +4155,7 @@ int security_netlink_send(struct sock *sk, struct sk_buff *skb) } /** - * security_ismaclabel() - Check is the named attribute is a MAC label + * security_ismaclabel() - Check if the named attribute is a MAC label * @name: full extended attribute name * * Check if the extended attribute specified by @name represents a MAC label. @@ -4528,7 +4528,7 @@ int security_socket_accept(struct socket *sock, struct socket *newsock) } /** - * security_socket_sendmsg() - Check is sending a message is allowed + * security_socket_sendmsg() - Check if sending a message is allowed * @sock: sending socket * @msg: message to send * @size: size of message From 7c86e17455de1a442ec906d3449148b5e9a218a4 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 18 Feb 2024 16:15:32 +0100 Subject: [PATCH 0654/1038] ACPI: x86: Move acpi_quirk_skip_serdev_enumeration() out of CONFIG_X86_ANDROID_TABLETS Some recent(ish) Dell AIO devices have a backlight controller board connected to an UART. This UART has a DELL0501 HID with CID set to PNP0501 so that the UART is still handled by 8250_pnp.c. Unfortunately there is no separate ACPI device with an UartSerialBusV2() resource to model the backlight-controller. The next patch in this series will use acpi_quirk_skip_serdev_enumeration() to still create a serdev for this for a backlight driver to bind to instead of creating a /dev/ttyS0. This new acpi_quirk_skip_serdev_enumeration() use is not limited to Android X86 tablets, so move it out of the ifdef CONFIG_X86_ANDROID_TABLETS block. Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/x86/utils.c | 18 ++++++++++++++---- include/acpi/acpi_bus.h | 22 +++++++++++----------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index bc65ebfcdf76..8829a907eee0 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -428,7 +428,7 @@ bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev) } EXPORT_SYMBOL_GPL(acpi_quirk_skip_i2c_client_enumeration); -int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) +static int acpi_dmi_skip_serdev_enumeration(struct device *controller_parent, bool *skip) { struct acpi_device *adev = ACPI_COMPANION(controller_parent); const struct dmi_system_id *dmi_id; @@ -436,8 +436,6 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s u64 uid; int ret; - *skip = false; - ret = acpi_dev_uid_to_integer(adev, &uid); if (ret) return 0; @@ -463,7 +461,6 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s return 0; } -EXPORT_SYMBOL_GPL(acpi_quirk_skip_serdev_enumeration); bool acpi_quirk_skip_gpio_event_handlers(void) { @@ -478,8 +475,21 @@ bool acpi_quirk_skip_gpio_event_handlers(void) return (quirks & ACPI_QUIRK_SKIP_GPIO_EVENT_HANDLERS); } EXPORT_SYMBOL_GPL(acpi_quirk_skip_gpio_event_handlers); +#else +static int acpi_dmi_skip_serdev_enumeration(struct device *controller_parent, bool *skip) +{ + return 0; +} #endif +int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) +{ + *skip = false; + + return acpi_dmi_skip_serdev_enumeration(controller_parent, skip); +} +EXPORT_SYMBOL_GPL(acpi_quirk_skip_serdev_enumeration); + /* Lists of PMIC ACPI HIDs with an (often better) native charger driver */ static const struct { const char *hid; diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index e4d24d3f9abb..446225aada50 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -749,6 +749,7 @@ bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *s bool acpi_quirk_skip_acpi_ac_and_battery(void); int acpi_install_cmos_rtc_space_handler(acpi_handle handle); void acpi_remove_cmos_rtc_space_handler(acpi_handle handle); +int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip); #else static inline bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *status) @@ -766,23 +767,22 @@ static inline int acpi_install_cmos_rtc_space_handler(acpi_handle handle) static inline void acpi_remove_cmos_rtc_space_handler(acpi_handle handle) { } -#endif - -#if IS_ENABLED(CONFIG_X86_ANDROID_TABLETS) -bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev); -int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip); -bool acpi_quirk_skip_gpio_event_handlers(void); -#else -static inline bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev) -{ - return false; -} static inline int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) { *skip = false; return 0; } +#endif + +#if IS_ENABLED(CONFIG_X86_ANDROID_TABLETS) +bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev); +bool acpi_quirk_skip_gpio_event_handlers(void); +#else +static inline bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev) +{ + return false; +} static inline bool acpi_quirk_skip_gpio_event_handlers(void) { return false; From 99b572e6136eab69a8c91d72cf8595b256e304b5 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 18 Feb 2024 16:15:33 +0100 Subject: [PATCH 0655/1038] ACPI: x86: Add DELL0501 handling to acpi_quirk_skip_serdev_enumeration() Some recent(ish) Dell AIO devices have a backlight controller board connected to an UART. This UART has a DELL0501 HID with CID set to PNP0501 so that the UART is still handled by 8250_pnp.c. Unfortunately there is no separate ACPI device with an UartSerialBusV2() resource to model the backlight-controller. This causes the kernel to create a /dev/ttyS0 char-device for the UART instead of creating an in kernel serdev-controller + serdev-device pair for a kernel backlight driver. Use the existing acpi_quirk_skip_serdev_enumeration() mechanism to work around this by returning skip=true for tty-ctrl parents with a HID of DELL0501. Like other cases where the UartSerialBusV2() resource is missing or broken this will only create the serdev-controller device and the serdev-device itself will need to be instantiated by platform code. Unfortunately in this case there is no device for the platform-code instantiating the serdev-device to bind to. So also create a platform_device for this. Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/x86/utils.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index 8829a907eee0..90c3d2eab9e9 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -484,8 +484,28 @@ static int acpi_dmi_skip_serdev_enumeration(struct device *controller_parent, bo int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) { + struct acpi_device *adev = ACPI_COMPANION(controller_parent); + *skip = false; + /* + * The DELL0501 ACPI HID represents an UART (CID is set to PNP0501) with + * a backlight-controller attached. There is no separate ACPI device with + * an UartSerialBusV2() resource to model the backlight-controller. + * Set skip to true so that the tty core creates a serdev ctrl device. + * The backlight driver will manually create the serdev client device. + */ + if (acpi_dev_hid_match(adev, "DELL0501")) { + *skip = true; + /* + * Create a platform dev for dell-uart-backlight to bind to. + * This is a static device, so no need to store the result. + */ + platform_device_register_simple("dell-uart-backlight", PLATFORM_DEVID_NONE, + NULL, 0); + return 0; + } + return acpi_dmi_skip_serdev_enumeration(controller_parent, skip); } EXPORT_SYMBOL_GPL(acpi_quirk_skip_serdev_enumeration); From 0cc46f1a52b4220ec11d98a01575909ca820a7b4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 16 Feb 2024 17:38:55 +0100 Subject: [PATCH 0656/1038] ACPI: Drop the custom_method debugfs interface The ACPI custom_method debugfs interface is security-sensitive and concurrent access to it is broken [1]. Moreover, the recipe for preparing a customized version of a given control method has changed at one point due to ACPICA changes, which has not been reflected in its documentation, so whoever used it before has had to adapt and it had gone unnoticed for a long time. This interface was a bad idea to start with and its implementation is fragile at the design level. It's been always conceptually questionable, problematic from the security standpoint and implemented poorly. Patches fixing its most apparent functional issues (for example, [2]) don't actually address much of the above. Granted, at the time it was introduced, there was no alternative, but there is the AML debugger in the kernel now and there is the configfs interface allowing custom ACPI tables to be loaded. The former can be used for extensive AML debugging and the latter can be use for testing new AML. [3] Accordingly, drop custom_method along with its (outdated anyway) documentation. Link: https://lore.kernel.org/linux-acpi/20221227063335.61474-1-zh.nvgt@gmail.com/ # [1] Link: https://lore.kernel.org/linux-acpi/20231111132402.4142-1-shiqiang.deng213@gmail.com/ [2] Link: https://stackoverflow.com/questions/62849113/how-to-unload-an-overlay-loaded-using-acpi-config-sysfs # [3] Reported-by: Hang Zhang Signed-off-by: Rafael J. Wysocki Reviewed-by: Zhang Rui Reviewed-by: Kees Cook --- Documentation/firmware-guide/acpi/index.rst | 1 - .../acpi/method-customizing.rst | 89 --------------- drivers/acpi/Kconfig | 14 --- drivers/acpi/Makefile | 1 - drivers/acpi/custom_method.c | 103 ------------------ 5 files changed, 208 deletions(-) delete mode 100644 Documentation/firmware-guide/acpi/method-customizing.rst delete mode 100644 drivers/acpi/custom_method.c diff --git a/Documentation/firmware-guide/acpi/index.rst b/Documentation/firmware-guide/acpi/index.rst index b6a42f4ffe03..b246902f523f 100644 --- a/Documentation/firmware-guide/acpi/index.rst +++ b/Documentation/firmware-guide/acpi/index.rst @@ -14,7 +14,6 @@ ACPI Support dsd/phy enumeration osi - method-customizing method-tracing DSD-properties-rules debug diff --git a/Documentation/firmware-guide/acpi/method-customizing.rst b/Documentation/firmware-guide/acpi/method-customizing.rst deleted file mode 100644 index de3ebcaed4cf..000000000000 --- a/Documentation/firmware-guide/acpi/method-customizing.rst +++ /dev/null @@ -1,89 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -======================================= -Linux ACPI Custom Control Method How To -======================================= - -:Author: Zhang Rui - - -Linux supports customizing ACPI control methods at runtime. - -Users can use this to: - -1. override an existing method which may not work correctly, - or just for debugging purposes. -2. insert a completely new method in order to create a missing - method such as _OFF, _ON, _STA, _INI, etc. - -For these cases, it is far simpler to dynamically install a single -control method rather than override the entire DSDT, because kernel -rebuild/reboot is not needed and test result can be got in minutes. - -.. note:: - - - Only ACPI METHOD can be overridden, any other object types like - "Device", "OperationRegion", are not recognized. Methods - declared inside scope operators are also not supported. - - - The same ACPI control method can be overridden for many times, - and it's always the latest one that used by Linux/kernel. - - - To get the ACPI debug object output (Store (AAAA, Debug)), - please run:: - - echo 1 > /sys/module/acpi/parameters/aml_debug_output - - -1. override an existing method -============================== -a) get the ACPI table via ACPI sysfs I/F. e.g. to get the DSDT, - just run "cat /sys/firmware/acpi/tables/DSDT > /tmp/dsdt.dat" -b) disassemble the table by running "iasl -d dsdt.dat". -c) rewrite the ASL code of the method and save it in a new file, -d) package the new file (psr.asl) to an ACPI table format. - Here is an example of a customized \_SB._AC._PSR method:: - - DefinitionBlock ("", "SSDT", 1, "", "", 0x20080715) - { - Method (\_SB_.AC._PSR, 0, NotSerialized) - { - Store ("In AC _PSR", Debug) - Return (ACON) - } - } - - Note that the full pathname of the method in ACPI namespace - should be used. -e) assemble the file to generate the AML code of the method. - e.g. "iasl -vw 6084 psr.asl" (psr.aml is generated as a result) - If parameter "-vw 6084" is not supported by your iASL compiler, - please try a newer version. -f) mount debugfs by "mount -t debugfs none /sys/kernel/debug" -g) override the old method via the debugfs by running - "cat /tmp/psr.aml > /sys/kernel/debug/acpi/custom_method" - -2. insert a new method -====================== -This is easier than overriding an existing method. -We just need to create the ASL code of the method we want to -insert and then follow the step c) ~ g) in section 1. - -3. undo your changes -==================== -The "undo" operation is not supported for a new inserted method -right now, i.e. we can not remove a method currently. -For an overridden method, in order to undo your changes, please -save a copy of the method original ASL code in step c) section 1, -and redo step c) ~ g) to override the method with the original one. - - -.. note:: We can use a kernel with multiple custom ACPI method running, - But each individual write to debugfs can implement a SINGLE - method override. i.e. if we want to insert/override multiple - ACPI methods, we need to redo step c) ~ g) for multiple times. - -.. note:: Be aware that root can mis-use this driver to modify arbitrary - memory and gain additional rights, if root's privileges got - restricted (for example if root is not allowed to load additional - modules after boot). diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 3c3f8037ebed..c645bb453f3b 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -449,20 +449,6 @@ config ACPI_HED which is used to report some hardware errors notified via SCI, mainly the corrected errors. -config ACPI_CUSTOM_METHOD - tristate "Allow ACPI methods to be inserted/replaced at run time" - depends on DEBUG_FS - help - This debug facility allows ACPI AML methods to be inserted and/or - replaced without rebooting the system. For details refer to: - Documentation/firmware-guide/acpi/method-customizing.rst. - - NOTE: This option is security sensitive, because it allows arbitrary - kernel memory to be written to by root (uid=0) users, allowing them - to bypass certain security measures (e.g. if root is not allowed to - load additional kernel modules after boot, this feature may be used - to override that restriction). - config ACPI_BGRT bool "Boottime Graphics Resource Table support" depends on EFI && (X86 || ARM64) diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 12ef8180d272..8cc8c0d9c873 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -101,7 +101,6 @@ obj-$(CONFIG_ACPI_SBS) += sbshc.o obj-$(CONFIG_ACPI_SBS) += sbs.o obj-$(CONFIG_ACPI_HED) += hed.o obj-$(CONFIG_ACPI_EC_DEBUGFS) += ec_sys.o -obj-$(CONFIG_ACPI_CUSTOM_METHOD)+= custom_method.o obj-$(CONFIG_ACPI_BGRT) += bgrt.o obj-$(CONFIG_ACPI_CPPC_LIB) += cppc_acpi.o obj-$(CONFIG_ACPI_SPCR_TABLE) += spcr.o diff --git a/drivers/acpi/custom_method.c b/drivers/acpi/custom_method.c deleted file mode 100644 index d39a9b474727..000000000000 --- a/drivers/acpi/custom_method.c +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * custom_method.c - debugfs interface for customizing ACPI control method - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "internal.h" - -MODULE_LICENSE("GPL"); - -static struct dentry *cm_dentry; - -/* /sys/kernel/debug/acpi/custom_method */ - -static ssize_t cm_write(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - static char *buf; - static u32 max_size; - static u32 uncopied_bytes; - - struct acpi_table_header table; - acpi_status status; - int ret; - - ret = security_locked_down(LOCKDOWN_ACPI_TABLES); - if (ret) - return ret; - - if (!(*ppos)) { - /* parse the table header to get the table length */ - if (count <= sizeof(struct acpi_table_header)) - return -EINVAL; - if (copy_from_user(&table, user_buf, - sizeof(struct acpi_table_header))) - return -EFAULT; - uncopied_bytes = max_size = table.length; - /* make sure the buf is not allocated */ - kfree(buf); - buf = kzalloc(max_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - } - - if (buf == NULL) - return -EINVAL; - - if ((*ppos > max_size) || - (*ppos + count > max_size) || - (*ppos + count < count) || - (count > uncopied_bytes)) { - kfree(buf); - buf = NULL; - return -EINVAL; - } - - if (copy_from_user(buf + (*ppos), user_buf, count)) { - kfree(buf); - buf = NULL; - return -EFAULT; - } - - uncopied_bytes -= count; - *ppos += count; - - if (!uncopied_bytes) { - status = acpi_install_method(buf); - kfree(buf); - buf = NULL; - if (ACPI_FAILURE(status)) - return -EINVAL; - add_taint(TAINT_OVERRIDDEN_ACPI_TABLE, LOCKDEP_NOW_UNRELIABLE); - } - - return count; -} - -static const struct file_operations cm_fops = { - .write = cm_write, - .llseek = default_llseek, -}; - -static int __init acpi_custom_method_init(void) -{ - cm_dentry = debugfs_create_file("custom_method", S_IWUSR, - acpi_debugfs_dir, NULL, &cm_fops); - return 0; -} - -static void __exit acpi_custom_method_exit(void) -{ - debugfs_remove(cm_dentry); -} - -module_init(acpi_custom_method_init); -module_exit(acpi_custom_method_exit); From aac1b805a61ced530b4d427354a205b4bbff69c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:36 +0100 Subject: [PATCH 0657/1038] pwm: sprd: Drop duplicated tracking of the parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pwmchip stores a pointer to the parent device, so there is no need to store another copy in driver private data. Drop struct sprd_pwm_chip::dev and use the pwm_chip's parent pointer instead. Tested-by: Chunyan Zhang Link: https://lore.kernel.org/r/f85771f4bf659c0fdee30cf117b87fd877bad5e4.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sprd.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-sprd.c b/drivers/pwm/pwm-sprd.c index 2e87666ad2b9..a38f50c3dc00 100644 --- a/drivers/pwm/pwm-sprd.c +++ b/drivers/pwm/pwm-sprd.c @@ -34,7 +34,6 @@ struct sprd_pwm_chn { struct sprd_pwm_chip { void __iomem *base; - struct device *dev; struct pwm_chip chip; struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; }; @@ -85,7 +84,7 @@ static int sprd_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, */ ret = clk_bulk_prepare_enable(SPRD_PWM_CHN_CLKS_NUM, chn->clks); if (ret) { - dev_err(spc->dev, "failed to enable pwm%u clocks\n", + dev_err(pwmchip_parent(chip), "failed to enable pwm%u clocks\n", pwm->hwpwm); return ret; } @@ -182,7 +181,7 @@ static int sprd_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_bulk_prepare_enable(SPRD_PWM_CHN_CLKS_NUM, chn->clks); if (ret) { - dev_err(spc->dev, + dev_err(pwmchip_parent(chip), "failed to enable pwm%u clocks\n", pwm->hwpwm); return ret; @@ -265,7 +264,6 @@ static int sprd_pwm_probe(struct platform_device *pdev) if (IS_ERR(spc->base)) return PTR_ERR(spc->base); - spc->dev = &pdev->dev; memcpy(spc->chn, chn, sizeof(chn)); spc->chip.dev = &pdev->dev; From f2075c0b1a8f8285b72d2dc800345e854d3a711b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:37 +0100 Subject: [PATCH 0658/1038] pwm: sprd: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-sprd driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Tested-by: Chunyan Zhang Link: https://lore.kernel.org/r/543213f44686ee72d8f88897bf2ca616e837ae44.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sprd.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-sprd.c b/drivers/pwm/pwm-sprd.c index a38f50c3dc00..4c76ca5e4cdd 100644 --- a/drivers/pwm/pwm-sprd.c +++ b/drivers/pwm/pwm-sprd.c @@ -34,13 +34,12 @@ struct sprd_pwm_chn { struct sprd_pwm_chip { void __iomem *base; - struct pwm_chip chip; struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; }; static inline struct sprd_pwm_chip* sprd_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct sprd_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -248,6 +247,7 @@ static int sprd_pwm_clk_init(struct device *dev, static int sprd_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct sprd_pwm_chip *spc; struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; int ret, npwm; @@ -256,9 +256,10 @@ static int sprd_pwm_probe(struct platform_device *pdev) if (npwm < 0) return npwm; - spc = devm_kzalloc(&pdev->dev, sizeof(*spc), GFP_KERNEL); - if (!spc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*spc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + spc = sprd_pwm_from_chip(chip); spc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(spc->base)) @@ -266,11 +267,9 @@ static int sprd_pwm_probe(struct platform_device *pdev) memcpy(spc->chn, chn, sizeof(chn)); - spc->chip.dev = &pdev->dev; - spc->chip.ops = &sprd_pwm_ops; - spc->chip.npwm = npwm; + chip->ops = &sprd_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &spc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) dev_err(&pdev->dev, "failed to add PWM chip\n"); From 54272761ce7c475fa30a31b59b0cb89f7652b39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:38 +0100 Subject: [PATCH 0659/1038] pwm: sti: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct sti_pwm_chip. Use the pwm_chip as driver data instead of the sti_pwm_chip to get access to the pwm_chip in sti_pwm_remove() without using pc->chip. Link: https://lore.kernel.org/r/56d53372aacff6871df4d6c6779c9dac94592696.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sti.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-sti.c b/drivers/pwm/pwm-sti.c index 69b1113c6b82..826eb547cc96 100644 --- a/drivers/pwm/pwm-sti.c +++ b/drivers/pwm/pwm-sti.c @@ -570,6 +570,7 @@ static int sti_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct sti_pwm_compat_data *cdata; + struct pwm_chip *chip; struct sti_pwm_chip *pc; unsigned int i; int irq, ret; @@ -577,6 +578,7 @@ static int sti_pwm_probe(struct platform_device *pdev) pc = devm_kzalloc(dev, sizeof(*pc), GFP_KERNEL); if (!pc) return -ENOMEM; + chip = &pc->chip; cdata = devm_kzalloc(dev, sizeof(*cdata), GFP_KERNEL); if (!cdata) @@ -653,9 +655,9 @@ static int sti_pwm_probe(struct platform_device *pdev) return -ENOMEM; } - pc->chip.dev = dev; - pc->chip.ops = &sti_pwm_ops; - pc->chip.npwm = max(cdata->pwm_num_devs, cdata->cpt_num_devs); + chip->dev = dev; + chip->ops = &sti_pwm_ops; + chip->npwm = max(cdata->pwm_num_devs, cdata->cpt_num_devs); for (i = 0; i < cdata->cpt_num_devs; i++) { struct sti_cpt_ddata *ddata = &cdata->ddata[i]; @@ -664,23 +666,24 @@ static int sti_pwm_probe(struct platform_device *pdev) mutex_init(&ddata->lock); } - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { clk_unprepare(pc->pwm_clk); clk_unprepare(pc->cpt_clk); return ret; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); return 0; } static void sti_pwm_remove(struct platform_device *pdev) { - struct sti_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct sti_pwm_chip *pc = to_sti_pwmchip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); clk_unprepare(pc->pwm_clk); clk_unprepare(pc->cpt_clk); From 3f7dc7d863cd1d7a35165a0aeb8e28a637390097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:39 +0100 Subject: [PATCH 0660/1038] pwm: sti: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-sti driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/da6fbb5e98e988400e037b0d2ac0c1749822d702.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sti.c | 50 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/drivers/pwm/pwm-sti.c b/drivers/pwm/pwm-sti.c index 826eb547cc96..39d80da0e14a 100644 --- a/drivers/pwm/pwm-sti.c +++ b/drivers/pwm/pwm-sti.c @@ -94,7 +94,6 @@ struct sti_pwm_chip { struct regmap_field *pwm_cpt_en; struct regmap_field *pwm_cpt_int_en; struct regmap_field *pwm_cpt_int_stat; - struct pwm_chip chip; struct pwm_device *cur; unsigned long configured; unsigned int en_count; @@ -114,7 +113,7 @@ static const struct reg_field sti_pwm_regfields[MAX_REGFIELDS] = { static inline struct sti_pwm_chip *to_sti_pwmchip(struct pwm_chip *chip) { - return container_of(chip, struct sti_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -507,23 +506,7 @@ static int sti_pwm_probe_dt(struct sti_pwm_chip *pc) { struct device *dev = pc->dev; const struct reg_field *reg_fields; - struct device_node *np = dev->of_node; struct sti_pwm_compat_data *cdata = pc->cdata; - u32 num_devs; - int ret; - - ret = of_property_read_u32(np, "st,pwm-num-chan", &num_devs); - if (!ret) - cdata->pwm_num_devs = num_devs; - - ret = of_property_read_u32(np, "st,capture-num-chan", &num_devs); - if (!ret) - cdata->cpt_num_devs = num_devs; - - if (!cdata->pwm_num_devs && !cdata->cpt_num_devs) { - dev_err(dev, "No channels configured\n"); - return -EINVAL; - } reg_fields = cdata->reg_fields; @@ -569,16 +552,33 @@ static const struct regmap_config sti_pwm_regmap_config = { static int sti_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + u32 num_devs; + unsigned int pwm_num_devs = 0; + unsigned int cpt_num_devs = 0; struct sti_pwm_compat_data *cdata; struct pwm_chip *chip; struct sti_pwm_chip *pc; unsigned int i; int irq, ret; - pc = devm_kzalloc(dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; - chip = &pc->chip; + ret = of_property_read_u32(np, "st,pwm-num-chan", &num_devs); + if (!ret) + pwm_num_devs = num_devs; + + ret = of_property_read_u32(np, "st,capture-num-chan", &num_devs); + if (!ret) + cpt_num_devs = num_devs; + + if (!pwm_num_devs && !cpt_num_devs) { + dev_err(dev, "No channels configured\n"); + return -EINVAL; + } + + chip = devm_pwmchip_alloc(dev, max(pwm_num_devs, cpt_num_devs), sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_sti_pwmchip(chip); cdata = devm_kzalloc(dev, sizeof(*cdata), GFP_KERNEL); if (!cdata) @@ -611,8 +611,8 @@ static int sti_pwm_probe(struct platform_device *pdev) cdata->reg_fields = sti_pwm_regfields; cdata->max_prescale = 0xff; cdata->max_pwm_cnt = 255; - cdata->pwm_num_devs = 0; - cdata->cpt_num_devs = 0; + cdata->pwm_num_devs = pwm_num_devs; + cdata->cpt_num_devs = cpt_num_devs; pc->cdata = cdata; pc->dev = dev; @@ -655,9 +655,7 @@ static int sti_pwm_probe(struct platform_device *pdev) return -ENOMEM; } - chip->dev = dev; chip->ops = &sti_pwm_ops; - chip->npwm = max(cdata->pwm_num_devs, cdata->cpt_num_devs); for (i = 0; i < cdata->cpt_num_devs; i++) { struct sti_cpt_ddata *ddata = &cdata->ddata[i]; From da665d25f4cbf82a847a9fa85eb8554ebc9e8bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:40 +0100 Subject: [PATCH 0661/1038] pwm: stm32: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct stm32_pwm *priv just obtained from it. This also has the advantage of not using struct stm32_pwm::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/54ace92a3c02d22f15a79c7ecf00c29f28386a33.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 5f10cba492ec..202114c00ad3 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -170,7 +170,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_enable(priv->clk); if (ret) { - dev_err(priv->chip.dev, "failed to enable counter clock\n"); + dev_err(chip->dev, "failed to enable counter clock\n"); goto unlock; } From fbde128940991873847c92ed86a323adbce61ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:41 +0100 Subject: [PATCH 0662/1038] pwm: stm32: Change prototype of a helper to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given stm32_pwm. To just not have to do that, rework stm32_pwm_raw_capture() to take a pwm_chip. Link: https://lore.kernel.org/r/33790c64563cb0434d7156d96f189c6037b3eb0b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 202114c00ad3..ec15bc51d930 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -90,11 +90,12 @@ static u32 active_channels(struct stm32_pwm *dev) * - Period = t2 - t0 * - Duty cycle = t1 - t0 */ -static int stm32_pwm_raw_capture(struct stm32_pwm *priv, struct pwm_device *pwm, +static int stm32_pwm_raw_capture(struct pwm_chip *chip, struct pwm_device *pwm, unsigned long tmo_ms, u32 *raw_prd, u32 *raw_dty) { - struct device *parent = priv->chip.dev->parent; + struct stm32_pwm *priv = to_stm32_pwm_dev(chip); + struct device *parent = pwmchip_parent(chip)->parent; enum stm32_timers_dmas dma_id; u32 ccen, ccr; int ret; @@ -170,7 +171,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_enable(priv->clk); if (ret) { - dev_err(chip->dev, "failed to enable counter clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable counter clock\n"); goto unlock; } @@ -208,7 +209,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, TIM_CCER_CC12P : TIM_CCER_CC34P, pwm->hwpwm < 2 ? TIM_CCER_CC2P : TIM_CCER_CC4P); - ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, &raw_dty); + ret = stm32_pwm_raw_capture(chip, pwm, tmo_ms, &raw_prd, &raw_dty); if (ret) goto stop; @@ -229,7 +230,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, /* 2nd measure with new scale */ psc /= scale; regmap_write(priv->regmap, TIM_PSC, psc); - ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, + ret = stm32_pwm_raw_capture(chip, pwm, tmo_ms, &raw_prd, &raw_dty); if (ret) goto stop; @@ -257,7 +258,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, FIELD_PREP(TIM_CCMR_IC1PSC, icpsc) | FIELD_PREP(TIM_CCMR_IC2PSC, icpsc)); - ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, &raw_dty); + ret = stm32_pwm_raw_capture(chip, pwm, tmo_ms, &raw_prd, &raw_dty); if (ret) goto stop; From f29430710d92a9bdf4234dbc94f452cd72704fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:42 +0100 Subject: [PATCH 0663/1038] pwm: stm32: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct stm32_pwm. Use the pwm_chip as driver data instead of the stm32_pwm to get access to the pwm_chip in stm32_pwm_suspend() without using priv->chip. Link: https://lore.kernel.org/r/3db96cd915d9d8fc350a7193c0d55dd109b1f035.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index ec15bc51d930..4d12f3d849cd 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -630,6 +630,7 @@ static int stm32_pwm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct stm32_timers *ddata = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct stm32_pwm *priv; unsigned int num_enabled; unsigned int i; @@ -638,6 +639,7 @@ static int stm32_pwm_probe(struct platform_device *pdev) priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; + chip = &priv->chip; mutex_init(&priv->lock); priv->regmap = ddata->regmap; @@ -653,37 +655,38 @@ static int stm32_pwm_probe(struct platform_device *pdev) stm32_pwm_detect_complementary(priv); - priv->chip.dev = dev; - priv->chip.ops = &stm32pwm_ops; - priv->chip.npwm = stm32_pwm_detect_channels(priv, &num_enabled); + chip->dev = dev; + chip->ops = &stm32pwm_ops; + chip->npwm = stm32_pwm_detect_channels(priv, &num_enabled); /* Initialize clock refcount to number of enabled PWM channels. */ for (i = 0; i < num_enabled; i++) clk_enable(priv->clk); - ret = devm_pwmchip_add(dev, &priv->chip); + ret = devm_pwmchip_add(dev, chip); if (ret < 0) return ret; - platform_set_drvdata(pdev, priv); + platform_set_drvdata(pdev, chip); return 0; } static int stm32_pwm_suspend(struct device *dev) { - struct stm32_pwm *priv = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct stm32_pwm *priv = to_stm32_pwm_dev(chip); unsigned int i; u32 ccer, mask; /* Look for active channels */ ccer = active_channels(priv); - for (i = 0; i < priv->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { mask = TIM_CCER_CC1E << (i * 4); if (ccer & mask) { dev_err(dev, "PWM %u still in use by consumer %s\n", - i, priv->chip.pwms[i].label); + i, chip->pwms[i].label); return -EBUSY; } } @@ -693,7 +696,8 @@ static int stm32_pwm_suspend(struct device *dev) static int stm32_pwm_resume(struct device *dev) { - struct stm32_pwm *priv = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct stm32_pwm *priv = to_stm32_pwm_dev(chip); int ret; ret = pinctrl_pm_select_default_state(dev); From e315bf700b404c3b1b2efce63cb93fca347a215c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:43 +0100 Subject: [PATCH 0664/1038] pwm: stm32: Change prototype of helper that detects npwm to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the stm32 pwm driver is converted to pwmchip_alloc(), the number of available PWM lines must be known before the driver private data can be allocated. So rework the helper function that determines this number to not take the driver private data struct as input parameter. Link: https://lore.kernel.org/r/13d4d3e90a9ee1bcd04674dfdc16f242615b8320.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 4d12f3d849cd..1440b706ee57 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -606,7 +606,7 @@ static void stm32_pwm_detect_complementary(struct stm32_pwm *priv) priv->have_complementary_output = (ccer != 0); } -static unsigned int stm32_pwm_detect_channels(struct stm32_pwm *priv, +static unsigned int stm32_pwm_detect_channels(struct regmap *regmap, unsigned int *num_enabled) { u32 ccer, ccer_backup; @@ -615,10 +615,10 @@ static unsigned int stm32_pwm_detect_channels(struct stm32_pwm *priv, * If channels enable bits don't exist writing 1 will have no * effect so we can detect and count them. */ - regmap_read(priv->regmap, TIM_CCER, &ccer_backup); - regmap_set_bits(priv->regmap, TIM_CCER, TIM_CCER_CCXE); - regmap_read(priv->regmap, TIM_CCER, &ccer); - regmap_write(priv->regmap, TIM_CCER, ccer_backup); + regmap_read(regmap, TIM_CCER, &ccer_backup); + regmap_set_bits(regmap, TIM_CCER, TIM_CCER_CCXE); + regmap_read(regmap, TIM_CCER, &ccer); + regmap_write(regmap, TIM_CCER, ccer_backup); *num_enabled = hweight32(ccer_backup & TIM_CCER_CCXE); @@ -657,7 +657,7 @@ static int stm32_pwm_probe(struct platform_device *pdev) chip->dev = dev; chip->ops = &stm32pwm_ops; - chip->npwm = stm32_pwm_detect_channels(priv, &num_enabled); + chip->npwm = stm32_pwm_detect_channels(ddata->regmap, &num_enabled); /* Initialize clock refcount to number of enabled PWM channels. */ for (i = 0; i < num_enabled; i++) From 174821b7be22ed11659d6791c04b8aca13424ed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:44 +0100 Subject: [PATCH 0665/1038] pwm: stm32: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-stm32 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/59e5dfff2b878cc8590e286572672e4f10e35380.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 1440b706ee57..0c028d17c075 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -27,7 +27,6 @@ struct stm32_breakinput { }; struct stm32_pwm { - struct pwm_chip chip; struct mutex lock; /* protect pwm config/enable */ struct clk *clk; struct regmap *regmap; @@ -40,7 +39,7 @@ struct stm32_pwm { static inline struct stm32_pwm *to_stm32_pwm_dev(struct pwm_chip *chip) { - return container_of(chip, struct stm32_pwm, chip); + return pwmchip_get_drvdata(chip); } static u32 active_channels(struct stm32_pwm *dev) @@ -632,14 +631,16 @@ static int stm32_pwm_probe(struct platform_device *pdev) struct stm32_timers *ddata = dev_get_drvdata(pdev->dev.parent); struct pwm_chip *chip; struct stm32_pwm *priv; - unsigned int num_enabled; + unsigned int npwm, num_enabled; unsigned int i; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - chip = &priv->chip; + npwm = stm32_pwm_detect_channels(ddata->regmap, &num_enabled); + + chip = devm_pwmchip_alloc(dev, npwm, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_stm32_pwm_dev(chip); mutex_init(&priv->lock); priv->regmap = ddata->regmap; @@ -655,9 +656,7 @@ static int stm32_pwm_probe(struct platform_device *pdev) stm32_pwm_detect_complementary(priv); - chip->dev = dev; chip->ops = &stm32pwm_ops; - chip->npwm = stm32_pwm_detect_channels(ddata->regmap, &num_enabled); /* Initialize clock refcount to number of enabled PWM channels. */ for (i = 0; i < num_enabled; i++) From 3e4224a98b505ea4c6c1ea77ffda0457a7adaacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:45 +0100 Subject: [PATCH 0666/1038] pwm: stm32-lp: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct stm32_pwm_lp *priv just obtained from it. This also has the advantage of not using struct stm32_pwm_lp::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/9ad2399e1a683a6344b12d7f70498393b8f8b9de.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32-lp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c index 439068f3eca1..bbab6be314a8 100644 --- a/drivers/pwm/pwm-stm32-lp.c +++ b/drivers/pwm/pwm-stm32-lp.c @@ -61,7 +61,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, do_div(div, NSEC_PER_SEC); if (!div) { /* Clock is too slow to achieve requested period. */ - dev_dbg(priv->chip.dev, "Can't reach %llu ns\n", state->period); + dev_dbg(chip->dev, "Can't reach %llu ns\n", state->period); return -EINVAL; } @@ -69,7 +69,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, while (div > STM32_LPTIM_MAX_ARR) { presc++; if ((1 << presc) > STM32_LPTIM_MAX_PRESCALER) { - dev_err(priv->chip.dev, "max prescaler exceeded\n"); + dev_err(chip->dev, "max prescaler exceeded\n"); return -EINVAL; } div = prd >> presc; @@ -130,7 +130,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, (val & STM32_LPTIM_CMPOK_ARROK) == STM32_LPTIM_CMPOK_ARROK, 100, 1000); if (ret) { - dev_err(priv->chip.dev, "ARR/CMP registers write issue\n"); + dev_err(chip->dev, "ARR/CMP registers write issue\n"); goto err; } ret = regmap_write(priv->regmap, STM32_LPTIM_ICR, From 8f5e2dcbe234c9de294e87d03aef671d4bcbe21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:46 +0100 Subject: [PATCH 0667/1038] pwm: stm32-lp: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct stm32_pwm_lp. Use the pwm_chip as driver data instead of the stm32_pwm_lp to get access to the pwm_chip in stm32_pwm_lp_suspend() without using priv->chip. Link: https://lore.kernel.org/r/df47d1aff9b529c9a4762b6ba339a18cecba1497.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32-lp.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c index bbab6be314a8..b46d8193dd0f 100644 --- a/drivers/pwm/pwm-stm32-lp.c +++ b/drivers/pwm/pwm-stm32-lp.c @@ -197,6 +197,7 @@ static int stm32_pwm_lp_probe(struct platform_device *pdev) { struct stm32_lptimer *ddata = dev_get_drvdata(pdev->dev.parent); struct stm32_pwm_lp *priv; + struct pwm_chip *chip; int ret; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); @@ -205,28 +206,29 @@ static int stm32_pwm_lp_probe(struct platform_device *pdev) priv->regmap = ddata->regmap; priv->clk = ddata->clk; - priv->chip.dev = &pdev->dev; - priv->chip.ops = &stm32_pwm_lp_ops; - priv->chip.npwm = 1; + chip = &priv->chip; + chip->dev = &pdev->dev; + chip->ops = &stm32_pwm_lp_ops; + chip->npwm = 1; - ret = devm_pwmchip_add(&pdev->dev, &priv->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return ret; - platform_set_drvdata(pdev, priv); + platform_set_drvdata(pdev, chip); return 0; } static int stm32_pwm_lp_suspend(struct device *dev) { - struct stm32_pwm_lp *priv = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); struct pwm_state state; - pwm_get_state(&priv->chip.pwms[0], &state); + pwm_get_state(&chip->pwms[0], &state); if (state.enabled) { dev_err(dev, "The consumer didn't stop us (%s)\n", - priv->chip.pwms[0].label); + chip->pwms[0].label); return -EBUSY; } From 0e89637a4fb9010cd3f96ddff2f9e466e87fac8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:47 +0100 Subject: [PATCH 0668/1038] pwm: stm32-lp: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/d79148ed49389c657b72df05758032be2b516ceb.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32-lp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c index b46d8193dd0f..958043bc5c92 100644 --- a/drivers/pwm/pwm-stm32-lp.c +++ b/drivers/pwm/pwm-stm32-lp.c @@ -61,7 +61,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, do_div(div, NSEC_PER_SEC); if (!div) { /* Clock is too slow to achieve requested period. */ - dev_dbg(chip->dev, "Can't reach %llu ns\n", state->period); + dev_dbg(pwmchip_parent(chip), "Can't reach %llu ns\n", state->period); return -EINVAL; } @@ -69,7 +69,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, while (div > STM32_LPTIM_MAX_ARR) { presc++; if ((1 << presc) > STM32_LPTIM_MAX_PRESCALER) { - dev_err(chip->dev, "max prescaler exceeded\n"); + dev_err(pwmchip_parent(chip), "max prescaler exceeded\n"); return -EINVAL; } div = prd >> presc; @@ -130,7 +130,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, (val & STM32_LPTIM_CMPOK_ARROK) == STM32_LPTIM_CMPOK_ARROK, 100, 1000); if (ret) { - dev_err(chip->dev, "ARR/CMP registers write issue\n"); + dev_err(pwmchip_parent(chip), "ARR/CMP registers write issue\n"); goto err; } ret = regmap_write(priv->regmap, STM32_LPTIM_ICR, From 5d481e0742dde0d94e3facc39a243ed50c6ce948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:48 +0100 Subject: [PATCH 0669/1038] pwm: stm32-lp: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-stm32-lp driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/04af7b3d00bc932dd025200a3bf74527c29ca47a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stm32-lp.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c index 958043bc5c92..989731256f50 100644 --- a/drivers/pwm/pwm-stm32-lp.c +++ b/drivers/pwm/pwm-stm32-lp.c @@ -18,14 +18,13 @@ #include struct stm32_pwm_lp { - struct pwm_chip chip; struct clk *clk; struct regmap *regmap; }; static inline struct stm32_pwm_lp *to_stm32_pwm_lp(struct pwm_chip *chip) { - return container_of(chip, struct stm32_pwm_lp, chip); + return pwmchip_get_drvdata(chip); } /* STM32 Low-Power Timer is preceded by a configurable power-of-2 prescaler */ @@ -200,16 +199,14 @@ static int stm32_pwm_lp_probe(struct platform_device *pdev) struct pwm_chip *chip; int ret; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_stm32_pwm_lp(chip); priv->regmap = ddata->regmap; priv->clk = ddata->clk; - chip = &priv->chip; - chip->dev = &pdev->dev; chip->ops = &stm32_pwm_lp_ops; - chip->npwm = 1; ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) From c3492db2884e215d596b5809df8b56dbf8f7b91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:49 +0100 Subject: [PATCH 0670/1038] pwm: stmpe: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/2136fbdf9b1e6bac479b935b439e2be73a003b97.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stmpe.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-stmpe.c b/drivers/pwm/pwm-stmpe.c index 19c0c0f39675..05f3f38031ee 100644 --- a/drivers/pwm/pwm-stmpe.c +++ b/drivers/pwm/pwm-stmpe.c @@ -44,7 +44,7 @@ static int stmpe_24xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = stmpe_reg_read(stmpe_pwm->stmpe, STMPE24XX_PWMCS); if (ret < 0) { - dev_dbg(chip->dev, "error reading PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error reading PWM#%u control\n", pwm->hwpwm); return ret; } @@ -53,7 +53,7 @@ static int stmpe_24xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = stmpe_reg_write(stmpe_pwm->stmpe, STMPE24XX_PWMCS, value); if (ret) { - dev_dbg(chip->dev, "error writing PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error writing PWM#%u control\n", pwm->hwpwm); return ret; } @@ -70,7 +70,7 @@ static int stmpe_24xx_pwm_disable(struct pwm_chip *chip, ret = stmpe_reg_read(stmpe_pwm->stmpe, STMPE24XX_PWMCS); if (ret < 0) { - dev_dbg(chip->dev, "error reading PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error reading PWM#%u control\n", pwm->hwpwm); return ret; } @@ -79,7 +79,7 @@ static int stmpe_24xx_pwm_disable(struct pwm_chip *chip, ret = stmpe_reg_write(stmpe_pwm->stmpe, STMPE24XX_PWMCS, value); if (ret) - dev_dbg(chip->dev, "error writing PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error writing PWM#%u control\n", pwm->hwpwm); return ret; } @@ -125,7 +125,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = stmpe_set_altfunc(stmpe_pwm->stmpe, BIT(pin), STMPE_BLOCK_PWM); if (ret) { - dev_err(chip->dev, "unable to connect PWM#%u to pin\n", + dev_err(pwmchip_parent(chip), "unable to connect PWM#%u to pin\n", pwm->hwpwm); return ret; } @@ -150,7 +150,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, return -ENODEV; } - dev_dbg(chip->dev, "PWM#%u: config duty %d ns, period %d ns\n", + dev_dbg(pwmchip_parent(chip), "PWM#%u: config duty %d ns, period %d ns\n", pwm->hwpwm, duty_ns, period_ns); if (duty_ns == 0) { @@ -216,7 +216,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, program[1] = BRANCH; } - dev_dbg(chip->dev, + dev_dbg(pwmchip_parent(chip), "PWM#%u: value = %02x, last_duty = %02x, program=%04x,%04x,%04x\n", pwm->hwpwm, value, last, program[0], program[1], program[2]); @@ -233,7 +233,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = stmpe_reg_write(stmpe_pwm->stmpe, offset, value); if (ret) { - dev_dbg(chip->dev, "error writing register %02x: %d\n", + dev_dbg(pwmchip_parent(chip), "error writing register %02x: %d\n", offset, ret); return ret; } @@ -242,7 +242,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = stmpe_reg_write(stmpe_pwm->stmpe, offset, value); if (ret) { - dev_dbg(chip->dev, "error writing register %02x: %d\n", + dev_dbg(pwmchip_parent(chip), "error writing register %02x: %d\n", offset, ret); return ret; } @@ -255,7 +255,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* Sleep for 200ms so we're sure it will take effect */ msleep(200); - dev_dbg(chip->dev, "programmed PWM#%u, %u bytes\n", pwm->hwpwm, i); + dev_dbg(pwmchip_parent(chip), "programmed PWM#%u, %u bytes\n", pwm->hwpwm, i); return 0; } From f20fb5c85892a7f951ac9be6360f0178b1cc0392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:50 +0100 Subject: [PATCH 0671/1038] pwm: stmpe: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-stmpe driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/7e3dbf3b70126038c0ba16331ca8c07cab575bd3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-stmpe.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/drivers/pwm/pwm-stmpe.c b/drivers/pwm/pwm-stmpe.c index 05f3f38031ee..bb91062d5f1d 100644 --- a/drivers/pwm/pwm-stmpe.c +++ b/drivers/pwm/pwm-stmpe.c @@ -27,13 +27,12 @@ struct stmpe_pwm { struct stmpe *stmpe; - struct pwm_chip chip; u8 last_duty; }; static inline struct stmpe_pwm *to_stmpe_pwm(struct pwm_chip *chip) { - return container_of(chip, struct stmpe_pwm, chip); + return pwmchip_get_drvdata(chip); } static int stmpe_24xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) @@ -292,33 +291,36 @@ static const struct pwm_ops stmpe_24xx_pwm_ops = { static int __init stmpe_pwm_probe(struct platform_device *pdev) { struct stmpe *stmpe = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct stmpe_pwm *stmpe_pwm; int ret; - stmpe_pwm = devm_kzalloc(&pdev->dev, sizeof(*stmpe_pwm), GFP_KERNEL); - if (!stmpe_pwm) - return -ENOMEM; + switch (stmpe->partnum) { + case STMPE2401: + case STMPE2403: + break; + case STMPE1601: + return dev_err_probe(&pdev->dev, -ENODEV, + "STMPE1601 not yet supported\n"); + default: + return dev_err_probe(&pdev->dev, -ENODEV, + "Unknown STMPE PWM\n"); + } + + chip = devm_pwmchip_alloc(&pdev->dev, 3, sizeof(*stmpe_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + stmpe_pwm = to_stmpe_pwm(chip); stmpe_pwm->stmpe = stmpe; - stmpe_pwm->chip.dev = &pdev->dev; - if (stmpe->partnum == STMPE2401 || stmpe->partnum == STMPE2403) { - stmpe_pwm->chip.ops = &stmpe_24xx_pwm_ops; - stmpe_pwm->chip.npwm = 3; - } else { - if (stmpe->partnum == STMPE1601) - dev_err(&pdev->dev, "STMPE1601 not yet supported\n"); - else - dev_err(&pdev->dev, "Unknown STMPE PWM\n"); - - return -ENODEV; - } + chip->ops = &stmpe_24xx_pwm_ops; ret = stmpe_enable(stmpe, STMPE_BLOCK_PWM); if (ret) return ret; - ret = pwmchip_add(&stmpe_pwm->chip); + ret = pwmchip_add(chip); if (ret) { stmpe_disable(stmpe, STMPE_BLOCK_PWM); return ret; From d6ada1d4ca134ad78c5ec5d5ffa518227895dd7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:51 +0100 Subject: [PATCH 0672/1038] pwm: sun4i: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Reviewed-by: Jernej Skrabec Link: https://lore.kernel.org/r/3ddaec73f3abefb45625d0a469026fa8d13da8c0.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sun4i.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 1a439025540d..44edf1ce5739 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -245,7 +245,7 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!cstate.enabled) { ret = clk_prepare_enable(sun4i_pwm->clk); if (ret) { - dev_err(chip->dev, "failed to enable PWM clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable PWM clock\n"); return ret; } } @@ -253,7 +253,7 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = sun4i_pwm_calculate(sun4i_pwm, state, &duty, &period, &prescaler, &bypass); if (ret) { - dev_err(chip->dev, "period exceeds the maximum value\n"); + dev_err(pwmchip_parent(chip), "period exceeds the maximum value\n"); if (!cstate.enabled) clk_disable_unprepare(sun4i_pwm->clk); return ret; From 2d7224f49da15e66fa470fd9e909f5e1f988569b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:52 +0100 Subject: [PATCH 0673/1038] pwm: sun4i: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct sun4i_pwm_chip. Use the pwm_chip as driver data instead of the sun4i_pwm_chip to get access to the pwm_chip in sun4i_pwm_remove() without using sun4ichip->chip. Reviewed-by: Jernej Skrabec Link: https://lore.kernel.org/r/f8e3fb96fe341ba0a4bed982aa731c5c7c355b83.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sun4i.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 44edf1ce5739..c2f579043915 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -384,12 +384,14 @@ MODULE_DEVICE_TABLE(of, sun4i_pwm_dt_ids); static int sun4i_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct sun4i_pwm_chip *sun4ichip; int ret; sun4ichip = devm_kzalloc(&pdev->dev, sizeof(*sun4ichip), GFP_KERNEL); if (!sun4ichip) return -ENOMEM; + chip = &sun4ichip->chip; sun4ichip->data = of_device_get_match_data(&pdev->dev); if (!sun4ichip->data) @@ -451,19 +453,19 @@ static int sun4i_pwm_probe(struct platform_device *pdev) goto err_bus; } - sun4ichip->chip.dev = &pdev->dev; - sun4ichip->chip.ops = &sun4i_pwm_ops; - sun4ichip->chip.npwm = sun4ichip->data->npwm; + chip->dev = &pdev->dev; + chip->ops = &sun4i_pwm_ops; + chip->npwm = sun4ichip->data->npwm; spin_lock_init(&sun4ichip->ctrl_lock); - ret = pwmchip_add(&sun4ichip->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); goto err_pwm_add; } - platform_set_drvdata(pdev, sun4ichip); + platform_set_drvdata(pdev, chip); return 0; @@ -477,9 +479,10 @@ err_bus: static void sun4i_pwm_remove(struct platform_device *pdev) { - struct sun4i_pwm_chip *sun4ichip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct sun4i_pwm_chip *sun4ichip = to_sun4i_pwm_chip(chip); - pwmchip_remove(&sun4ichip->chip); + pwmchip_remove(chip); clk_disable_unprepare(sun4ichip->bus_clk); reset_control_assert(sun4ichip->rst); From 2eb3ff5f4c21dfacdf7740d686564c397889e595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:53 +0100 Subject: [PATCH 0674/1038] pwm: sun4i: Consistently name driver data sun4ichip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver uses three different names for variables of type sun4i_pwm_chip: $ git grep 'struct sun4i_pwm_chip \*' v6.8-rc1 -- drivers/pwm/pwm-sun4i.c v6.8-rc1:drivers/pwm/pwm-sun4i.c:static inline struct sun4i_pwm_chip *to_sun4i_pwm_chip(struct pwm_chip *chip) v6.8-rc1:drivers/pwm/pwm-sun4i.c:static inline u32 sun4i_pwm_readl(struct sun4i_pwm_chip *chip, v6.8-rc1:drivers/pwm/pwm-sun4i.c:static inline void sun4i_pwm_writel(struct sun4i_pwm_chip *chip, v6.8-rc1:drivers/pwm/pwm-sun4i.c: struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-sun4i.c:static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, v6.8-rc1:drivers/pwm/pwm-sun4i.c: struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); v6.8-rc1:drivers/pwm/pwm-sun4i.c: struct sun4i_pwm_chip *sun4ichip; v6.8-rc1:drivers/pwm/pwm-sun4i.c: struct sun4i_pwm_chip *sun4ichip = platform_get_drvdata(pdev); "chip" is usually reserved for variables of type struct pwm_chip. So pick sun4ichip as common name which better matches the type name than sun4i_pwm. Acked-by: Jernej Skrabec Link: https://lore.kernel.org/r/705f54a13b59fff50eaa345d8b1e0c691345b996.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sun4i.c | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index c2f579043915..2437e5961f5e 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -95,32 +95,32 @@ static inline struct sun4i_pwm_chip *to_sun4i_pwm_chip(struct pwm_chip *chip) return container_of(chip, struct sun4i_pwm_chip, chip); } -static inline u32 sun4i_pwm_readl(struct sun4i_pwm_chip *chip, +static inline u32 sun4i_pwm_readl(struct sun4i_pwm_chip *sun4ichip, unsigned long offset) { - return readl(chip->base + offset); + return readl(sun4ichip->base + offset); } -static inline void sun4i_pwm_writel(struct sun4i_pwm_chip *chip, +static inline void sun4i_pwm_writel(struct sun4i_pwm_chip *sun4ichip, u32 val, unsigned long offset) { - writel(val, chip->base + offset); + writel(val, sun4ichip->base + offset); } static int sun4i_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { - struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); + struct sun4i_pwm_chip *sun4ichip = to_sun4i_pwm_chip(chip); u64 clk_rate, tmp; u32 val; unsigned int prescaler; - clk_rate = clk_get_rate(sun4i_pwm->clk); + clk_rate = clk_get_rate(sun4ichip->clk); if (!clk_rate) return -EINVAL; - val = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + val = sun4i_pwm_readl(sun4ichip, PWM_CTRL_REG); /* * PWM chapter in H6 manual has a diagram which explains that if bypass @@ -128,7 +128,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, * proved that also enable bit is ignored in this case. */ if ((val & BIT_CH(PWM_BYPASS, pwm->hwpwm)) && - sun4i_pwm->data->has_direct_mod_clk_output) { + sun4ichip->data->has_direct_mod_clk_output) { state->period = DIV_ROUND_UP_ULL(NSEC_PER_SEC, clk_rate); state->duty_cycle = DIV_ROUND_UP_ULL(state->period, 2); state->polarity = PWM_POLARITY_NORMAL; @@ -137,7 +137,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, } if ((PWM_REG_PRESCAL(val, pwm->hwpwm) == PWM_PRESCAL_MASK) && - sun4i_pwm->data->has_prescaler_bypass) + sun4ichip->data->has_prescaler_bypass) prescaler = 1; else prescaler = prescaler_table[PWM_REG_PRESCAL(val, pwm->hwpwm)]; @@ -156,7 +156,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, else state->enabled = false; - val = sun4i_pwm_readl(sun4i_pwm, PWM_CH_PRD(pwm->hwpwm)); + val = sun4i_pwm_readl(sun4ichip, PWM_CH_PRD(pwm->hwpwm)); tmp = (u64)prescaler * NSEC_PER_SEC * PWM_REG_DTY(val); state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, clk_rate); @@ -167,7 +167,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, return 0; } -static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, +static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4ichip, const struct pwm_state *state, u32 *dty, u32 *prd, unsigned int *prsclr, bool *bypass) @@ -175,9 +175,9 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, u64 clk_rate, div = 0; unsigned int prescaler = 0; - clk_rate = clk_get_rate(sun4i_pwm->clk); + clk_rate = clk_get_rate(sun4ichip->clk); - *bypass = sun4i_pwm->data->has_direct_mod_clk_output && + *bypass = sun4ichip->data->has_direct_mod_clk_output && state->enabled && (state->period * clk_rate >= NSEC_PER_SEC) && (state->period * clk_rate < 2 * NSEC_PER_SEC) && @@ -187,7 +187,7 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, if (*bypass) return 0; - if (sun4i_pwm->data->has_prescaler_bypass) { + if (sun4ichip->data->has_prescaler_bypass) { /* First, test without any prescaler when available */ prescaler = PWM_PRESCAL_MASK; /* @@ -233,7 +233,7 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { - struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); + struct sun4i_pwm_chip *sun4ichip = to_sun4i_pwm_chip(chip); struct pwm_state cstate; u32 ctrl, duty = 0, period = 0, val; int ret; @@ -243,31 +243,31 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, pwm_get_state(pwm, &cstate); if (!cstate.enabled) { - ret = clk_prepare_enable(sun4i_pwm->clk); + ret = clk_prepare_enable(sun4ichip->clk); if (ret) { dev_err(pwmchip_parent(chip), "failed to enable PWM clock\n"); return ret; } } - ret = sun4i_pwm_calculate(sun4i_pwm, state, &duty, &period, &prescaler, + ret = sun4i_pwm_calculate(sun4ichip, state, &duty, &period, &prescaler, &bypass); if (ret) { dev_err(pwmchip_parent(chip), "period exceeds the maximum value\n"); if (!cstate.enabled) - clk_disable_unprepare(sun4i_pwm->clk); + clk_disable_unprepare(sun4ichip->clk); return ret; } - spin_lock(&sun4i_pwm->ctrl_lock); - ctrl = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + spin_lock(&sun4ichip->ctrl_lock); + ctrl = sun4i_pwm_readl(sun4ichip, PWM_CTRL_REG); - if (sun4i_pwm->data->has_direct_mod_clk_output) { + if (sun4ichip->data->has_direct_mod_clk_output) { if (bypass) { ctrl |= BIT_CH(PWM_BYPASS, pwm->hwpwm); /* We can skip other parameter */ - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); - spin_unlock(&sun4i_pwm->ctrl_lock); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); + spin_unlock(&sun4ichip->ctrl_lock); return 0; } @@ -277,14 +277,14 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (PWM_REG_PRESCAL(ctrl, pwm->hwpwm) != prescaler) { /* Prescaler changed, the clock has to be gated */ ctrl &= ~BIT_CH(PWM_CLK_GATING, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); ctrl &= ~BIT_CH(PWM_PRESCAL_MASK, pwm->hwpwm); ctrl |= BIT_CH(prescaler, pwm->hwpwm); } val = (duty & PWM_DTY_MASK) | PWM_PRD(period); - sun4i_pwm_writel(sun4i_pwm, val, PWM_CH_PRD(pwm->hwpwm)); + sun4i_pwm_writel(sun4ichip, val, PWM_CH_PRD(pwm->hwpwm)); if (state->polarity != PWM_POLARITY_NORMAL) ctrl &= ~BIT_CH(PWM_ACT_STATE, pwm->hwpwm); @@ -296,9 +296,9 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->enabled) ctrl |= BIT_CH(PWM_EN, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); - spin_unlock(&sun4i_pwm->ctrl_lock); + spin_unlock(&sun4ichip->ctrl_lock); if (state->enabled) return 0; @@ -310,14 +310,14 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, else usleep_range(delay_us, delay_us * 2); - spin_lock(&sun4i_pwm->ctrl_lock); - ctrl = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + spin_lock(&sun4ichip->ctrl_lock); + ctrl = sun4i_pwm_readl(sun4ichip, PWM_CTRL_REG); ctrl &= ~BIT_CH(PWM_CLK_GATING, pwm->hwpwm); ctrl &= ~BIT_CH(PWM_EN, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); - spin_unlock(&sun4i_pwm->ctrl_lock); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); + spin_unlock(&sun4ichip->ctrl_lock); - clk_disable_unprepare(sun4i_pwm->clk); + clk_disable_unprepare(sun4ichip->clk); return 0; } From 362e3f883d048d786d7dbc6974eee1ed86b23091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:54 +0100 Subject: [PATCH 0675/1038] pwm: sun4i: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-sun4i driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Reviewed-by: Jernej Skrabec Link: https://lore.kernel.org/r/9d175b4e27878618cef2e75b6ecbf01ad5d18164.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sun4i.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 2437e5961f5e..5c29590d1821 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -81,7 +81,6 @@ struct sun4i_pwm_data { }; struct sun4i_pwm_chip { - struct pwm_chip chip; struct clk *bus_clk; struct clk *clk; struct reset_control *rst; @@ -92,7 +91,7 @@ struct sun4i_pwm_chip { static inline struct sun4i_pwm_chip *to_sun4i_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct sun4i_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 sun4i_pwm_readl(struct sun4i_pwm_chip *sun4ichip, @@ -385,18 +384,20 @@ MODULE_DEVICE_TABLE(of, sun4i_pwm_dt_ids); static int sun4i_pwm_probe(struct platform_device *pdev) { struct pwm_chip *chip; + const struct sun4i_pwm_data *data; struct sun4i_pwm_chip *sun4ichip; int ret; - sun4ichip = devm_kzalloc(&pdev->dev, sizeof(*sun4ichip), GFP_KERNEL); - if (!sun4ichip) - return -ENOMEM; - chip = &sun4ichip->chip; - - sun4ichip->data = of_device_get_match_data(&pdev->dev); - if (!sun4ichip->data) + data = of_device_get_match_data(&pdev->dev); + if (!data) return -ENODEV; + chip = devm_pwmchip_alloc(&pdev->dev, data->npwm, sizeof(*sun4ichip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + sun4ichip = to_sun4i_pwm_chip(chip); + + sun4ichip->data = data; sun4ichip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sun4ichip->base)) return PTR_ERR(sun4ichip->base); @@ -453,9 +454,7 @@ static int sun4i_pwm_probe(struct platform_device *pdev) goto err_bus; } - chip->dev = &pdev->dev; chip->ops = &sun4i_pwm_ops; - chip->npwm = sun4ichip->data->npwm; spin_lock_init(&sun4ichip->ctrl_lock); From 11ee0a124cb48bb837a1d90c3504a9c3376e96d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:55 +0100 Subject: [PATCH 0676/1038] pwm: sunplus: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-sunplus driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/192be7e428eff17dd922c9c0d0d168225b89bb34.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-sunplus.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-sunplus.c b/drivers/pwm/pwm-sunplus.c index 773e2f80526e..b342b843247b 100644 --- a/drivers/pwm/pwm-sunplus.c +++ b/drivers/pwm/pwm-sunplus.c @@ -43,14 +43,13 @@ #define SP7021_PWM_NUM 4 struct sunplus_pwm { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct sunplus_pwm *to_sunplus_pwm(struct pwm_chip *chip) { - return container_of(chip, struct sunplus_pwm, chip); + return pwmchip_get_drvdata(chip); } static int sunplus_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -175,12 +174,14 @@ static void sunplus_pwm_clk_release(void *data) static int sunplus_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct sunplus_pwm *priv; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, SP7021_PWM_NUM, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_sunplus_pwm(chip); priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) @@ -203,11 +204,9 @@ static int sunplus_pwm_probe(struct platform_device *pdev) return ret; } - priv->chip.dev = dev; - priv->chip.ops = &sunplus_pwm_ops; - priv->chip.npwm = SP7021_PWM_NUM; + chip->ops = &sunplus_pwm_ops; - ret = devm_pwmchip_add(dev, &priv->chip); + ret = devm_pwmchip_add(dev, chip); if (ret < 0) return dev_err_probe(dev, ret, "Cannot register sunplus PWM\n"); From b662c6e80976ece851dc2d164a5e32885704e4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:56 +0100 Subject: [PATCH 0677/1038] pwm: tegra: Drop duplicated tracking of the parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pwmchip stores a pointer to the parent device, so there is no need to store another copy in driver private data. Drop struct tegra_pwm_chip::dev and use the pwm_chip's parent pointer instead. Link: https://lore.kernel.org/r/225f4bfcb15fb69eb818ddb71d623157c447180a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tegra.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c index 82ee2f0754f9..0d5f57c9ee26 100644 --- a/drivers/pwm/pwm-tegra.c +++ b/drivers/pwm/pwm-tegra.c @@ -66,7 +66,6 @@ struct tegra_pwm_soc { struct tegra_pwm_chip { struct pwm_chip chip; - struct device *dev; struct clk *clk; struct reset_control*rst; @@ -158,7 +157,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, */ required_clk_rate *= 2; - err = dev_pm_opp_set_rate(pc->dev, required_clk_rate); + err = dev_pm_opp_set_rate(pwmchip_parent(chip), required_clk_rate); if (err < 0) return -EINVAL; @@ -194,7 +193,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, * before writing the register. Otherwise, keep it enabled. */ if (!pwm_is_enabled(pwm)) { - err = pm_runtime_resume_and_get(pc->dev); + err = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (err) return err; } else @@ -206,7 +205,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, * If the PWM is not enabled, turn the clock off again to save power. */ if (!pwm_is_enabled(pwm)) - pm_runtime_put(pc->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -217,7 +216,7 @@ static int tegra_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) int rc = 0; u32 val; - rc = pm_runtime_resume_and_get(pc->dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; @@ -237,7 +236,7 @@ static void tegra_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) val &= ~PWM_ENABLE; pwm_writel(pc, pwm->hwpwm, val); - pm_runtime_put_sync(pc->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static int tegra_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -280,7 +279,6 @@ static int tegra_pwm_probe(struct platform_device *pdev) return -ENOMEM; pc->soc = of_device_get_match_data(&pdev->dev); - pc->dev = &pdev->dev; pc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->regs)) @@ -302,7 +300,7 @@ static int tegra_pwm_probe(struct platform_device *pdev) return ret; /* Set maximum frequency of the IP */ - ret = dev_pm_opp_set_rate(pc->dev, pc->soc->max_frequency); + ret = dev_pm_opp_set_rate(&pdev->dev, pc->soc->max_frequency); if (ret < 0) { dev_err(&pdev->dev, "Failed to set max frequency: %d\n", ret); goto put_pm; From aa37f83f7bfadc7e9dac8b32d87f93d185cd5555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:57 +0100 Subject: [PATCH 0678/1038] pwm: tegra: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct tegra_pwm_chip. Use the pwm_chip as driver data instead of the tegra_pwm_chip to get access to the pwm_chip in tegra_pwm_remove() without using pc->chip. Link: https://lore.kernel.org/r/2813c63bf1317dee808f4c5c4a9411999f2d5746.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tegra.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c index 0d5f57c9ee26..f61c24376523 100644 --- a/drivers/pwm/pwm-tegra.c +++ b/drivers/pwm/pwm-tegra.c @@ -271,12 +271,14 @@ static const struct pwm_ops tegra_pwm_ops = { static int tegra_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct tegra_pwm_chip *pc; int ret; pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); if (!pc) return -ENOMEM; + chip = &pc->chip; pc->soc = of_device_get_match_data(&pdev->dev); @@ -284,7 +286,7 @@ static int tegra_pwm_probe(struct platform_device *pdev) if (IS_ERR(pc->regs)) return PTR_ERR(pc->regs); - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pc->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(pc->clk)) @@ -326,11 +328,11 @@ static int tegra_pwm_probe(struct platform_device *pdev) reset_control_deassert(pc->rst); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &tegra_pwm_ops; - pc->chip.npwm = pc->soc->num_channels; + chip->dev = &pdev->dev; + chip->ops = &tegra_pwm_ops; + chip->npwm = pc->soc->num_channels; - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); reset_control_assert(pc->rst); @@ -348,9 +350,10 @@ put_pm: static void tegra_pwm_remove(struct platform_device *pdev) { - struct tegra_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); reset_control_assert(pc->rst); @@ -359,7 +362,8 @@ static void tegra_pwm_remove(struct platform_device *pdev) static int __maybe_unused tegra_pwm_runtime_suspend(struct device *dev) { - struct tegra_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); int err; clk_disable_unprepare(pc->clk); @@ -375,7 +379,8 @@ static int __maybe_unused tegra_pwm_runtime_suspend(struct device *dev) static int __maybe_unused tegra_pwm_runtime_resume(struct device *dev) { - struct tegra_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); int err; err = pinctrl_pm_select_default_state(dev); From 7550ebf04c05de66e504c17dc7c6ac5cf8802d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:58 +0100 Subject: [PATCH 0679/1038] pwm: tegra: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-tegra driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/8719be3d57b0b5cf575b312e5ff41fe0717e3a43.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tegra.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c index f61c24376523..a3d69976148f 100644 --- a/drivers/pwm/pwm-tegra.c +++ b/drivers/pwm/pwm-tegra.c @@ -65,8 +65,6 @@ struct tegra_pwm_soc { }; struct tegra_pwm_chip { - struct pwm_chip chip; - struct clk *clk; struct reset_control*rst; @@ -80,7 +78,7 @@ struct tegra_pwm_chip { static inline struct tegra_pwm_chip *to_tegra_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct tegra_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 pwm_readl(struct tegra_pwm_chip *pc, unsigned int offset) @@ -273,14 +271,17 @@ static int tegra_pwm_probe(struct platform_device *pdev) { struct pwm_chip *chip; struct tegra_pwm_chip *pc; + const struct tegra_pwm_soc *soc; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; - chip = &pc->chip; + soc = of_device_get_match_data(&pdev->dev); - pc->soc = of_device_get_match_data(&pdev->dev); + chip = devm_pwmchip_alloc(&pdev->dev, soc->num_channels, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_tegra_pwm_chip(chip); + + pc->soc = soc; pc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->regs)) @@ -328,9 +329,7 @@ static int tegra_pwm_probe(struct platform_device *pdev) reset_control_deassert(pc->rst); - chip->dev = &pdev->dev; chip->ops = &tegra_pwm_ops; - chip->npwm = pc->soc->num_channels; ret = pwmchip_add(chip); if (ret < 0) { From 62f59c107aac4ed7dd770699fbe8de625e6b8755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:32:59 +0100 Subject: [PATCH 0680/1038] pwm: tiecap: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct ecap_pwm_chip *pc just obtained from it. This also has the advantage of not using struct ecap_pwm_chip::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/628f4b8c4ba1321075fc1dff70453a1c79ffb814.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiecap.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c index d974f4414ac9..b93e3be318d5 100644 --- a/drivers/pwm/pwm-tiecap.c +++ b/drivers/pwm/pwm-tiecap.c @@ -70,7 +70,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, duty_cycles = (u32)c; } - pm_runtime_get_sync(pc->chip.dev); + pm_runtime_get_sync(chip->dev); value = readw(pc->mmio_base + ECCTL2); @@ -100,7 +100,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, writew(value, pc->mmio_base + ECCTL2); } - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(chip->dev); return 0; } @@ -111,7 +111,7 @@ static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); u16 value; - pm_runtime_get_sync(pc->chip.dev); + pm_runtime_get_sync(chip->dev); value = readw(pc->mmio_base + ECCTL2); @@ -124,7 +124,7 @@ static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, writew(value, pc->mmio_base + ECCTL2); - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(chip->dev); return 0; } @@ -135,7 +135,7 @@ static int ecap_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) u16 value; /* Leave clock enabled on enabling PWM */ - pm_runtime_get_sync(pc->chip.dev); + pm_runtime_get_sync(chip->dev); /* * Enable 'Free run Time stamp counter mode' to start counter @@ -162,7 +162,7 @@ static void ecap_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) writew(value, pc->mmio_base + ECCTL2); /* Disable clock on PWM disable */ - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(chip->dev); } static int ecap_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, From 02aa760d293e5036655cc7bdda927e42300e26fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:00 +0100 Subject: [PATCH 0681/1038] pwm: tiecap: Change prototype of helpers to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given ecap_pwm_chip. To just not have to do that, rework ecap_pwm_save_context() and ecap_pwm_restore_context take a pwm_chip. Also use the pwm_chip as driver data instead of the ecap_pwm_chip. Link: https://lore.kernel.org/r/ed031f201ff52c6b298de2dc81b06aad3a0207f8.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiecap.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c index b93e3be318d5..0d10e8357731 100644 --- a/drivers/pwm/pwm-tiecap.c +++ b/drivers/pwm/pwm-tiecap.c @@ -218,6 +218,7 @@ static int ecap_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct ecap_pwm_chip *pc; + struct pwm_chip *chip; struct clk *clk; int ret; @@ -244,21 +245,22 @@ static int ecap_pwm_probe(struct platform_device *pdev) return -EINVAL; } - pc->chip.dev = &pdev->dev; - pc->chip.ops = &ecap_pwm_ops; - pc->chip.npwm = 1; + chip = &pc->chip; + chip->dev = &pdev->dev; + chip->ops = &ecap_pwm_ops; + chip->npwm = 1; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) return PTR_ERR(pc->mmio_base); - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); return ret; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pm_runtime_enable(&pdev->dev); return 0; @@ -269,17 +271,21 @@ static void ecap_pwm_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); } -static void ecap_pwm_save_context(struct ecap_pwm_chip *pc) +static void ecap_pwm_save_context(struct pwm_chip *chip) { - pm_runtime_get_sync(pc->chip.dev); + struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); + + pm_runtime_get_sync(chip->dev); pc->ctx.ecctl2 = readw(pc->mmio_base + ECCTL2); pc->ctx.cap4 = readl(pc->mmio_base + CAP4); pc->ctx.cap3 = readl(pc->mmio_base + CAP3); - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(chip->dev); } -static void ecap_pwm_restore_context(struct ecap_pwm_chip *pc) +static void ecap_pwm_restore_context(struct pwm_chip *chip) { + struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); + writel(pc->ctx.cap3, pc->mmio_base + CAP3); writel(pc->ctx.cap4, pc->mmio_base + CAP4); writew(pc->ctx.ecctl2, pc->mmio_base + ECCTL2); @@ -287,10 +293,10 @@ static void ecap_pwm_restore_context(struct ecap_pwm_chip *pc) static int ecap_pwm_suspend(struct device *dev) { - struct ecap_pwm_chip *pc = dev_get_drvdata(dev); - struct pwm_device *pwm = pc->chip.pwms; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct pwm_device *pwm = chip->pwms; - ecap_pwm_save_context(pc); + ecap_pwm_save_context(chip); /* Disable explicitly if PWM is running */ if (pwm_is_enabled(pwm)) @@ -301,14 +307,14 @@ static int ecap_pwm_suspend(struct device *dev) static int ecap_pwm_resume(struct device *dev) { - struct ecap_pwm_chip *pc = dev_get_drvdata(dev); - struct pwm_device *pwm = pc->chip.pwms; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct pwm_device *pwm = chip->pwms; /* Enable explicitly if PWM was running */ if (pwm_is_enabled(pwm)) pm_runtime_get_sync(dev); - ecap_pwm_restore_context(pc); + ecap_pwm_restore_context(chip); return 0; } From ba60fbf92d25fafe13c7d9089a983024be4bc151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:01 +0100 Subject: [PATCH 0682/1038] pwm: tiecap: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/ae92e06b49437ca7e768b1f8b405170e33948a70.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiecap.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c index 0d10e8357731..558b244f074a 100644 --- a/drivers/pwm/pwm-tiecap.c +++ b/drivers/pwm/pwm-tiecap.c @@ -70,7 +70,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, duty_cycles = (u32)c; } - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); value = readw(pc->mmio_base + ECCTL2); @@ -100,7 +100,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, writew(value, pc->mmio_base + ECCTL2); } - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -111,7 +111,7 @@ static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); u16 value; - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); value = readw(pc->mmio_base + ECCTL2); @@ -124,7 +124,7 @@ static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, writew(value, pc->mmio_base + ECCTL2); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -135,7 +135,7 @@ static int ecap_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) u16 value; /* Leave clock enabled on enabling PWM */ - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); /* * Enable 'Free run Time stamp counter mode' to start counter @@ -162,7 +162,7 @@ static void ecap_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) writew(value, pc->mmio_base + ECCTL2); /* Disable clock on PWM disable */ - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static int ecap_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -275,11 +275,11 @@ static void ecap_pwm_save_context(struct pwm_chip *chip) { struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); pc->ctx.ecctl2 = readw(pc->mmio_base + ECCTL2); pc->ctx.cap4 = readl(pc->mmio_base + CAP4); pc->ctx.cap3 = readl(pc->mmio_base + CAP3); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static void ecap_pwm_restore_context(struct pwm_chip *chip) From 76b4accb7c62d904f7e3c78d2ded7041cd662b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:02 +0100 Subject: [PATCH 0683/1038] pwm: tiecap: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-tegra driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/af7846109c0df225126c8e5cd186b89ace70afc0.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiecap.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c index 558b244f074a..d6c2b1b1387e 100644 --- a/drivers/pwm/pwm-tiecap.c +++ b/drivers/pwm/pwm-tiecap.c @@ -32,7 +32,6 @@ struct ecap_context { }; struct ecap_pwm_chip { - struct pwm_chip chip; unsigned int clk_rate; void __iomem *mmio_base; struct ecap_context ctx; @@ -40,7 +39,7 @@ struct ecap_pwm_chip { static inline struct ecap_pwm_chip *to_ecap_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct ecap_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -222,9 +221,10 @@ static int ecap_pwm_probe(struct platform_device *pdev) struct clk *clk; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_ecap_pwm_chip(chip); clk = devm_clk_get(&pdev->dev, "fck"); if (IS_ERR(clk)) { @@ -245,10 +245,7 @@ static int ecap_pwm_probe(struct platform_device *pdev) return -EINVAL; } - chip = &pc->chip; - chip->dev = &pdev->dev; chip->ops = &ecap_pwm_ops; - chip->npwm = 1; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) From b514a1b29df8aadba641f8cb8153aa386489eb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:03 +0100 Subject: [PATCH 0684/1038] pwm: tiehrpwm: Simplify code to determine the pwmchip's parent device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is already a pointer to the pwmchip, make use of it directly instead of using the struct ehrpwm_pwm_chip *ddata just obtained from it. This also has the advantage of not using struct ehrpwm_pwm_chip::chip any more which will be dropped soon. Link: https://lore.kernel.org/r/b2b06a3aabf8c04969d59ddf7ba565b303855878.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiehrpwm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index af231fa74fa9..9848493dee97 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -347,7 +347,7 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = clk_enable(pc->tbclk); if (ret) { dev_err(chip->dev, "Failed to enable TBCLK for %s: %d\n", - dev_name(pc->chip.dev), ret); + dev_name(chip->dev), ret); return ret; } From 4e1c8593e5fa395fdbdf7f908912967819086c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:04 +0100 Subject: [PATCH 0685/1038] pwm: tiehrpwm: Change prototype of helpers to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given ehrpwm_pwm_chip. To just not have to do that, rework ehrpwm_pwm_save_context() and ehrpwm_pwm_restore_context() take a pwm_chip. Also use the pwm_chip as driver data instead of the ehrpwm_pwm_chip. Link: https://lore.kernel.org/r/79052207cdf71f0882ae13fe1a192ef6f6dba35b.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiehrpwm.c | 45 ++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 9848493dee97..6d7babf6fdb0 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -450,6 +450,7 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct ehrpwm_pwm_chip *pc; + struct pwm_chip *chip; struct clk *clk; int ret; @@ -474,9 +475,10 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) return -EINVAL; } - pc->chip.dev = &pdev->dev; - pc->chip.ops = &ehrpwm_pwm_ops; - pc->chip.npwm = NUM_PWM_CHANNEL; + chip = &pc->chip; + chip->dev = &pdev->dev; + chip->ops = &ehrpwm_pwm_ops; + chip->npwm = NUM_PWM_CHANNEL; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) @@ -493,13 +495,13 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) return ret; } - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); goto err_clk_unprepare; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pm_runtime_enable(&pdev->dev); return 0; @@ -512,18 +514,21 @@ err_clk_unprepare: static void ehrpwm_pwm_remove(struct platform_device *pdev) { - struct ehrpwm_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); clk_unprepare(pc->tbclk); pm_runtime_disable(&pdev->dev); } -static void ehrpwm_pwm_save_context(struct ehrpwm_pwm_chip *pc) +static void ehrpwm_pwm_save_context(struct pwm_chip *chip) { - pm_runtime_get_sync(pc->chip.dev); + struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); + + pm_runtime_get_sync(chip->dev); pc->ctx.tbctl = ehrpwm_read(pc->mmio_base, TBCTL); pc->ctx.tbprd = ehrpwm_read(pc->mmio_base, TBPRD); @@ -534,11 +539,13 @@ static void ehrpwm_pwm_save_context(struct ehrpwm_pwm_chip *pc) pc->ctx.aqsfrc = ehrpwm_read(pc->mmio_base, AQSFRC); pc->ctx.aqcsfrc = ehrpwm_read(pc->mmio_base, AQCSFRC); - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(chip->dev); } -static void ehrpwm_pwm_restore_context(struct ehrpwm_pwm_chip *pc) +static void ehrpwm_pwm_restore_context(struct pwm_chip *chip) { + struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); + ehrpwm_write(pc->mmio_base, TBPRD, pc->ctx.tbprd); ehrpwm_write(pc->mmio_base, CMPA, pc->ctx.cmpa); ehrpwm_write(pc->mmio_base, CMPB, pc->ctx.cmpb); @@ -551,13 +558,13 @@ static void ehrpwm_pwm_restore_context(struct ehrpwm_pwm_chip *pc) static int ehrpwm_pwm_suspend(struct device *dev) { - struct ehrpwm_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); unsigned int i; - ehrpwm_pwm_save_context(pc); + ehrpwm_pwm_save_context(chip); - for (i = 0; i < pc->chip.npwm; i++) { - struct pwm_device *pwm = &pc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!pwm_is_enabled(pwm)) continue; @@ -571,11 +578,11 @@ static int ehrpwm_pwm_suspend(struct device *dev) static int ehrpwm_pwm_resume(struct device *dev) { - struct ehrpwm_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); unsigned int i; - for (i = 0; i < pc->chip.npwm; i++) { - struct pwm_device *pwm = &pc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!pwm_is_enabled(pwm)) continue; @@ -584,7 +591,7 @@ static int ehrpwm_pwm_resume(struct device *dev) pm_runtime_get_sync(dev); } - ehrpwm_pwm_restore_context(pc); + ehrpwm_pwm_restore_context(chip); return 0; } From e003a687aac4f867354e96531ef46207f8dc6723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:05 +0100 Subject: [PATCH 0686/1038] pwm: tiehrpwm: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/9badd116d0e26a5656b222c5b4adad7e111a53c7.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiehrpwm.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 6d7babf6fdb0..9a7c72196173 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -256,7 +256,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (i == pwm->hwpwm) continue; - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "period value conflicts with channel %u\n", i); return -EINVAL; @@ -268,11 +268,11 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* Configure clock prescaler to support Low frequency PWM wave */ if (set_prescale_div(period_cycles/PERIOD_MAX, &ps_divval, &tb_divval)) { - dev_err(chip->dev, "Unsupported values\n"); + dev_err(pwmchip_parent(chip), "Unsupported values\n"); return -EINVAL; } - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); /* Update clock prescaler values */ ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CLKDIV_MASK, tb_divval); @@ -299,7 +299,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -323,7 +323,7 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) int ret; /* Leave clock enabled on enabling PWM */ - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); /* Disabling Action Qualifier on PWM output */ if (pwm->hwpwm) { @@ -346,8 +346,8 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) /* Enable TBCLK */ ret = clk_enable(pc->tbclk); if (ret) { - dev_err(chip->dev, "Failed to enable TBCLK for %s: %d\n", - dev_name(chip->dev), ret); + dev_err(pwmchip_parent(chip), "Failed to enable TBCLK for %s: %d\n", + dev_name(pwmchip_parent(chip)), ret); return ret; } @@ -385,7 +385,7 @@ static void ehrpwm_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) clk_disable(pc->tbclk); /* Disable clock on PWM disable */ - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static void ehrpwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) @@ -393,8 +393,8 @@ static void ehrpwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); if (pwm_is_enabled(pwm)) { - dev_warn(chip->dev, "Removing PWM device without disabling\n"); - pm_runtime_put_sync(chip->dev); + dev_warn(pwmchip_parent(chip), "Removing PWM device without disabling\n"); + pm_runtime_put_sync(pwmchip_parent(chip)); } /* set period value to zero on free */ @@ -528,7 +528,7 @@ static void ehrpwm_pwm_save_context(struct pwm_chip *chip) { struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); pc->ctx.tbctl = ehrpwm_read(pc->mmio_base, TBCTL); pc->ctx.tbprd = ehrpwm_read(pc->mmio_base, TBPRD); @@ -539,7 +539,7 @@ static void ehrpwm_pwm_save_context(struct pwm_chip *chip) pc->ctx.aqsfrc = ehrpwm_read(pc->mmio_base, AQSFRC); pc->ctx.aqcsfrc = ehrpwm_read(pc->mmio_base, AQCSFRC); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static void ehrpwm_pwm_restore_context(struct pwm_chip *chip) From de830057d72cff27f68df63f9cb8e863d84f9cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:06 +0100 Subject: [PATCH 0687/1038] pwm: tiehrpwm: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-tiecap driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/62fbac428cae0942f8e88234bf249537fcd890a3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-tiehrpwm.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 9a7c72196173..e5104725d9b7 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -105,7 +105,6 @@ struct ehrpwm_context { }; struct ehrpwm_pwm_chip { - struct pwm_chip chip; unsigned long clk_rate; void __iomem *mmio_base; unsigned long period_cycles[NUM_PWM_CHANNEL]; @@ -116,7 +115,7 @@ struct ehrpwm_pwm_chip { static inline struct ehrpwm_pwm_chip *to_ehrpwm_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct ehrpwm_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u16 ehrpwm_read(void __iomem *base, unsigned int offset) @@ -454,9 +453,10 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) struct clk *clk; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, NUM_PWM_CHANNEL, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_ehrpwm_pwm_chip(chip); clk = devm_clk_get(&pdev->dev, "fck"); if (IS_ERR(clk)) { @@ -475,10 +475,7 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) return -EINVAL; } - chip = &pc->chip; - chip->dev = &pdev->dev; chip->ops = &ehrpwm_pwm_ops; - chip->npwm = NUM_PWM_CHANNEL; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) From 6a20d5ed0e46a9ef87b70e2113a1102c2c0bd05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:07 +0100 Subject: [PATCH 0688/1038] pwm: twl: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/f11beb6b3a398d1257219a635a78ed0b02263978.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-twl.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/pwm/pwm-twl.c b/drivers/pwm/pwm-twl.c index 68e02c9a6bf9..7233ae039bdb 100644 --- a/drivers/pwm/pwm-twl.c +++ b/drivers/pwm/pwm-twl.c @@ -86,7 +86,7 @@ static int twl_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = twl_i2c_write(TWL_MODULE_PWM, pwm_config, base, 2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to configure PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to configure PWM\n", pwm->label); return ret; } @@ -100,7 +100,7 @@ static int twl4030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_GPBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read GPBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read GPBR1\n", pwm->label); goto out; } @@ -108,13 +108,13 @@ static int twl4030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); val |= TWL4030_PWM_TOGGLE(pwm->hwpwm, TWL4030_PWMX_ENABLE); ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -130,7 +130,7 @@ static void twl4030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_GPBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read GPBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read GPBR1\n", pwm->label); goto out; } @@ -138,13 +138,13 @@ static void twl4030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); val &= ~TWL4030_PWM_TOGGLE(pwm->hwpwm, TWL4030_PWMXCLK_ENABLE); ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -167,7 +167,7 @@ static int twl4030_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_PMBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PMBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read PMBR1\n", pwm->label); goto out; } @@ -181,7 +181,7 @@ static int twl4030_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_PMBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to request PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to request PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -202,7 +202,7 @@ static void twl4030_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_PMBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PMBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read PMBR1\n", pwm->label); goto out; } @@ -212,7 +212,7 @@ static void twl4030_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_PMBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to free PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to free PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -231,7 +231,7 @@ static int twl6030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); goto out; } @@ -254,7 +254,7 @@ static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); goto out; } @@ -262,7 +262,7 @@ static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); goto out; } @@ -270,7 +270,7 @@ static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); goto out; } From 12fa8803b6a2f1271a7f1315c921d12f941be03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:08 +0100 Subject: [PATCH 0689/1038] pwm: twl: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-twol driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/c65e796d46df71cd8d0d0941921997b9501f1cb3.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-twl.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/pwm/pwm-twl.c b/drivers/pwm/pwm-twl.c index 7233ae039bdb..8f981ffff4b4 100644 --- a/drivers/pwm/pwm-twl.c +++ b/drivers/pwm/pwm-twl.c @@ -46,7 +46,6 @@ #define TWL6030_PWM_TOGGLE(pwm, x) ((x) << (pwm * 3)) struct twl_pwm_chip { - struct pwm_chip chip; struct mutex mutex; u8 twl6030_toggle3; u8 twl4030_pwm_mux; @@ -54,7 +53,7 @@ struct twl_pwm_chip { static inline struct twl_pwm_chip *to_twl(struct pwm_chip *chip) { - return container_of(chip, struct twl_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int twl_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, @@ -341,23 +340,22 @@ static const struct pwm_ops twl6030_pwm_ops = { static int twl_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct twl_pwm_chip *twl; - twl = devm_kzalloc(&pdev->dev, sizeof(*twl), GFP_KERNEL); - if (!twl) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*twl)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + twl = to_twl(chip); if (twl_class_is_4030()) - twl->chip.ops = &twl4030_pwm_ops; + chip->ops = &twl4030_pwm_ops; else - twl->chip.ops = &twl6030_pwm_ops; - - twl->chip.dev = &pdev->dev; - twl->chip.npwm = 2; + chip->ops = &twl6030_pwm_ops; mutex_init(&twl->mutex); - return devm_pwmchip_add(&pdev->dev, &twl->chip); + return devm_pwmchip_add(&pdev->dev, chip); } #ifdef CONFIG_OF From e3ad4f23f9b26cb6c0723ffd748635079f4b0d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:09 +0100 Subject: [PATCH 0690/1038] pwm: twl-led: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/43c35b7116a637501b51ac93ec24c00ea92ee1af.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-twl-led.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/pwm/pwm-twl-led.c b/drivers/pwm/pwm-twl-led.c index c670ccb81653..00ef798dae52 100644 --- a/drivers/pwm/pwm-twl-led.c +++ b/drivers/pwm/pwm-twl-led.c @@ -100,7 +100,7 @@ static int twl4030_pwmled_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = twl_i2c_write(TWL4030_MODULE_LED, pwm_config, base, 2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to configure PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to configure PWM\n", pwm->label); return ret; } @@ -114,7 +114,7 @@ static int twl4030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_LED, &val, TWL4030_LEDEN_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read LEDEN\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read LEDEN\n", pwm->label); goto out; } @@ -122,7 +122,7 @@ static int twl4030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_LED, val, TWL4030_LEDEN_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -139,7 +139,7 @@ static void twl4030_pwmled_disable(struct pwm_chip *chip, mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_LED, &val, TWL4030_LEDEN_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read LEDEN\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read LEDEN\n", pwm->label); goto out; } @@ -147,7 +147,7 @@ static void twl4030_pwmled_disable(struct pwm_chip *chip, ret = twl_i2c_write_u8(TWL4030_MODULE_LED, val, TWL4030_LEDEN_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -203,7 +203,7 @@ static int twl6030_pwmled_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, on_time, TWL6030_LED_PWM_CTRL1); if (ret < 0) - dev_err(chip->dev, "%s: Failed to configure PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to configure PWM\n", pwm->label); return ret; } @@ -217,7 +217,7 @@ static int twl6030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -227,7 +227,7 @@ static int twl6030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -244,7 +244,7 @@ static void twl6030_pwmled_disable(struct pwm_chip *chip, mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -254,7 +254,7 @@ static void twl6030_pwmled_disable(struct pwm_chip *chip, ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -295,7 +295,7 @@ static int twl6030_pwmled_request(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -305,7 +305,7 @@ static int twl6030_pwmled_request(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to request PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to request PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -321,7 +321,7 @@ static void twl6030_pwmled_free(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -331,7 +331,7 @@ static void twl6030_pwmled_free(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to free PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to free PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); From b860648da16c9996259d26227f273bb4416b162a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:10 +0100 Subject: [PATCH 0691/1038] pwm: twl-led: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-twl-led driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/5bac90addb3a178ef958a2a524c6ec7e3eea3e6a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-twl-led.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/drivers/pwm/pwm-twl-led.c b/drivers/pwm/pwm-twl-led.c index 00ef798dae52..4b10a8dab312 100644 --- a/drivers/pwm/pwm-twl-led.c +++ b/drivers/pwm/pwm-twl-led.c @@ -62,13 +62,12 @@ #define TWL6040_LED_MODE_MASK 0x03 struct twl_pwmled_chip { - struct pwm_chip chip; struct mutex mutex; }; static inline struct twl_pwmled_chip *to_twl(struct pwm_chip *chip) { - return container_of(chip, struct twl_pwmled_chip, chip); + return pwmchip_get_drvdata(chip); } static int twl4030_pwmled_config(struct pwm_chip *chip, struct pwm_device *pwm, @@ -345,25 +344,29 @@ static const struct pwm_ops twl6030_pwmled_ops = { static int twl_pwmled_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct twl_pwmled_chip *twl; - - twl = devm_kzalloc(&pdev->dev, sizeof(*twl), GFP_KERNEL); - if (!twl) - return -ENOMEM; + unsigned int npwm; + const struct pwm_ops *ops; if (twl_class_is_4030()) { - twl->chip.ops = &twl4030_pwmled_ops; - twl->chip.npwm = 2; + ops = &twl4030_pwmled_ops; + npwm = 2; } else { - twl->chip.ops = &twl6030_pwmled_ops; - twl->chip.npwm = 1; + ops = &twl6030_pwmled_ops; + npwm = 1; } - twl->chip.dev = &pdev->dev; + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*twl)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + twl = to_twl(chip); + + chip->ops = ops; mutex_init(&twl->mutex); - return devm_pwmchip_add(&pdev->dev, &twl->chip); + return devm_pwmchip_add(&pdev->dev, chip); } #ifdef CONFIG_OF From 7fe09324dfbedeac3efdff93beab5fc01e771c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:11 +0100 Subject: [PATCH 0692/1038] pwm: visconti: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-visconti driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/24e779de69365686bb004742cd8f07cbda131212.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-visconti.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-visconti.c b/drivers/pwm/pwm-visconti.c index 8d736d558122..9e55380957be 100644 --- a/drivers/pwm/pwm-visconti.c +++ b/drivers/pwm/pwm-visconti.c @@ -34,13 +34,12 @@ #define PIPGM_PWMC_POLARITY_MASK GENMASK(5, 5) struct visconti_pwm_chip { - struct pwm_chip chip; void __iomem *base; }; static inline struct visconti_pwm_chip *visconti_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct visconti_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int visconti_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -134,22 +133,22 @@ static const struct pwm_ops visconti_pwm_ops = { static int visconti_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct visconti_pwm_chip *priv; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 4, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = visconti_pwm_from_chip(chip); priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); - priv->chip.dev = dev; - priv->chip.ops = &visconti_pwm_ops; - priv->chip.npwm = 4; + chip->ops = &visconti_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &priv->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Cannot register visconti PWM\n"); From 58803adcb70e90b525e5242e55a3838f8b605dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:12 +0100 Subject: [PATCH 0693/1038] pwm: vt8500: Change prototype of a helper to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given vt8500_chip. To just not have to do that, rework vt8500_pwm_busy_wait() to take a pwm_chip. Link: https://lore.kernel.org/r/fb384c550b359e7707219f87872bcf36482875ff.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-vt8500.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c index 7bfeacee05d0..3646bd45b101 100644 --- a/drivers/pwm/pwm-vt8500.c +++ b/drivers/pwm/pwm-vt8500.c @@ -53,8 +53,9 @@ struct vt8500_chip { #define to_vt8500_chip(chip) container_of(chip, struct vt8500_chip, chip) #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) -static inline void vt8500_pwm_busy_wait(struct vt8500_chip *vt8500, int nr, u8 bitmask) +static inline void vt8500_pwm_busy_wait(struct pwm_chip *chip, int nr, u8 bitmask) { + struct vt8500_chip *vt8500 = to_vt8500_chip(chip); int loops = msecs_to_loops(10); u32 mask = bitmask << (nr << 8); @@ -62,7 +63,7 @@ static inline void vt8500_pwm_busy_wait(struct vt8500_chip *vt8500, int nr, u8 b cpu_relax(); if (unlikely(!loops)) - dev_warn(vt8500->chip.dev, "Waiting for status bits 0x%x to clear timed out\n", + dev_warn(chip->dev, "Waiting for status bits 0x%x to clear timed out\n", mask); } @@ -103,18 +104,18 @@ static int vt8500_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, dc = div64_u64(c, period_ns); writel(prescale, vt8500->base + REG_SCALAR(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_SCALAR_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_SCALAR_UPDATE); writel(pv, vt8500->base + REG_PERIOD(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_PERIOD_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_PERIOD_UPDATE); writel(dc, vt8500->base + REG_DUTY(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_DUTY_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_DUTY_UPDATE); val = readl(vt8500->base + REG_CTRL(pwm->hwpwm)); val |= CTRL_AUTOLOAD; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); clk_disable(vt8500->clk); return 0; @@ -135,7 +136,7 @@ static int vt8500_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) val = readl(vt8500->base + REG_CTRL(pwm->hwpwm)); val |= CTRL_ENABLE; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); return 0; } @@ -148,7 +149,7 @@ static void vt8500_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) val = readl(vt8500->base + REG_CTRL(pwm->hwpwm)); val &= ~CTRL_ENABLE; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); clk_disable(vt8500->clk); } @@ -168,7 +169,7 @@ static int vt8500_pwm_set_polarity(struct pwm_chip *chip, val &= ~CTRL_INVERT; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); return 0; } From 9767db33373f125e0ed370c943cf7cb12f402ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:13 +0100 Subject: [PATCH 0694/1038] pwm: vt8500: Introduce a local pwm_chip variable in .probe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This simplifies converting the driver to pwmchip_alloc() as there is only a single code line left that makes use of struct vt8500_chip::chip. Link: https://lore.kernel.org/r/7d903b608609d46cf1ee1e06530f516f42af1ebb.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-vt8500.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c index 3646bd45b101..6da9b8f88afc 100644 --- a/drivers/pwm/pwm-vt8500.c +++ b/drivers/pwm/pwm-vt8500.c @@ -232,6 +232,7 @@ MODULE_DEVICE_TABLE(of, vt8500_pwm_dt_ids); static int vt8500_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct vt8500_chip *vt8500; struct device_node *np = pdev->dev.of_node; int ret; @@ -243,9 +244,10 @@ static int vt8500_pwm_probe(struct platform_device *pdev) if (vt8500 == NULL) return -ENOMEM; - vt8500->chip.dev = &pdev->dev; - vt8500->chip.ops = &vt8500_pwm_ops; - vt8500->chip.npwm = VT8500_NR_PWMS; + chip = &vt8500->chip; + chip->dev = &pdev->dev; + chip->ops = &vt8500_pwm_ops; + chip->npwm = VT8500_NR_PWMS; vt8500->clk = devm_clk_get_prepared(&pdev->dev, NULL); if (IS_ERR(vt8500->clk)) @@ -255,7 +257,7 @@ static int vt8500_pwm_probe(struct platform_device *pdev) if (IS_ERR(vt8500->base)) return PTR_ERR(vt8500->base); - ret = devm_pwmchip_add(&pdev->dev, &vt8500->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); From 68e34b3e6629b747c1d4ec1813455221a56d3b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:14 +0100 Subject: [PATCH 0695/1038] pwm: vt8500: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Link: https://lore.kernel.org/r/c3c45a08f2ccb8bb13b4042c73f93064876586eb.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-vt8500.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c index 6da9b8f88afc..fb4dbc88b0a3 100644 --- a/drivers/pwm/pwm-vt8500.c +++ b/drivers/pwm/pwm-vt8500.c @@ -63,7 +63,7 @@ static inline void vt8500_pwm_busy_wait(struct pwm_chip *chip, int nr, u8 bitmas cpu_relax(); if (unlikely(!loops)) - dev_warn(chip->dev, "Waiting for status bits 0x%x to clear timed out\n", + dev_warn(pwmchip_parent(chip), "Waiting for status bits 0x%x to clear timed out\n", mask); } @@ -78,7 +78,7 @@ static int vt8500_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, err = clk_enable(vt8500->clk); if (err < 0) { - dev_err(chip->dev, "failed to enable clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable clock\n"); return err; } @@ -129,7 +129,7 @@ static int vt8500_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) err = clk_enable(vt8500->clk); if (err < 0) { - dev_err(chip->dev, "failed to enable clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable clock\n"); return err; } From ae8635e99c5cc752e204ab9ee8869ec54a9223f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:15 +0100 Subject: [PATCH 0696/1038] pwm: vt8500: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-vt8500 driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Also convert the to_vt8500_chip() helper macro to a static inline to get some type safety. Link: https://lore.kernel.org/r/b203c4448db23ebad1165b7bce43ac41468c4e89.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-vt8500.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c index fb4dbc88b0a3..016c82d65527 100644 --- a/drivers/pwm/pwm-vt8500.c +++ b/drivers/pwm/pwm-vt8500.c @@ -45,12 +45,14 @@ #define STATUS_ALL_UPDATE 0x0F struct vt8500_chip { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; -#define to_vt8500_chip(chip) container_of(chip, struct vt8500_chip, chip) +static inline struct vt8500_chip *to_vt8500_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) static inline void vt8500_pwm_busy_wait(struct pwm_chip *chip, int nr, u8 bitmask) @@ -240,14 +242,12 @@ static int vt8500_pwm_probe(struct platform_device *pdev) if (!np) return dev_err_probe(&pdev->dev, -EINVAL, "invalid devicetree node\n"); - vt8500 = devm_kzalloc(&pdev->dev, sizeof(*vt8500), GFP_KERNEL); - if (vt8500 == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, VT8500_NR_PWMS, sizeof(*vt8500)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + vt8500 = to_vt8500_chip(chip); - chip = &vt8500->chip; - chip->dev = &pdev->dev; chip->ops = &vt8500_pwm_ops; - chip->npwm = VT8500_NR_PWMS; vt8500->clk = devm_clk_get_prepared(&pdev->dev, NULL); if (IS_ERR(vt8500->clk)) From 4caa211522f4799c18f1346f324272c3e9fe0c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:16 +0100 Subject: [PATCH 0697/1038] pwm: xilinx: Prepare removing pwm_chip from driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will drop struct pwm_chip chip from struct xilinx_pwm_device. Use the pwm_chip as driver data instead of the xilinx_pwm_device to get access to the pwm_chip in xilinx_pwm_remove() without using xilinx_pwm->chip. Link: https://lore.kernel.org/r/738b9929c1d13bde64050f8bbc4ce8d85f58cc7a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-xilinx.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-xilinx.c b/drivers/pwm/pwm-xilinx.c index 5f3c2a6fed11..0ca79fe8c105 100644 --- a/drivers/pwm/pwm-xilinx.c +++ b/drivers/pwm/pwm-xilinx.c @@ -214,6 +214,7 @@ static int xilinx_pwm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct xilinx_timer_priv *priv; + struct pwm_chip *chip; struct xilinx_pwm_device *xilinx_pwm; u32 pwm_cells, one_timer, width; void __iomem *regs; @@ -228,8 +229,9 @@ static int xilinx_pwm_probe(struct platform_device *pdev) xilinx_pwm = devm_kzalloc(dev, sizeof(*xilinx_pwm), GFP_KERNEL); if (!xilinx_pwm) return -ENOMEM; - platform_set_drvdata(pdev, xilinx_pwm); priv = &xilinx_pwm->priv; + chip = &xilinx_pwm->chip; + platform_set_drvdata(pdev, chip); regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(regs)) @@ -278,10 +280,10 @@ static int xilinx_pwm_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Clock enable failed\n"); clk_rate_exclusive_get(priv->clk); - xilinx_pwm->chip.dev = dev; - xilinx_pwm->chip.ops = &xilinx_pwm_ops; - xilinx_pwm->chip.npwm = 1; - ret = pwmchip_add(&xilinx_pwm->chip); + chip->dev = dev; + chip->ops = &xilinx_pwm_ops; + chip->npwm = 1; + ret = pwmchip_add(chip); if (ret) { clk_rate_exclusive_put(priv->clk); clk_disable_unprepare(priv->clk); @@ -293,11 +295,12 @@ static int xilinx_pwm_probe(struct platform_device *pdev) static void xilinx_pwm_remove(struct platform_device *pdev) { - struct xilinx_pwm_device *xilinx_pwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct xilinx_timer_priv *priv = xilinx_pwm_chip_to_priv(chip); - pwmchip_remove(&xilinx_pwm->chip); - clk_rate_exclusive_put(xilinx_pwm->priv.clk); - clk_disable_unprepare(xilinx_pwm->priv.clk); + pwmchip_remove(chip); + clk_rate_exclusive_put(priv->clk); + clk_disable_unprepare(priv->clk); } static const struct of_device_id xilinx_pwm_of_match[] = { From dda59d242477326968c322d930fe8f794e468c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:17 +0100 Subject: [PATCH 0698/1038] pwm: xilinx: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm-xilinx driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Link: https://lore.kernel.org/r/7cbc32771e94103b8c1c817cfdd613d7a2fc01b9.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-xilinx.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/drivers/pwm/pwm-xilinx.c b/drivers/pwm/pwm-xilinx.c index 0ca79fe8c105..3a7deebb0d0c 100644 --- a/drivers/pwm/pwm-xilinx.c +++ b/drivers/pwm/pwm-xilinx.c @@ -80,15 +80,10 @@ unsigned int xilinx_timer_get_period(struct xilinx_timer_priv *priv, #define TCSR_PWM_CLEAR (TCSR_MDT | TCSR_LOAD) #define TCSR_PWM_MASK (TCSR_PWM_SET | TCSR_PWM_CLEAR) -struct xilinx_pwm_device { - struct pwm_chip chip; - struct xilinx_timer_priv priv; -}; - static inline struct xilinx_timer_priv *xilinx_pwm_chip_to_priv(struct pwm_chip *chip) { - return &container_of(chip, struct xilinx_pwm_device, chip)->priv; + return pwmchip_get_drvdata(chip); } static bool xilinx_timer_pwm_enabled(u32 tcsr0, u32 tcsr1) @@ -215,7 +210,6 @@ static int xilinx_pwm_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; struct xilinx_timer_priv *priv; struct pwm_chip *chip; - struct xilinx_pwm_device *xilinx_pwm; u32 pwm_cells, one_timer, width; void __iomem *regs; @@ -226,11 +220,10 @@ static int xilinx_pwm_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "could not read #pwm-cells\n"); - xilinx_pwm = devm_kzalloc(dev, sizeof(*xilinx_pwm), GFP_KERNEL); - if (!xilinx_pwm) - return -ENOMEM; - priv = &xilinx_pwm->priv; - chip = &xilinx_pwm->chip; + chip = devm_pwmchip_alloc(dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = xilinx_pwm_chip_to_priv(chip); platform_set_drvdata(pdev, chip); regs = devm_platform_ioremap_resource(pdev, 0); @@ -280,9 +273,7 @@ static int xilinx_pwm_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Clock enable failed\n"); clk_rate_exclusive_get(priv->clk); - chip->dev = dev; chip->ops = &xilinx_pwm_ops; - chip->npwm = 1; ret = pwmchip_add(chip); if (ret) { clk_rate_exclusive_put(priv->clk); From 7ae57b104deb4b922cc7d0d123cd2f9c6baced76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:18 +0100 Subject: [PATCH 0699/1038] gpio: mvebu: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm sub-driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Acked-by: Linus Walleij Link: https://lore.kernel.org/r/2edc3adbb2c40b76b3b3dac82de82f3036bec1d5.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/gpio/gpio-mvebu.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index a13f3c18ccd4..8cfd3a89c018 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -99,7 +99,6 @@ struct mvebu_pwm { u32 offset; unsigned long clk_rate; struct gpio_desc *gpiod; - struct pwm_chip chip; spinlock_t lock; struct mvebu_gpio_chip *mvchip; @@ -615,7 +614,7 @@ static const struct regmap_config mvebu_gpio_regmap_config = { */ static struct mvebu_pwm *to_mvebu_pwm(struct pwm_chip *chip) { - return container_of(chip, struct mvebu_pwm, chip); + return pwmchip_get_drvdata(chip); } static int mvebu_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -789,6 +788,7 @@ static int mvebu_pwm_probe(struct platform_device *pdev, { struct device *dev = &pdev->dev; struct mvebu_pwm *mvpwm; + struct pwm_chip *chip; void __iomem *base; u32 offset; u32 set; @@ -813,9 +813,11 @@ static int mvebu_pwm_probe(struct platform_device *pdev, if (IS_ERR(mvchip->clk)) return PTR_ERR(mvchip->clk); - mvpwm = devm_kzalloc(dev, sizeof(struct mvebu_pwm), GFP_KERNEL); - if (!mvpwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, mvchip->chip.ngpio, sizeof(*mvpwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mvpwm = to_mvebu_pwm(chip); + mvchip->mvpwm = mvpwm; mvpwm->mvchip = mvchip; mvpwm->offset = offset; @@ -868,13 +870,11 @@ static int mvebu_pwm_probe(struct platform_device *pdev, return -EINVAL; } - mvpwm->chip.dev = dev; - mvpwm->chip.ops = &mvebu_pwm_ops; - mvpwm->chip.npwm = mvchip->chip.ngpio; + chip->ops = &mvebu_pwm_ops; spin_lock_init(&mvpwm->lock); - return devm_pwmchip_add(dev, &mvpwm->chip); + return devm_pwmchip_add(dev, chip); } #ifdef CONFIG_DEBUG_FS From 4867424625e0637caffcfb6252c29cf3537d5ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:19 +0100 Subject: [PATCH 0700/1038] drm/bridge: ti-sn65dsi86: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Acked-by: Douglas Anderson Link: https://lore.kernel.org/r/10a8d55110fc48a4759e65cc19556858587e94cc.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/gpu/drm/bridge/ti-sn65dsi86.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index 1f6e929c2f6a..f1fffbef3324 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -1415,7 +1415,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, int ret; if (!pdata->pwm_enabled) { - ret = pm_runtime_resume_and_get(chip->dev); + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (ret < 0) return ret; } @@ -1431,7 +1431,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, SN_GPIO_MUX_MASK << (2 * SN_PWM_GPIO_IDX), SN_GPIO_MUX_SPECIAL << (2 * SN_PWM_GPIO_IDX)); if (ret) { - dev_err(chip->dev, "failed to mux in PWM function\n"); + dev_err(pwmchip_parent(chip), "failed to mux in PWM function\n"); goto out; } } @@ -1507,7 +1507,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = regmap_write(pdata->regmap, SN_PWM_PRE_DIV_REG, pre_div); if (ret) { - dev_err(chip->dev, "failed to update PWM_PRE_DIV\n"); + dev_err(pwmchip_parent(chip), "failed to update PWM_PRE_DIV\n"); goto out; } @@ -1519,7 +1519,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, FIELD_PREP(SN_PWM_INV_MASK, state->polarity == PWM_POLARITY_INVERSED); ret = regmap_write(pdata->regmap, SN_PWM_EN_INV_REG, pwm_en_inv); if (ret) { - dev_err(chip->dev, "failed to update PWM_EN/PWM_INV\n"); + dev_err(pwmchip_parent(chip), "failed to update PWM_EN/PWM_INV\n"); goto out; } @@ -1527,7 +1527,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, out: if (!pdata->pwm_enabled) - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return ret; } From 596de87ddfc72fa2d62e1dab6be10415a6293565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:20 +0100 Subject: [PATCH 0701/1038] drm/bridge: ti-sn65dsi86: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm driver of the ti-sn65dsi86 to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Acked-by: Douglas Anderson Link: https://lore.kernel.org/r/a56cbaf049f5f23c0e0fe36b0799dd20189675e0.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/gpu/drm/bridge/ti-sn65dsi86.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index f1fffbef3324..7fbc307cc025 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -197,7 +197,7 @@ struct ti_sn65dsi86 { DECLARE_BITMAP(gchip_output, SN_NUM_GPIOS); #endif #if defined(CONFIG_PWM) - struct pwm_chip pchip; + struct pwm_chip *pchip; bool pwm_enabled; atomic_t pwm_pin_busy; #endif @@ -1374,7 +1374,7 @@ static void ti_sn_pwm_pin_release(struct ti_sn65dsi86 *pdata) static struct ti_sn65dsi86 *pwm_chip_to_ti_sn_bridge(struct pwm_chip *chip) { - return container_of(chip, struct ti_sn65dsi86, pchip); + return pwmchip_get_drvdata(chip); } static int ti_sn_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -1585,23 +1585,28 @@ static const struct pwm_ops ti_sn_pwm_ops = { static int ti_sn_pwm_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { + struct pwm_chip *chip; struct ti_sn65dsi86 *pdata = dev_get_drvdata(adev->dev.parent); - pdata->pchip.dev = &adev->dev; - pdata->pchip.ops = &ti_sn_pwm_ops; - pdata->pchip.npwm = 1; - pdata->pchip.of_xlate = of_pwm_single_xlate; + pdata->pchip = chip = devm_pwmchip_alloc(&adev->dev, 1, 0); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + pwmchip_set_drvdata(chip, pdata); + + chip->ops = &ti_sn_pwm_ops; + chip->of_xlate = of_pwm_single_xlate; devm_pm_runtime_enable(&adev->dev); - return pwmchip_add(&pdata->pchip); + return pwmchip_add(chip); } static void ti_sn_pwm_remove(struct auxiliary_device *adev) { struct ti_sn65dsi86 *pdata = dev_get_drvdata(adev->dev.parent); - pwmchip_remove(&pdata->pchip); + pwmchip_remove(pdata->pchip); if (pdata->pwm_enabled) pm_runtime_put_sync(&adev->dev); From 4b2b7b1e8730d51542c62ba75dabeb52243dfb49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:21 +0100 Subject: [PATCH 0702/1038] leds: qcom-lpg: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the pwm sub-driver to further changes of the pwm core outlined in the commit introducing devm_pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Acked-by: Lee Jones Link: https://lore.kernel.org/r/0be073477092eeccaac6c021cf07e38fc30c74fc.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/leds/rgb/leds-qcom-lpg.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index 156b73d1f4a2..0a7acf59a420 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -77,7 +77,7 @@ struct lpg { struct mutex lock; - struct pwm_chip pwm; + struct pwm_chip *pwm; const struct lpg_data *data; @@ -978,7 +978,7 @@ static int lpg_pattern_mc_clear(struct led_classdev *cdev) static inline struct lpg *lpg_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct lpg, pwm); + return pwmchip_get_drvdata(chip); } static int lpg_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -1093,13 +1093,17 @@ static const struct pwm_ops lpg_pwm_ops = { static int lpg_add_pwm(struct lpg *lpg) { + struct pwm_chip *chip; int ret; - lpg->pwm.dev = lpg->dev; - lpg->pwm.npwm = lpg->num_channels; - lpg->pwm.ops = &lpg_pwm_ops; + lpg->pwm = chip = devm_pwmchip_alloc(lpg->dev, lpg->num_channels, 0); + if (IS_ERR(chip)) + return PTR_ERR(chip); - ret = devm_pwmchip_add(lpg->dev, &lpg->pwm); + chip->ops = &lpg_pwm_ops; + pwmchip_set_drvdata(chip, lpg); + + ret = devm_pwmchip_add(lpg->dev, chip); if (ret) dev_err_probe(lpg->dev, ret, "failed to add PWM chip\n"); From 46eec872d512ccb44ae3f93ce25dd4875c1b24dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:22 +0100 Subject: [PATCH 0703/1038] staging: greybus: pwm: Change prototype of helpers to prepare further changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the driver for further changes that will make it harder to determine the pwm_chip from a given gb_pwm_chip. To just not have to do that, rework gb_pwm_activate_operation(), gb_pwm_deactivate_operation(), gb_pwm_config_operation(), gb_pwm_set_polarity_operation(), gb_pwm_enable_operation() and gb_pwm_disable_operation() to take a pwm_chip. Also use the pwm_chip as driver data instead of the gb_pwm_chip. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/ef9b346d5bab508d4ded81cf115bf244938d04f1.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/staging/greybus/pwm.c | 60 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index a3cb68cfa0f9..97b49d436c54 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -39,9 +39,9 @@ static int gb_pwm_count_operation(struct gb_pwm_chip *pwmc) return 0; } -static int gb_pwm_activate_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_activate_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_activate_request request; struct gbphy_device *gbphy_dev; int ret; @@ -51,7 +51,7 @@ static int gb_pwm_activate_operation(struct gb_pwm_chip *pwmc, request.which = which; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(chip->dev); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -64,9 +64,9 @@ static int gb_pwm_activate_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_deactivate_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_deactivate_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_deactivate_request request; struct gbphy_device *gbphy_dev; int ret; @@ -76,7 +76,7 @@ static int gb_pwm_deactivate_operation(struct gb_pwm_chip *pwmc, request.which = which; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(chip->dev); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -89,9 +89,10 @@ static int gb_pwm_deactivate_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_config_operation(struct gb_pwm_chip *pwmc, +static int gb_pwm_config_operation(struct pwm_chip *chip, u8 which, u32 duty, u32 period) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_config_request request; struct gbphy_device *gbphy_dev; int ret; @@ -103,7 +104,7 @@ static int gb_pwm_config_operation(struct gb_pwm_chip *pwmc, request.duty = cpu_to_le32(duty); request.period = cpu_to_le32(period); - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(chip->dev); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -116,9 +117,10 @@ static int gb_pwm_config_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_set_polarity_operation(struct gb_pwm_chip *pwmc, +static int gb_pwm_set_polarity_operation(struct pwm_chip *chip, u8 which, u8 polarity) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_polarity_request request; struct gbphy_device *gbphy_dev; int ret; @@ -129,7 +131,7 @@ static int gb_pwm_set_polarity_operation(struct gb_pwm_chip *pwmc, request.which = which; request.polarity = polarity; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(chip->dev); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -142,9 +144,9 @@ static int gb_pwm_set_polarity_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_enable_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_enable_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_enable_request request; struct gbphy_device *gbphy_dev; int ret; @@ -154,7 +156,7 @@ static int gb_pwm_enable_operation(struct gb_pwm_chip *pwmc, request.which = which; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(chip->dev); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -167,9 +169,9 @@ static int gb_pwm_enable_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_disable_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_disable_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_disable_request request; struct gbphy_device *gbphy_dev; int ret; @@ -182,7 +184,7 @@ static int gb_pwm_disable_operation(struct gb_pwm_chip *pwmc, ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_DISABLE, &request, sizeof(request), NULL, 0); - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(chip->dev); gbphy_runtime_put_autosuspend(gbphy_dev); return ret; @@ -190,19 +192,15 @@ static int gb_pwm_disable_operation(struct gb_pwm_chip *pwmc, static int gb_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { - struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); - - return gb_pwm_activate_operation(pwmc, pwm->hwpwm); + return gb_pwm_activate_operation(chip, pwm->hwpwm); }; static void gb_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { - struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); - if (pwm_is_enabled(pwm)) dev_warn(chip->dev, "freeing PWM device without disabling\n"); - gb_pwm_deactivate_operation(pwmc, pwm->hwpwm); + gb_pwm_deactivate_operation(chip, pwm->hwpwm); } static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -212,22 +210,21 @@ static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, bool enabled = pwm->state.enabled; u64 period = state->period; u64 duty_cycle = state->duty_cycle; - struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); /* Set polarity */ if (state->polarity != pwm->state.polarity) { if (enabled) { - gb_pwm_disable_operation(pwmc, pwm->hwpwm); + gb_pwm_disable_operation(chip, pwm->hwpwm); enabled = false; } - err = gb_pwm_set_polarity_operation(pwmc, pwm->hwpwm, state->polarity); + err = gb_pwm_set_polarity_operation(chip, pwm->hwpwm, state->polarity); if (err) return err; } if (!state->enabled) { if (enabled) - gb_pwm_disable_operation(pwmc, pwm->hwpwm); + gb_pwm_disable_operation(chip, pwm->hwpwm); return 0; } @@ -243,13 +240,13 @@ static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (duty_cycle > period) duty_cycle = period; - err = gb_pwm_config_operation(pwmc, pwm->hwpwm, duty_cycle, period); + err = gb_pwm_config_operation(chip, pwm->hwpwm, duty_cycle, period); if (err) return err; /* enable/disable */ if (!enabled) - return gb_pwm_enable_operation(pwmc, pwm->hwpwm); + return gb_pwm_enable_operation(chip, pwm->hwpwm); return 0; } @@ -282,7 +279,7 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, pwmc->connection = connection; gb_connection_set_data(connection, pwmc); - gb_gbphy_set_data(gbphy_dev, pwmc); + gb_gbphy_set_data(gbphy_dev, chip); ret = gb_connection_enable(connection); if (ret) @@ -320,7 +317,8 @@ exit_pwmc_free: static void gb_pwm_remove(struct gbphy_device *gbphy_dev) { - struct gb_pwm_chip *pwmc = gb_gbphy_get_data(gbphy_dev); + struct pwm_chip *chip = gb_gbphy_get_data(gbphy_dev); + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_connection *connection = pwmc->connection; int ret; @@ -328,7 +326,7 @@ static void gb_pwm_remove(struct gbphy_device *gbphy_dev) if (ret) gbphy_runtime_get_noresume(gbphy_dev); - pwmchip_remove(&pwmc->chip); + pwmchip_remove(chip); gb_connection_disable(connection); gb_connection_destroy(connection); kfree(pwmc); From 4aacf5fdf5aa759b3dbbee3e6b6ba59e564b6811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:23 +0100 Subject: [PATCH 0704/1038] staging: greybus: pwm: Make use of pwmchip_parent() accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct pwm_chip::dev is about to change. To not have to touch this driver in the same commit as struct pwm_chip::dev, use the accessor function provided for exactly this purpose. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/7e7517527b825a18ca10cb0faa837577d4f0ec8a.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/staging/greybus/pwm.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index 97b49d436c54..0cd1dab2d888 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -51,7 +51,7 @@ static int gb_pwm_activate_operation(struct pwm_chip *chip, u8 which) request.which = which; - gbphy_dev = to_gbphy_dev(chip->dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -76,7 +76,7 @@ static int gb_pwm_deactivate_operation(struct pwm_chip *chip, u8 which) request.which = which; - gbphy_dev = to_gbphy_dev(chip->dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -104,7 +104,7 @@ static int gb_pwm_config_operation(struct pwm_chip *chip, request.duty = cpu_to_le32(duty); request.period = cpu_to_le32(period); - gbphy_dev = to_gbphy_dev(chip->dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -131,7 +131,7 @@ static int gb_pwm_set_polarity_operation(struct pwm_chip *chip, request.which = which; request.polarity = polarity; - gbphy_dev = to_gbphy_dev(chip->dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -156,7 +156,7 @@ static int gb_pwm_enable_operation(struct pwm_chip *chip, u8 which) request.which = which; - gbphy_dev = to_gbphy_dev(chip->dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -184,7 +184,7 @@ static int gb_pwm_disable_operation(struct pwm_chip *chip, u8 which) ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_DISABLE, &request, sizeof(request), NULL, 0); - gbphy_dev = to_gbphy_dev(chip->dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); gbphy_runtime_put_autosuspend(gbphy_dev); return ret; @@ -198,7 +198,7 @@ static int gb_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) static void gb_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { if (pwm_is_enabled(pwm)) - dev_warn(chip->dev, "freeing PWM device without disabling\n"); + dev_warn(pwmchip_parent(chip), "freeing PWM device without disabling\n"); gb_pwm_deactivate_operation(chip, pwm->hwpwm); } From 9bda6a81ddaed8a61ca0211d6c14697177b98fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:24 +0100 Subject: [PATCH 0705/1038] staging: greybus: pwm: Rely on pwm framework to pass a valid hwpwm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pwm framework already asserts to only pass a hwpwm value (= which) less than npwm (= pwmc->pwm_max + 1). So there is no need to recheck this condition. Drop the respective checks. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/e003bc5e8e66f27f2b8fdc525a536d865888cffe.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/staging/greybus/pwm.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index 0cd1dab2d888..3099c2a3611c 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -46,9 +46,6 @@ static int gb_pwm_activate_operation(struct pwm_chip *chip, u8 which) struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); @@ -71,9 +68,6 @@ static int gb_pwm_deactivate_operation(struct pwm_chip *chip, u8 which) struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); @@ -97,9 +91,6 @@ static int gb_pwm_config_operation(struct pwm_chip *chip, struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; request.duty = cpu_to_le32(duty); request.period = cpu_to_le32(period); @@ -125,9 +116,6 @@ static int gb_pwm_set_polarity_operation(struct pwm_chip *chip, struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; request.polarity = polarity; @@ -151,9 +139,6 @@ static int gb_pwm_enable_operation(struct pwm_chip *chip, u8 which) struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); @@ -176,9 +161,6 @@ static int gb_pwm_disable_operation(struct pwm_chip *chip, u8 which) struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_DISABLE, From 1dd173fc96244d4b67f642cf90c29f21aa2b49b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:25 +0100 Subject: [PATCH 0706/1038] staging: greybus: pwm: Drop unused gb_connection_set_data() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver never calls gb_connection_get_data(). If there was another caller (say the greybus core) it cannot use the value because the type of pwmc (= struct gb_pwm_chip) is only defined in the pwm driver. So drop the call to gb_connection_set_data(). Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/bd2759c325c295f3d9f990609d97eb83a8ca88b8.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/staging/greybus/pwm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index 3099c2a3611c..c7a2e874a62b 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -260,7 +260,6 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, } pwmc->connection = connection; - gb_connection_set_data(connection, pwmc); gb_gbphy_set_data(gbphy_dev, chip); ret = gb_connection_enable(connection); From e0d32775716508942657cc5778bd881b40659287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:26 +0100 Subject: [PATCH 0707/1038] staging: greybus: pwm: Rework how the number of PWM lines is determined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With a later patch it becomes necessary to already now the number of PWM lines when pwmc is allocated. So make the function not use pwmc but a plain connection and return the number of lines instead of storing it in pwmc. This allows to get rid of the pwm_max member. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/3efd84ac03e7dc288f20b0de20b142b6404cb1fa.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/staging/greybus/pwm.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index c7a2e874a62b..35e98e7c00c1 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -16,8 +16,6 @@ struct gb_pwm_chip { struct gb_connection *connection; - u8 pwm_max; /* max pwm number */ - struct pwm_chip chip; }; @@ -26,17 +24,21 @@ static inline struct gb_pwm_chip *pwm_chip_to_gb_pwm_chip(struct pwm_chip *chip) return container_of(chip, struct gb_pwm_chip, chip); } -static int gb_pwm_count_operation(struct gb_pwm_chip *pwmc) +static int gb_pwm_get_npwm(struct gb_connection *connection) { struct gb_pwm_count_response response; int ret; - ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_PWM_COUNT, + ret = gb_operation_sync(connection, GB_PWM_TYPE_PWM_COUNT, NULL, 0, &response, sizeof(response)); if (ret) return ret; - pwmc->pwm_max = response.count; - return 0; + + /* + * The request returns the highest allowed PWM id parameter. So add one + * to get the number of PWMs. + */ + return response.count + 1; } static int gb_pwm_activate_operation(struct pwm_chip *chip, u8 which) @@ -245,7 +247,7 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, struct gb_connection *connection; struct gb_pwm_chip *pwmc; struct pwm_chip *chip; - int ret; + int ret, npwm; pwmc = kzalloc(sizeof(*pwmc), GFP_KERNEL); if (!pwmc) @@ -267,15 +269,16 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, goto exit_connection_destroy; /* Query number of pwms present */ - ret = gb_pwm_count_operation(pwmc); - if (ret) + ret = gb_pwm_get_npwm(connection); + if (ret < 0) goto exit_connection_disable; + npwm = ret; chip = &pwmc->chip; chip->dev = &gbphy_dev->dev; chip->ops = &gb_pwm_ops; - chip->npwm = pwmc->pwm_max + 1; + chip->npwm = npwm; ret = pwmchip_add(chip); if (ret) { From 1159c66fc77be66c948b45c6dc4e1d4e789725f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 14 Feb 2024 10:33:27 +0100 Subject: [PATCH 0708/1038] staging: greybus: pwm: Make use of devm_pwmchip_alloc() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the greybus pwm driver to further changes of the pwm core outlined in the commit introducing pwmchip_alloc(). There is no intended semantical change and the driver should behave as before. Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/3206ab7f49c2c1704ea69446f3b7a7d1e71200fa.1707900770.git.u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König --- drivers/staging/greybus/pwm.c | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index 35e98e7c00c1..01883fbcd79b 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -249,20 +249,11 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, struct pwm_chip *chip; int ret, npwm; - pwmc = kzalloc(sizeof(*pwmc), GFP_KERNEL); - if (!pwmc) - return -ENOMEM; - connection = gb_connection_create(gbphy_dev->bundle, le16_to_cpu(gbphy_dev->cport_desc->id), NULL); - if (IS_ERR(connection)) { - ret = PTR_ERR(connection); - goto exit_pwmc_free; - } - - pwmc->connection = connection; - gb_gbphy_set_data(gbphy_dev, chip); + if (IS_ERR(connection)) + return PTR_ERR(connection); ret = gb_connection_enable(connection); if (ret) @@ -274,28 +265,34 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, goto exit_connection_disable; npwm = ret; - chip = &pwmc->chip; + chip = pwmchip_alloc(&gbphy_dev->dev, npwm, sizeof(*pwmc)); + if (IS_ERR(chip)) { + ret = PTR_ERR(chip); + goto exit_connection_disable; + } + gb_gbphy_set_data(gbphy_dev, chip); + + pwmc = pwm_chip_to_gb_pwm_chip(chip); + pwmc->connection = connection; - chip->dev = &gbphy_dev->dev; chip->ops = &gb_pwm_ops; - chip->npwm = npwm; ret = pwmchip_add(chip); if (ret) { dev_err(&gbphy_dev->dev, "failed to register PWM: %d\n", ret); - goto exit_connection_disable; + goto exit_pwmchip_put; } gbphy_runtime_put_autosuspend(gbphy_dev); return 0; +exit_pwmchip_put: + pwmchip_put(chip); exit_connection_disable: gb_connection_disable(connection); exit_connection_destroy: gb_connection_destroy(connection); -exit_pwmc_free: - kfree(pwmc); return ret; } @@ -311,9 +308,9 @@ static void gb_pwm_remove(struct gbphy_device *gbphy_dev) gbphy_runtime_get_noresume(gbphy_dev); pwmchip_remove(chip); + pwmchip_put(chip); gb_connection_disable(connection); gb_connection_destroy(connection); - kfree(pwmc); } static const struct gbphy_device_id gb_pwm_id_table[] = { From 0eca305f8e0def5e742b3ce4ebc4ab5453ed1c0c Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Mon, 19 Feb 2024 14:35:46 +0100 Subject: [PATCH 0709/1038] dt-bindings: arm-smmu: Add QCM2290 GPU SMMU The GPU SMMU on QCM2290 nicely fits into the description we already have for SM61[12]5. Add it. Signed-off-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240219-topic-rb1_gpu-v1-1-d260fa854707@linaro.org Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index a4042ae24770..4a8d4022aebc 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -83,6 +83,7 @@ properties: - description: Qcom Adreno GPUs implementing "qcom,smmu-500" and "arm,mmu-500" items: - enum: + - qcom,qcm2290-smmu-500 - qcom,sa8775p-smmu-500 - qcom,sc7280-smmu-500 - qcom,sc8280xp-smmu-500 @@ -462,6 +463,7 @@ allOf: compatible: items: - enum: + - qcom,qcm2290-smmu-500 - qcom,sm6115-smmu-500 - qcom,sm6125-smmu-500 - const: qcom,adreno-smmu @@ -534,7 +536,6 @@ allOf: - cavium,smmu-v2 - marvell,ap806-smmu-500 - nvidia,smmu-500 - - qcom,qcm2290-smmu-500 - qcom,qdu1000-smmu-500 - qcom,sc7180-smmu-500 - qcom,sc8180x-smmu-500 From 12721e66005798b2a07bd8309060c230e52ba2b0 Mon Sep 17 00:00:00 2001 From: Abel Vesa Date: Wed, 31 Jan 2024 14:35:16 +0200 Subject: [PATCH 0710/1038] iommu/arm-smmu-qcom: Add X1E80100 MDSS compatible Add the X1E80100 MDSS compatible to clients compatible list, as it also needs the workarounds. Signed-off-by: Abel Vesa Link: https://lore.kernel.org/r/20240131-x1e80100-iommu-arm-smmu-qcom-v1-1-c1240419c718@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 8b04ece00420..5c7cfc51b57c 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -260,6 +260,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { { .compatible = "qcom,sm6375-mdss" }, { .compatible = "qcom,sm8150-mdss" }, { .compatible = "qcom,sm8250-mdss" }, + { .compatible = "qcom,x1e80100-mdss" }, { } }; From 2777781ca9b63c919bfe261aa2dbc1705067c59b Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Fri, 16 Feb 2024 12:03:49 +0100 Subject: [PATCH 0711/1038] dt-bindings: arm-smmu: Fix SM8[45]50 GPU SMMU 'if' condition The 'if' condition for the SM8[45]50 GPU SMMU is too large, add the other compatible strings to the condition to only allow the clocks for the GPU SMMU nodes. Fixes: 4fff78dc2490 ("dt-bindings: arm-smmu: Document SM8[45]50 GPU SMMU") Suggested-by: Dmitry Baryshkov Signed-off-by: Neil Armstrong Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216-topic-sm8650-gpu-v3-2-eb1f4b86d8d3@linaro.org Signed-off-by: Will Deacon --- .../devicetree/bindings/iommu/arm,smmu.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 4a8d4022aebc..7197e0eaa657 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -486,7 +486,12 @@ allOf: - if: properties: compatible: - const: qcom,sm8450-smmu-500 + items: + - const: qcom,sm8450-smmu-500 + - const: qcom,adreno-smmu + - const: qcom,smmu-500 + - const: arm,mmu-500 + then: properties: clock-names: @@ -510,7 +515,11 @@ allOf: - if: properties: compatible: - const: qcom,sm8550-smmu-500 + items: + - const: qcom,sm8550-smmu-500 + - const: qcom,adreno-smmu + - const: qcom,smmu-500 + - const: arm,mmu-500 then: properties: clock-names: From 8a05f74d567a556bde2c890e09914d7c56fe1ac3 Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Fri, 16 Feb 2024 12:03:50 +0100 Subject: [PATCH 0712/1038] dt-bindings: arm-smmu: Document SM8650 GPU SMMU Document the GPU SMMU found on the SM8650 platform. Signed-off-by: Neil Armstrong Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216-topic-sm8650-gpu-v3-3-eb1f4b86d8d3@linaro.org Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 7197e0eaa657..5c130cf06a21 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -94,6 +94,7 @@ properties: - qcom,sm8350-smmu-500 - qcom,sm8450-smmu-500 - qcom,sm8550-smmu-500 + - qcom,sm8650-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 - const: arm,mmu-500 @@ -516,7 +517,9 @@ allOf: properties: compatible: items: - - const: qcom,sm8550-smmu-500 + - enum: + - qcom,sm8550-smmu-500 + - qcom,sm8650-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 - const: arm,mmu-500 @@ -554,7 +557,6 @@ allOf: - qcom,sdx65-smmu-500 - qcom,sm6350-smmu-500 - qcom,sm6375-smmu-500 - - qcom,sm8650-smmu-500 - qcom,x1e80100-smmu-500 then: properties: From 3fec6e5961b77af6a952b77f5c2ea26f7513b216 Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Wed, 14 Feb 2024 13:09:32 +0530 Subject: [PATCH 0713/1038] PM: hibernate: Support to select compression algorithm Currently the default compression algorithm is selected based on compile time options. Introduce a module parameter "hibernate.compressor" to override this behaviour. Different compression algorithms have different characteristics and hibernation may benefit when it uses any of these algorithms, especially when a secondary algorithm(LZ4) offers better decompression speeds over a default algorithm(LZO), which in turn reduces hibernation image restore time. Users can override the default algorithm in two ways: 1) Passing "hibernate.compressor" as kernel command line parameter. Usage: LZO: hibernate.compressor=lzo LZ4: hibernate.compressor=lz4 2) Specifying the algorithm at runtime. Usage: LZO: echo lzo > /sys/module/hibernate/parameters/compressor LZ4: echo lz4 > /sys/module/hibernate/parameters/compressor Currently LZO and LZ4 are the supported algorithms. LZO is the default compression algorithm used with hibernation. Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki --- .../admin-guide/kernel-parameters.txt | 11 ++++ kernel/power/hibernate.c | 57 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 31b3a25680d0..8f7fb911b2cc 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1748,6 +1748,17 @@ (that will set all pages holding image data during restoration read-only). + hibernate.compressor= [HIBERNATION] Compression algorithm to be + used with hibernation. + Format: { lzo | lz4 } + Default: lzo + + lzo: Select LZO compression algorithm to + compress/decompress hibernation image. + + lz4: Select LZ4 compression algorithm to + compress/decompress hibernation image. + highmem=nn[KMG] [KNL,BOOT] forces the highmem zone to have an exact size of . This works even on boxes that have no highmem otherwise. This also works to reduce highmem diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 219191d6d0e8..43b1a82e800c 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -47,7 +47,7 @@ dev_t swsusp_resume_device; sector_t swsusp_resume_block; __visible int in_suspend __nosavedata; -static const char *default_compressor = CONFIG_HIBERNATION_DEF_COMP; +static char hibernate_compressor[CRYPTO_MAX_ALG_NAME] = CONFIG_HIBERNATION_DEF_COMP; /* * Compression/decompression algorithm to be used while saving/loading @@ -748,7 +748,7 @@ int hibernate(void) * Query for the compression algorithm support if compression is enabled. */ if (!nocompress) { - strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + strscpy(hib_comp_algo, hibernate_compressor, sizeof(hib_comp_algo)); if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { pr_err("%s compression is not available\n", hib_comp_algo); return -EOPNOTSUPP; @@ -999,7 +999,7 @@ static int software_resume(void) if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4) strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4, sizeof(hib_comp_algo)); else - strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + strscpy(hib_comp_algo, COMPRESSION_ALGO_LZO, sizeof(hib_comp_algo)); if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { pr_err("%s compression is not available\n", hib_comp_algo); error = -EOPNOTSUPP; @@ -1422,6 +1422,57 @@ static int __init nohibernate_setup(char *str) return 1; } +static const char * const comp_alg_enabled[] = { +#if IS_ENABLED(CONFIG_CRYPTO_LZO) + COMPRESSION_ALGO_LZO, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4) + COMPRESSION_ALGO_LZ4, +#endif +}; + +static int hibernate_compressor_param_set(const char *compressor, + const struct kernel_param *kp) +{ + unsigned int sleep_flags; + int index, ret; + + sleep_flags = lock_system_sleep(); + + index = sysfs_match_string(comp_alg_enabled, compressor); + if (index >= 0) { + ret = param_set_copystring(comp_alg_enabled[index], kp); + if (!ret) + strscpy(hib_comp_algo, comp_alg_enabled[index], + sizeof(hib_comp_algo)); + } else { + ret = index; + } + + unlock_system_sleep(sleep_flags); + + if (ret) + pr_debug("Cannot set specified compressor %s\n", + compressor); + + return ret; +} + +static const struct kernel_param_ops hibernate_compressor_param_ops = { + .set = hibernate_compressor_param_set, + .get = param_get_string, +}; + +static struct kparam_string hibernate_compressor_param_string = { + .maxlen = sizeof(hibernate_compressor), + .string = hibernate_compressor, +}; + +module_param_cb(compressor, &hibernate_compressor_param_ops, + &hibernate_compressor_param_string, 0644); +MODULE_PARM_DESC(compressor, + "Compression algorithm to be used with hibernation"); + __setup("noresume", noresume_setup); __setup("resume_offset=", resume_offset_setup); __setup("resume=", resume_setup); From f85450f134f0b4ca7e042dc3dc89155656a2299d Mon Sep 17 00:00:00 2001 From: Samasth Norway Ananda Date: Tue, 13 Feb 2024 16:19:56 -0800 Subject: [PATCH 0714/1038] tools/power x86_energy_perf_policy: Fix file leak in get_pkg_num() In function get_pkg_num() if fopen_or_die() succeeds it returns a file pointer to be used. But fclose() is never called before returning from the function. Signed-off-by: Samasth Norway Ananda Signed-off-by: Rafael J. Wysocki --- tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index 5fd9e594079c..ebda9c366b2b 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -1241,6 +1241,7 @@ unsigned int get_pkg_num(int cpu) retval = fscanf(fp, "%d\n", &pkg); if (retval != 1) errx(1, "%s: failed to parse", pathname); + fclose(fp); return pkg; } From f4311756a83fb01c28a9bf841cbb7eb2b318eebf Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Sun, 18 Feb 2024 09:40:58 +0100 Subject: [PATCH 0715/1038] PM: hibernate: Don't ignore return from set_memory_ro() set_memory_ro() and set_memory_rw() can fail, leaving memory unprotected. Take the returned value into account and abort in case of failure. Signed-off-by: Christophe Leroy Reviewed-by: Kees Cook Signed-off-by: Rafael J. Wysocki --- kernel/power/power.h | 2 +- kernel/power/snapshot.c | 25 ++++++++++++++++--------- kernel/power/swap.c | 8 ++++---- kernel/power/user.c | 4 +++- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/kernel/power/power.h b/kernel/power/power.h index 518349272848..de0e6b1077f2 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -153,7 +153,7 @@ extern unsigned int snapshot_additional_pages(struct zone *zone); extern unsigned long snapshot_get_image_size(void); extern int snapshot_read_next(struct snapshot_handle *handle); extern int snapshot_write_next(struct snapshot_handle *handle); -extern void snapshot_write_finalize(struct snapshot_handle *handle); +int snapshot_write_finalize(struct snapshot_handle *handle); extern int snapshot_image_loaded(struct snapshot_handle *handle); extern bool hibernate_acquire(void); diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 5c96ff067c64..405eddbda4fc 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -58,22 +58,24 @@ static inline void hibernate_restore_protection_end(void) hibernate_restore_protection_active = false; } -static inline void hibernate_restore_protect_page(void *page_address) +static inline int __must_check hibernate_restore_protect_page(void *page_address) { if (hibernate_restore_protection_active) - set_memory_ro((unsigned long)page_address, 1); + return set_memory_ro((unsigned long)page_address, 1); + return 0; } -static inline void hibernate_restore_unprotect_page(void *page_address) +static inline int hibernate_restore_unprotect_page(void *page_address) { if (hibernate_restore_protection_active) - set_memory_rw((unsigned long)page_address, 1); + return set_memory_rw((unsigned long)page_address, 1); + return 0; } #else static inline void hibernate_restore_protection_begin(void) {} static inline void hibernate_restore_protection_end(void) {} -static inline void hibernate_restore_protect_page(void *page_address) {} -static inline void hibernate_restore_unprotect_page(void *page_address) {} +static inline int __must_check hibernate_restore_protect_page(void *page_address) {return 0; } +static inline int hibernate_restore_unprotect_page(void *page_address) {return 0; } #endif /* CONFIG_STRICT_KERNEL_RWX && CONFIG_ARCH_HAS_SET_MEMORY */ @@ -2832,7 +2834,9 @@ next: } } else { copy_last_highmem_page(); - hibernate_restore_protect_page(handle->buffer); + error = hibernate_restore_protect_page(handle->buffer); + if (error) + return error; handle->buffer = get_buffer(&orig_bm, &ca); if (IS_ERR(handle->buffer)) return PTR_ERR(handle->buffer); @@ -2858,15 +2862,18 @@ next: * stored in highmem. Additionally, it recycles bitmap memory that's not * necessary any more. */ -void snapshot_write_finalize(struct snapshot_handle *handle) +int snapshot_write_finalize(struct snapshot_handle *handle) { + int error; + copy_last_highmem_page(); - hibernate_restore_protect_page(handle->buffer); + error = hibernate_restore_protect_page(handle->buffer); /* Do that only if we have loaded the image entirely */ if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) { memory_bm_recycle(&orig_bm); free_highmem_data(); } + return error; } int snapshot_image_loaded(struct snapshot_handle *handle) diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 6513035f2f7f..364342cc7f2d 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -1134,8 +1134,8 @@ static int load_image(struct swap_map_handle *handle, ret = err2; if (!ret) { pr_info("Image loading done\n"); - snapshot_write_finalize(snapshot); - if (!snapshot_image_loaded(snapshot)) + ret = snapshot_write_finalize(snapshot); + if (!ret && !snapshot_image_loaded(snapshot)) ret = -ENODATA; } swsusp_show_speed(start, stop, nr_to_read, "Read"); @@ -1486,8 +1486,8 @@ out_finish: stop = ktime_get(); if (!ret) { pr_info("Image loading done\n"); - snapshot_write_finalize(snapshot); - if (!snapshot_image_loaded(snapshot)) + ret = snapshot_write_finalize(snapshot); + if (!ret && !snapshot_image_loaded(snapshot)) ret = -ENODATA; if (!ret) { if (swsusp_header->flags & SF_CRC32_MODE) { diff --git a/kernel/power/user.c b/kernel/power/user.c index 3a4e70366f35..3aa41ba22129 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -317,7 +317,9 @@ static long snapshot_ioctl(struct file *filp, unsigned int cmd, break; case SNAPSHOT_ATOMIC_RESTORE: - snapshot_write_finalize(&data->handle); + error = snapshot_write_finalize(&data->handle); + if (error) + break; if (data->mode != O_WRONLY || !data->frozen || !snapshot_image_loaded(&data->handle)) { error = -EPERM; From 3a561ea2413ea5a740f3b1d6b5355d46f88a7456 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Wed, 21 Feb 2024 14:25:50 +0000 Subject: [PATCH 0716/1038] PM: EM: Fix nr_states warnings in static checks During the static checks nr_states has been mentioned by the kernel test robot. Fix the warning in those 2 places. Reported-by: kernel test robot Signed-off-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- kernel/power/energy_model.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 7101fa3fa0c0..b686ac0345bd 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -280,6 +280,7 @@ static int em_compute_costs(struct device *dev, struct em_perf_state *table, * em_dev_compute_costs() - Calculate cost values for new runtime EM table * @dev : Device for which the EM table is to be updated * @table : The new EM table that is going to get the costs calculated + * @nr_states : Number of performance states * * Calculate the em_perf_state::cost values for new runtime EM table. The * values are used for EAS during task placement. It also calculates and sets @@ -728,7 +729,6 @@ static void em_check_capacity_update(void) struct cpufreq_policy *policy; unsigned long em_max_perf; struct device *dev; - int nr_states; if (cpumask_test_cpu(cpu, cpu_done_mask)) continue; @@ -749,7 +749,6 @@ static void em_check_capacity_update(void) cpumask_or(cpu_done_mask, cpu_done_mask, em_span_cpus(pd)); - nr_states = pd->nr_perf_states; cpu_capacity = arch_scale_cpu_capacity(cpu); rcu_read_lock(); From 015abee404760249a5c968b9ce29216b94b8ced1 Mon Sep 17 00:00:00 2001 From: Vilas Bhat Date: Wed, 21 Feb 2024 19:47:53 +0000 Subject: [PATCH 0717/1038] PM: runtime: add tracepoint for runtime_status changes Existing runtime PM ftrace events (`rpm_suspend`, `rpm_resume`, `rpm_return_int`) offer limited visibility into the exact timing of device runtime power state transitions, particularly when asynchronous operations are involved. When the `rpm_suspend` or `rpm_resume` functions are invoked with the `RPM_ASYNC` flag, a return value of 0 i.e., success merely indicates that the device power state request has been queued, not that the device has yet transitioned. A new ftrace event, `rpm_status`, is introduced. This event directly logs the `power.runtime_status` value of a device whenever it changes providing granular tracking of runtime power state transitions regardless of synchronous or asynchronous `rpm_suspend` / `rpm_resume` usage. Signed-off-by: Vilas Bhat Signed-off-by: Rafael J. Wysocki --- drivers/base/power/runtime.c | 1 + include/trace/events/rpm.h | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 5275a6b2e980..2ee45841486b 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -94,6 +94,7 @@ static void update_pm_runtime_accounting(struct device *dev) static void __update_runtime_status(struct device *dev, enum rpm_status status) { update_pm_runtime_accounting(dev); + trace_rpm_status(dev, status); dev->power.runtime_status = status; } diff --git a/include/trace/events/rpm.h b/include/trace/events/rpm.h index 3c716214dab1..bd120e23ce12 100644 --- a/include/trace/events/rpm.h +++ b/include/trace/events/rpm.h @@ -101,6 +101,48 @@ TRACE_EVENT(rpm_return_int, __entry->ret) ); +#define RPM_STATUS_STRINGS \ + EM(RPM_INVALID, "RPM_INVALID") \ + EM(RPM_ACTIVE, "RPM_ACTIVE") \ + EM(RPM_RESUMING, "RPM_RESUMING") \ + EM(RPM_SUSPENDED, "RPM_SUSPENDED") \ + EMe(RPM_SUSPENDING, "RPM_SUSPENDING") + +/* Enums require being exported to userspace, for user tool parsing. */ +#undef EM +#undef EMe +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define EMe(a, b) TRACE_DEFINE_ENUM(a); + +RPM_STATUS_STRINGS + +/* + * Now redefine the EM() and EMe() macros to map the enums to the strings that + * will be printed in the output. + */ +#undef EM +#undef EMe +#define EM(a, b) { a, b }, +#define EMe(a, b) { a, b } + +TRACE_EVENT(rpm_status, + TP_PROTO(struct device *dev, enum rpm_status status), + TP_ARGS(dev, status), + + TP_STRUCT__entry( + __string(name, dev_name(dev)) + __field(int, status) + ), + + TP_fast_assign( + __assign_str(name, dev_name(dev)); + __entry->status = status; + ), + + TP_printk("%s status=%s", __get_str(name), + __print_symbolic(__entry->status, RPM_STATUS_STRINGS)) +); + #endif /* _TRACE_RUNTIME_POWER_H */ /* This part must be outside protection */ From 2b959bdde4dde14a789070c04b2b8c029884db85 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 22 Feb 2024 10:57:23 +0000 Subject: [PATCH 0718/1038] ACPI: thermal_lib: Initialize temp_decik to zero Static analysis with clang scan build is warning that uninitialized data is being passed into various functions. Stop these warnings by initializing temp_decik to zero. Cleans up clang scan warnings in lines 106, 125, 146 and 164 such as: drivers/acpi/thermal_lib.c:106:9: warning: 2nd function call argument is an uninitialized value [core.CallAndMessage] Kudos to Dan Carpenter for the deeper analysis of this issue. Signed-off-by: Colin Ian King Signed-off-by: Rafael J. Wysocki --- drivers/acpi/thermal_lib.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/thermal_lib.c b/drivers/acpi/thermal_lib.c index 4e0519ca9739..6214d6ebe1fa 100644 --- a/drivers/acpi/thermal_lib.c +++ b/drivers/acpi/thermal_lib.c @@ -100,7 +100,7 @@ static int thermal_temp(int error, int temp_decik, int *ret_temp) */ int thermal_acpi_active_trip_temp(struct acpi_device *adev, int id, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_active_trip_temp(adev, id, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); @@ -119,7 +119,7 @@ EXPORT_SYMBOL_GPL(thermal_acpi_active_trip_temp); */ int thermal_acpi_passive_trip_temp(struct acpi_device *adev, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_passive_trip_temp(adev, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); @@ -139,7 +139,7 @@ EXPORT_SYMBOL_GPL(thermal_acpi_passive_trip_temp); */ int thermal_acpi_hot_trip_temp(struct acpi_device *adev, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_hot_trip_temp(adev, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); @@ -158,7 +158,7 @@ EXPORT_SYMBOL_GPL(thermal_acpi_hot_trip_temp); */ int thermal_acpi_critical_trip_temp(struct acpi_device *adev, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_critical_trip_temp(adev, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); From 260017f31a8c3879be5f9048a46f382b06c1923a Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Tue, 30 Jan 2024 13:56:59 +0100 Subject: [PATCH 0719/1038] lsm: use default hook return value in call_int_hook() Change the definition of call_int_hook() to treat LSM_RET_DEFAULT(...) as the "continue" value instead of 0. To further simplify this macro, also drop the IRC argument and replace it with LSM_RET_DEFAULT(...). After this the macro can be used in a couple more hooks, where similar logic is currently open-coded. At the same time, some other existing call_int_hook() users now need to be open-coded, but overall it's still a net simplification. There should be no functional change resulting from this patch. Signed-off-by: Ondrej Mosnacek Reviewed-by: Casey Schaufler [PM: merge fuzz due to other hook changes, tweaks from list discussion] Signed-off-by: Paul Moore --- security/security.c | 532 +++++++++++++++++++------------------------- 1 file changed, 225 insertions(+), 307 deletions(-) diff --git a/security/security.c b/security/security.c index c31e45da672b..4cb832b00c40 100644 --- a/security/security.c +++ b/security/security.c @@ -855,14 +855,14 @@ out: P->hook.FUNC(__VA_ARGS__); \ } while (0) -#define call_int_hook(FUNC, IRC, ...) ({ \ - int RC = IRC; \ +#define call_int_hook(FUNC, ...) ({ \ + int RC = LSM_RET_DEFAULT(FUNC); \ do { \ struct security_hook_list *P; \ \ hlist_for_each_entry(P, &security_hook_heads.FUNC, list) { \ RC = P->hook.FUNC(__VA_ARGS__); \ - if (RC != 0) \ + if (RC != LSM_RET_DEFAULT(FUNC)) \ break; \ } \ } while (0); \ @@ -881,7 +881,7 @@ out: */ int security_binder_set_context_mgr(const struct cred *mgr) { - return call_int_hook(binder_set_context_mgr, 0, mgr); + return call_int_hook(binder_set_context_mgr, mgr); } /** @@ -896,7 +896,7 @@ int security_binder_set_context_mgr(const struct cred *mgr) int security_binder_transaction(const struct cred *from, const struct cred *to) { - return call_int_hook(binder_transaction, 0, from, to); + return call_int_hook(binder_transaction, from, to); } /** @@ -911,7 +911,7 @@ int security_binder_transaction(const struct cred *from, int security_binder_transfer_binder(const struct cred *from, const struct cred *to) { - return call_int_hook(binder_transfer_binder, 0, from, to); + return call_int_hook(binder_transfer_binder, from, to); } /** @@ -927,7 +927,7 @@ int security_binder_transfer_binder(const struct cred *from, int security_binder_transfer_file(const struct cred *from, const struct cred *to, const struct file *file) { - return call_int_hook(binder_transfer_file, 0, from, to, file); + return call_int_hook(binder_transfer_file, from, to, file); } /** @@ -946,7 +946,7 @@ int security_binder_transfer_file(const struct cred *from, */ int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { - return call_int_hook(ptrace_access_check, 0, child, mode); + return call_int_hook(ptrace_access_check, child, mode); } /** @@ -961,7 +961,7 @@ int security_ptrace_access_check(struct task_struct *child, unsigned int mode) */ int security_ptrace_traceme(struct task_struct *parent) { - return call_int_hook(ptrace_traceme, 0, parent); + return call_int_hook(ptrace_traceme, parent); } /** @@ -983,8 +983,7 @@ int security_capget(const struct task_struct *target, kernel_cap_t *inheritable, kernel_cap_t *permitted) { - return call_int_hook(capget, 0, target, - effective, inheritable, permitted); + return call_int_hook(capget, target, effective, inheritable, permitted); } /** @@ -1005,8 +1004,8 @@ int security_capset(struct cred *new, const struct cred *old, const kernel_cap_t *inheritable, const kernel_cap_t *permitted) { - return call_int_hook(capset, 0, new, old, - effective, inheritable, permitted); + return call_int_hook(capset, new, old, effective, inheritable, + permitted); } /** @@ -1027,7 +1026,7 @@ int security_capable(const struct cred *cred, int cap, unsigned int opts) { - return call_int_hook(capable, 0, cred, ns, cap, opts); + return call_int_hook(capable, cred, ns, cap, opts); } /** @@ -1043,7 +1042,7 @@ int security_capable(const struct cred *cred, */ int security_quotactl(int cmds, int type, int id, const struct super_block *sb) { - return call_int_hook(quotactl, 0, cmds, type, id, sb); + return call_int_hook(quotactl, cmds, type, id, sb); } /** @@ -1056,7 +1055,7 @@ int security_quotactl(int cmds, int type, int id, const struct super_block *sb) */ int security_quota_on(struct dentry *dentry) { - return call_int_hook(quota_on, 0, dentry); + return call_int_hook(quota_on, dentry); } /** @@ -1071,7 +1070,7 @@ int security_quota_on(struct dentry *dentry) */ int security_syslog(int type) { - return call_int_hook(syslog, 0, type); + return call_int_hook(syslog, type); } /** @@ -1086,7 +1085,7 @@ int security_syslog(int type) */ int security_settime64(const struct timespec64 *ts, const struct timezone *tz) { - return call_int_hook(settime, 0, ts, tz); + return call_int_hook(settime, ts, tz); } /** @@ -1141,7 +1140,7 @@ int security_vm_enough_memory_mm(struct mm_struct *mm, long pages) */ int security_bprm_creds_for_exec(struct linux_binprm *bprm) { - return call_int_hook(bprm_creds_for_exec, 0, bprm); + return call_int_hook(bprm_creds_for_exec, bprm); } /** @@ -1165,7 +1164,7 @@ int security_bprm_creds_for_exec(struct linux_binprm *bprm) */ int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file) { - return call_int_hook(bprm_creds_from_file, 0, bprm, file); + return call_int_hook(bprm_creds_from_file, bprm, file); } /** @@ -1182,7 +1181,7 @@ int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file * */ int security_bprm_check(struct linux_binprm *bprm) { - return call_int_hook(bprm_check_security, 0, bprm); + return call_int_hook(bprm_check_security, bprm); } /** @@ -1229,7 +1228,7 @@ void security_bprm_committed_creds(const struct linux_binprm *bprm) */ int security_fs_context_submount(struct fs_context *fc, struct super_block *reference) { - return call_int_hook(fs_context_submount, 0, fc, reference); + return call_int_hook(fs_context_submount, fc, reference); } /** @@ -1245,7 +1244,7 @@ int security_fs_context_submount(struct fs_context *fc, struct super_block *refe */ int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc) { - return call_int_hook(fs_context_dup, 0, fc, src_fc); + return call_int_hook(fs_context_dup, fc, src_fc); } /** @@ -1294,7 +1293,7 @@ int security_sb_alloc(struct super_block *sb) if (unlikely(rc)) return rc; - rc = call_int_hook(sb_alloc_security, 0, sb); + rc = call_int_hook(sb_alloc_security, sb); if (unlikely(rc)) security_sb_free(sb); return rc; @@ -1352,7 +1351,7 @@ EXPORT_SYMBOL(security_free_mnt_opts); */ int security_sb_eat_lsm_opts(char *options, void **mnt_opts) { - return call_int_hook(sb_eat_lsm_opts, 0, options, mnt_opts); + return call_int_hook(sb_eat_lsm_opts, options, mnt_opts); } EXPORT_SYMBOL(security_sb_eat_lsm_opts); @@ -1369,7 +1368,7 @@ EXPORT_SYMBOL(security_sb_eat_lsm_opts); int security_sb_mnt_opts_compat(struct super_block *sb, void *mnt_opts) { - return call_int_hook(sb_mnt_opts_compat, 0, sb, mnt_opts); + return call_int_hook(sb_mnt_opts_compat, sb, mnt_opts); } EXPORT_SYMBOL(security_sb_mnt_opts_compat); @@ -1386,7 +1385,7 @@ EXPORT_SYMBOL(security_sb_mnt_opts_compat); int security_sb_remount(struct super_block *sb, void *mnt_opts) { - return call_int_hook(sb_remount, 0, sb, mnt_opts); + return call_int_hook(sb_remount, sb, mnt_opts); } EXPORT_SYMBOL(security_sb_remount); @@ -1400,7 +1399,7 @@ EXPORT_SYMBOL(security_sb_remount); */ int security_sb_kern_mount(const struct super_block *sb) { - return call_int_hook(sb_kern_mount, 0, sb); + return call_int_hook(sb_kern_mount, sb); } /** @@ -1414,7 +1413,7 @@ int security_sb_kern_mount(const struct super_block *sb) */ int security_sb_show_options(struct seq_file *m, struct super_block *sb) { - return call_int_hook(sb_show_options, 0, m, sb); + return call_int_hook(sb_show_options, m, sb); } /** @@ -1428,7 +1427,7 @@ int security_sb_show_options(struct seq_file *m, struct super_block *sb) */ int security_sb_statfs(struct dentry *dentry) { - return call_int_hook(sb_statfs, 0, dentry); + return call_int_hook(sb_statfs, dentry); } /** @@ -1451,7 +1450,7 @@ int security_sb_statfs(struct dentry *dentry) int security_sb_mount(const char *dev_name, const struct path *path, const char *type, unsigned long flags, void *data) { - return call_int_hook(sb_mount, 0, dev_name, path, type, flags, data); + return call_int_hook(sb_mount, dev_name, path, type, flags, data); } /** @@ -1465,7 +1464,7 @@ int security_sb_mount(const char *dev_name, const struct path *path, */ int security_sb_umount(struct vfsmount *mnt, int flags) { - return call_int_hook(sb_umount, 0, mnt, flags); + return call_int_hook(sb_umount, mnt, flags); } /** @@ -1480,7 +1479,7 @@ int security_sb_umount(struct vfsmount *mnt, int flags) int security_sb_pivotroot(const struct path *old_path, const struct path *new_path) { - return call_int_hook(sb_pivotroot, 0, old_path, new_path); + return call_int_hook(sb_pivotroot, old_path, new_path); } /** @@ -1499,9 +1498,17 @@ int security_sb_set_mnt_opts(struct super_block *sb, unsigned long kern_flags, unsigned long *set_kern_flags) { - return call_int_hook(sb_set_mnt_opts, - mnt_opts ? -EOPNOTSUPP : 0, sb, - mnt_opts, kern_flags, set_kern_flags); + struct security_hook_list *hp; + int rc = mnt_opts ? -EOPNOTSUPP : LSM_RET_DEFAULT(sb_set_mnt_opts); + + hlist_for_each_entry(hp, &security_hook_heads.sb_set_mnt_opts, + list) { + rc = hp->hook.sb_set_mnt_opts(sb, mnt_opts, kern_flags, + set_kern_flags); + if (rc != LSM_RET_DEFAULT(sb_set_mnt_opts)) + break; + } + return rc; } EXPORT_SYMBOL(security_sb_set_mnt_opts); @@ -1521,7 +1528,7 @@ int security_sb_clone_mnt_opts(const struct super_block *oldsb, unsigned long kern_flags, unsigned long *set_kern_flags) { - return call_int_hook(sb_clone_mnt_opts, 0, oldsb, newsb, + return call_int_hook(sb_clone_mnt_opts, oldsb, newsb, kern_flags, set_kern_flags); } EXPORT_SYMBOL(security_sb_clone_mnt_opts); @@ -1538,7 +1545,7 @@ EXPORT_SYMBOL(security_sb_clone_mnt_opts); int security_move_mount(const struct path *from_path, const struct path *to_path) { - return call_int_hook(move_mount, 0, from_path, to_path); + return call_int_hook(move_mount, from_path, to_path); } /** @@ -1555,7 +1562,7 @@ int security_move_mount(const struct path *from_path, int security_path_notify(const struct path *path, u64 mask, unsigned int obj_type) { - return call_int_hook(path_notify, 0, path, mask, obj_type); + return call_int_hook(path_notify, path, mask, obj_type); } /** @@ -1574,7 +1581,7 @@ int security_inode_alloc(struct inode *inode) if (unlikely(rc)) return rc; - rc = call_int_hook(inode_alloc_security, 0, inode); + rc = call_int_hook(inode_alloc_security, inode); if (unlikely(rc)) security_inode_free(inode); return rc; @@ -1631,20 +1638,8 @@ int security_dentry_init_security(struct dentry *dentry, int mode, const char **xattr_name, void **ctx, u32 *ctxlen) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.dentry_init_security, - list) { - rc = hp->hook.dentry_init_security(dentry, mode, name, - xattr_name, ctx, ctxlen); - if (rc != LSM_RET_DEFAULT(dentry_init_security)) - return rc; - } - return LSM_RET_DEFAULT(dentry_init_security); + return call_int_hook(dentry_init_security, dentry, mode, name, + xattr_name, ctx, ctxlen); } EXPORT_SYMBOL(security_dentry_init_security); @@ -1667,7 +1662,7 @@ int security_dentry_create_files_as(struct dentry *dentry, int mode, struct qstr *name, const struct cred *old, struct cred *new) { - return call_int_hook(dentry_create_files_as, 0, dentry, mode, + return call_int_hook(dentry_create_files_as, dentry, mode, name, old, new); } EXPORT_SYMBOL(security_dentry_create_files_as); @@ -1764,7 +1759,7 @@ int security_inode_init_security_anon(struct inode *inode, const struct qstr *name, const struct inode *context_inode) { - return call_int_hook(inode_init_security_anon, 0, inode, name, + return call_int_hook(inode_init_security_anon, inode, name, context_inode); } @@ -1786,7 +1781,7 @@ int security_path_mknod(const struct path *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_mknod, 0, dir, dentry, mode, dev); + return call_int_hook(path_mknod, dir, dentry, mode, dev); } EXPORT_SYMBOL(security_path_mknod); @@ -1819,7 +1814,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_mkdir, 0, dir, dentry, mode); + return call_int_hook(path_mkdir, dir, dentry, mode); } EXPORT_SYMBOL(security_path_mkdir); @@ -1836,7 +1831,7 @@ int security_path_rmdir(const struct path *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_rmdir, 0, dir, dentry); + return call_int_hook(path_rmdir, dir, dentry); } /** @@ -1852,7 +1847,7 @@ int security_path_unlink(const struct path *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_unlink, 0, dir, dentry); + return call_int_hook(path_unlink, dir, dentry); } EXPORT_SYMBOL(security_path_unlink); @@ -1871,7 +1866,7 @@ int security_path_symlink(const struct path *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_symlink, 0, dir, dentry, old_name); + return call_int_hook(path_symlink, dir, dentry, old_name); } /** @@ -1889,7 +1884,7 @@ int security_path_link(struct dentry *old_dentry, const struct path *new_dir, { if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)))) return 0; - return call_int_hook(path_link, 0, old_dentry, new_dir, new_dentry); + return call_int_hook(path_link, old_dentry, new_dir, new_dentry); } /** @@ -1913,7 +1908,7 @@ int security_path_rename(const struct path *old_dir, struct dentry *old_dentry, IS_PRIVATE(d_backing_inode(new_dentry))))) return 0; - return call_int_hook(path_rename, 0, old_dir, old_dentry, new_dir, + return call_int_hook(path_rename, old_dir, old_dentry, new_dir, new_dentry, flags); } EXPORT_SYMBOL(security_path_rename); @@ -1932,7 +1927,7 @@ int security_path_truncate(const struct path *path) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(path_truncate, 0, path); + return call_int_hook(path_truncate, path); } /** @@ -1950,7 +1945,7 @@ int security_path_chmod(const struct path *path, umode_t mode) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(path_chmod, 0, path, mode); + return call_int_hook(path_chmod, path, mode); } /** @@ -1967,7 +1962,7 @@ int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(path_chown, 0, path, uid, gid); + return call_int_hook(path_chown, path, uid, gid); } /** @@ -1980,7 +1975,7 @@ int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) */ int security_path_chroot(const struct path *path) { - return call_int_hook(path_chroot, 0, path); + return call_int_hook(path_chroot, path); } #endif /* CONFIG_SECURITY_PATH */ @@ -1999,7 +1994,7 @@ int security_inode_create(struct inode *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_create, 0, dir, dentry, mode); + return call_int_hook(inode_create, dir, dentry, mode); } EXPORT_SYMBOL_GPL(security_inode_create); @@ -2033,7 +2028,7 @@ int security_inode_link(struct dentry *old_dentry, struct inode *dir, { if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)))) return 0; - return call_int_hook(inode_link, 0, old_dentry, dir, new_dentry); + return call_int_hook(inode_link, old_dentry, dir, new_dentry); } /** @@ -2049,7 +2044,7 @@ int security_inode_unlink(struct inode *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_unlink, 0, dir, dentry); + return call_int_hook(inode_unlink, dir, dentry); } /** @@ -2067,7 +2062,7 @@ int security_inode_symlink(struct inode *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_symlink, 0, dir, dentry, old_name); + return call_int_hook(inode_symlink, dir, dentry, old_name); } /** @@ -2085,7 +2080,7 @@ int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_mkdir, 0, dir, dentry, mode); + return call_int_hook(inode_mkdir, dir, dentry, mode); } EXPORT_SYMBOL_GPL(security_inode_mkdir); @@ -2102,7 +2097,7 @@ int security_inode_rmdir(struct inode *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_rmdir, 0, dir, dentry); + return call_int_hook(inode_rmdir, dir, dentry); } /** @@ -2124,7 +2119,7 @@ int security_inode_mknod(struct inode *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_mknod, 0, dir, dentry, mode, dev); + return call_int_hook(inode_mknod, dir, dentry, mode, dev); } /** @@ -2149,13 +2144,13 @@ int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry, return 0; if (flags & RENAME_EXCHANGE) { - int err = call_int_hook(inode_rename, 0, new_dir, new_dentry, + int err = call_int_hook(inode_rename, new_dir, new_dentry, old_dir, old_dentry); if (err) return err; } - return call_int_hook(inode_rename, 0, old_dir, old_dentry, + return call_int_hook(inode_rename, old_dir, old_dentry, new_dir, new_dentry); } @@ -2171,7 +2166,7 @@ int security_inode_readlink(struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_readlink, 0, dentry); + return call_int_hook(inode_readlink, dentry); } /** @@ -2190,7 +2185,7 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode, { if (unlikely(IS_PRIVATE(inode))) return 0; - return call_int_hook(inode_follow_link, 0, dentry, inode, rcu); + return call_int_hook(inode_follow_link, dentry, inode, rcu); } /** @@ -2211,7 +2206,7 @@ int security_inode_permission(struct inode *inode, int mask) { if (unlikely(IS_PRIVATE(inode))) return 0; - return call_int_hook(inode_permission, 0, inode, mask); + return call_int_hook(inode_permission, inode, mask); } /** @@ -2232,7 +2227,7 @@ int security_inode_setattr(struct mnt_idmap *idmap, { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_setattr, 0, idmap, dentry, attr); + return call_int_hook(inode_setattr, idmap, dentry, attr); } EXPORT_SYMBOL_GPL(security_inode_setattr); @@ -2264,7 +2259,7 @@ int security_inode_getattr(const struct path *path) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(inode_getattr, 0, path); + return call_int_hook(inode_getattr, path); } /** @@ -2292,8 +2287,8 @@ int security_inode_setxattr(struct mnt_idmap *idmap, * SELinux and Smack integrate the cap call, * so assume that all LSMs supplying this call do so. */ - ret = call_int_hook(inode_setxattr, 1, idmap, dentry, name, value, - size, flags); + ret = call_int_hook(inode_setxattr, idmap, dentry, name, value, size, + flags); if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); @@ -2318,8 +2313,7 @@ int security_inode_set_acl(struct mnt_idmap *idmap, { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, - kacl); + return call_int_hook(inode_set_acl, idmap, dentry, acl_name, kacl); } /** @@ -2355,7 +2349,7 @@ int security_inode_get_acl(struct mnt_idmap *idmap, { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_get_acl, 0, idmap, dentry, acl_name); + return call_int_hook(inode_get_acl, idmap, dentry, acl_name); } /** @@ -2374,7 +2368,7 @@ int security_inode_remove_acl(struct mnt_idmap *idmap, { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); + return call_int_hook(inode_remove_acl, idmap, dentry, acl_name); } /** @@ -2426,7 +2420,7 @@ int security_inode_getxattr(struct dentry *dentry, const char *name) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_getxattr, 0, dentry, name); + return call_int_hook(inode_getxattr, dentry, name); } /** @@ -2442,7 +2436,7 @@ int security_inode_listxattr(struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_listxattr, 0, dentry); + return call_int_hook(inode_listxattr, dentry); } /** @@ -2467,7 +2461,7 @@ int security_inode_removexattr(struct mnt_idmap *idmap, * SELinux and Smack integrate the cap call, * so assume that all LSMs supplying this call do so. */ - ret = call_int_hook(inode_removexattr, 1, idmap, dentry, name); + ret = call_int_hook(inode_removexattr, idmap, dentry, name); if (ret == 1) ret = cap_inode_removexattr(idmap, dentry, name); return ret; @@ -2500,7 +2494,7 @@ void security_inode_post_removexattr(struct dentry *dentry, const char *name) */ int security_inode_need_killpriv(struct dentry *dentry) { - return call_int_hook(inode_need_killpriv, 0, dentry); + return call_int_hook(inode_need_killpriv, dentry); } /** @@ -2517,7 +2511,7 @@ int security_inode_need_killpriv(struct dentry *dentry) int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry) { - return call_int_hook(inode_killpriv, 0, idmap, dentry); + return call_int_hook(inode_killpriv, idmap, dentry); } /** @@ -2540,21 +2534,11 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc) { - struct security_hook_list *hp; - int rc; - if (unlikely(IS_PRIVATE(inode))) return LSM_RET_DEFAULT(inode_getsecurity); - /* - * Only one module will provide an attribute with a given name. - */ - hlist_for_each_entry(hp, &security_hook_heads.inode_getsecurity, list) { - rc = hp->hook.inode_getsecurity(idmap, inode, name, buffer, - alloc); - if (rc != LSM_RET_DEFAULT(inode_getsecurity)) - return rc; - } - return LSM_RET_DEFAULT(inode_getsecurity); + + return call_int_hook(inode_getsecurity, idmap, inode, name, buffer, + alloc); } /** @@ -2575,21 +2559,11 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags) { - struct security_hook_list *hp; - int rc; - if (unlikely(IS_PRIVATE(inode))) return LSM_RET_DEFAULT(inode_setsecurity); - /* - * Only one module will provide an attribute with a given name. - */ - hlist_for_each_entry(hp, &security_hook_heads.inode_setsecurity, list) { - rc = hp->hook.inode_setsecurity(inode, name, value, size, - flags); - if (rc != LSM_RET_DEFAULT(inode_setsecurity)) - return rc; - } - return LSM_RET_DEFAULT(inode_setsecurity); + + return call_int_hook(inode_setsecurity, inode, name, value, size, + flags); } /** @@ -2610,7 +2584,7 @@ int security_inode_listsecurity(struct inode *inode, { if (unlikely(IS_PRIVATE(inode))) return 0; - return call_int_hook(inode_listsecurity, 0, inode, buffer, buffer_size); + return call_int_hook(inode_listsecurity, inode, buffer, buffer_size); } EXPORT_SYMBOL(security_inode_listsecurity); @@ -2641,7 +2615,7 @@ void security_inode_getsecid(struct inode *inode, u32 *secid) */ int security_inode_copy_up(struct dentry *src, struct cred **new) { - return call_int_hook(inode_copy_up, 0, src, new); + return call_int_hook(inode_copy_up, src, new); } EXPORT_SYMBOL(security_inode_copy_up); @@ -2659,7 +2633,6 @@ EXPORT_SYMBOL(security_inode_copy_up); */ int security_inode_copy_up_xattr(const char *name) { - struct security_hook_list *hp; int rc; /* @@ -2667,12 +2640,9 @@ int security_inode_copy_up_xattr(const char *name) * xattr), -EOPNOTSUPP if it does not know anything about the xattr or * any other error code in case of an error. */ - hlist_for_each_entry(hp, - &security_hook_heads.inode_copy_up_xattr, list) { - rc = hp->hook.inode_copy_up_xattr(name); - if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr)) - return rc; - } + rc = call_int_hook(inode_copy_up_xattr, name); + if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr)) + return rc; return LSM_RET_DEFAULT(inode_copy_up_xattr); } @@ -2691,7 +2661,7 @@ EXPORT_SYMBOL(security_inode_copy_up_xattr); int security_kernfs_init_security(struct kernfs_node *kn_dir, struct kernfs_node *kn) { - return call_int_hook(kernfs_init_security, 0, kn_dir, kn); + return call_int_hook(kernfs_init_security, kn_dir, kn); } /** @@ -2715,7 +2685,7 @@ int security_kernfs_init_security(struct kernfs_node *kn_dir, */ int security_file_permission(struct file *file, int mask) { - return call_int_hook(file_permission, 0, file, mask); + return call_int_hook(file_permission, file, mask); } /** @@ -2733,7 +2703,7 @@ int security_file_alloc(struct file *file) if (rc) return rc; - rc = call_int_hook(file_alloc_security, 0, file); + rc = call_int_hook(file_alloc_security, file); if (unlikely(rc)) security_file_free(file); return rc; @@ -2784,7 +2754,7 @@ void security_file_free(struct file *file) */ int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - return call_int_hook(file_ioctl, 0, file, cmd, arg); + return call_int_hook(file_ioctl, file, cmd, arg); } EXPORT_SYMBOL_GPL(security_file_ioctl); @@ -2802,7 +2772,7 @@ EXPORT_SYMBOL_GPL(security_file_ioctl); int security_file_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg) { - return call_int_hook(file_ioctl_compat, 0, file, cmd, arg); + return call_int_hook(file_ioctl_compat, file, cmd, arg); } EXPORT_SYMBOL_GPL(security_file_ioctl_compat); @@ -2853,7 +2823,7 @@ static inline unsigned long mmap_prot(struct file *file, unsigned long prot) int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags) { - return call_int_hook(mmap_file, 0, file, prot, mmap_prot(file, prot), + return call_int_hook(mmap_file, file, prot, mmap_prot(file, prot), flags); } @@ -2867,7 +2837,7 @@ int security_mmap_file(struct file *file, unsigned long prot, */ int security_mmap_addr(unsigned long addr) { - return call_int_hook(mmap_addr, 0, addr); + return call_int_hook(mmap_addr, addr); } /** @@ -2883,7 +2853,7 @@ int security_mmap_addr(unsigned long addr) int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { - return call_int_hook(file_mprotect, 0, vma, reqprot, prot); + return call_int_hook(file_mprotect, vma, reqprot, prot); } /** @@ -2898,7 +2868,7 @@ int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, */ int security_file_lock(struct file *file, unsigned int cmd) { - return call_int_hook(file_lock, 0, file, cmd); + return call_int_hook(file_lock, file, cmd); } /** @@ -2917,7 +2887,7 @@ int security_file_lock(struct file *file, unsigned int cmd) */ int security_file_fcntl(struct file *file, unsigned int cmd, unsigned long arg) { - return call_int_hook(file_fcntl, 0, file, cmd, arg); + return call_int_hook(file_fcntl, file, cmd, arg); } /** @@ -2951,7 +2921,7 @@ void security_file_set_fowner(struct file *file) int security_file_send_sigiotask(struct task_struct *tsk, struct fown_struct *fown, int sig) { - return call_int_hook(file_send_sigiotask, 0, tsk, fown, sig); + return call_int_hook(file_send_sigiotask, tsk, fown, sig); } /** @@ -2965,7 +2935,7 @@ int security_file_send_sigiotask(struct task_struct *tsk, */ int security_file_receive(struct file *file) { - return call_int_hook(file_receive, 0, file); + return call_int_hook(file_receive, file); } /** @@ -2981,7 +2951,7 @@ int security_file_open(struct file *file) { int ret; - ret = call_int_hook(file_open, 0, file); + ret = call_int_hook(file_open, file); if (ret) return ret; @@ -3001,7 +2971,7 @@ int security_file_open(struct file *file) */ int security_file_post_open(struct file *file, int mask) { - return call_int_hook(file_post_open, 0, file, mask); + return call_int_hook(file_post_open, file, mask); } EXPORT_SYMBOL_GPL(security_file_post_open); @@ -3017,7 +2987,7 @@ EXPORT_SYMBOL_GPL(security_file_post_open); */ int security_file_truncate(struct file *file) { - return call_int_hook(file_truncate, 0, file); + return call_int_hook(file_truncate, file); } /** @@ -3035,7 +3005,7 @@ int security_task_alloc(struct task_struct *task, unsigned long clone_flags) if (rc) return rc; - rc = call_int_hook(task_alloc, 0, task, clone_flags); + rc = call_int_hook(task_alloc, task, clone_flags); if (unlikely(rc)) security_task_free(task); return rc; @@ -3073,7 +3043,7 @@ int security_cred_alloc_blank(struct cred *cred, gfp_t gfp) if (rc) return rc; - rc = call_int_hook(cred_alloc_blank, 0, cred, gfp); + rc = call_int_hook(cred_alloc_blank, cred, gfp); if (unlikely(rc)) security_cred_free(cred); return rc; @@ -3117,7 +3087,7 @@ int security_prepare_creds(struct cred *new, const struct cred *old, gfp_t gfp) if (rc) return rc; - rc = call_int_hook(cred_prepare, 0, new, old, gfp); + rc = call_int_hook(cred_prepare, new, old, gfp); if (unlikely(rc)) security_cred_free(new); return rc; @@ -3162,7 +3132,7 @@ EXPORT_SYMBOL(security_cred_getsecid); */ int security_kernel_act_as(struct cred *new, u32 secid) { - return call_int_hook(kernel_act_as, 0, new, secid); + return call_int_hook(kernel_act_as, new, secid); } /** @@ -3178,7 +3148,7 @@ int security_kernel_act_as(struct cred *new, u32 secid) */ int security_kernel_create_files_as(struct cred *new, struct inode *inode) { - return call_int_hook(kernel_create_files_as, 0, new, inode); + return call_int_hook(kernel_create_files_as, new, inode); } /** @@ -3192,7 +3162,7 @@ int security_kernel_create_files_as(struct cred *new, struct inode *inode) */ int security_kernel_module_request(char *kmod_name) { - return call_int_hook(kernel_module_request, 0, kmod_name); + return call_int_hook(kernel_module_request, kmod_name); } /** @@ -3208,7 +3178,7 @@ int security_kernel_module_request(char *kmod_name) int security_kernel_read_file(struct file *file, enum kernel_read_file_id id, bool contents) { - return call_int_hook(kernel_read_file, 0, file, id, contents); + return call_int_hook(kernel_read_file, file, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_read_file); @@ -3228,7 +3198,7 @@ EXPORT_SYMBOL_GPL(security_kernel_read_file); int security_kernel_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id) { - return call_int_hook(kernel_post_read_file, 0, file, buf, size, id); + return call_int_hook(kernel_post_read_file, file, buf, size, id); } EXPORT_SYMBOL_GPL(security_kernel_post_read_file); @@ -3243,7 +3213,7 @@ EXPORT_SYMBOL_GPL(security_kernel_post_read_file); */ int security_kernel_load_data(enum kernel_load_data_id id, bool contents) { - return call_int_hook(kernel_load_data, 0, id, contents); + return call_int_hook(kernel_load_data, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_load_data); @@ -3265,8 +3235,7 @@ int security_kernel_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description) { - return call_int_hook(kernel_post_load_data, 0, buf, size, id, - description); + return call_int_hook(kernel_post_load_data, buf, size, id, description); } EXPORT_SYMBOL_GPL(security_kernel_post_load_data); @@ -3287,7 +3256,7 @@ EXPORT_SYMBOL_GPL(security_kernel_post_load_data); int security_task_fix_setuid(struct cred *new, const struct cred *old, int flags) { - return call_int_hook(task_fix_setuid, 0, new, old, flags); + return call_int_hook(task_fix_setuid, new, old, flags); } /** @@ -3307,7 +3276,7 @@ int security_task_fix_setuid(struct cred *new, const struct cred *old, int security_task_fix_setgid(struct cred *new, const struct cred *old, int flags) { - return call_int_hook(task_fix_setgid, 0, new, old, flags); + return call_int_hook(task_fix_setgid, new, old, flags); } /** @@ -3324,7 +3293,7 @@ int security_task_fix_setgid(struct cred *new, const struct cred *old, */ int security_task_fix_setgroups(struct cred *new, const struct cred *old) { - return call_int_hook(task_fix_setgroups, 0, new, old); + return call_int_hook(task_fix_setgroups, new, old); } /** @@ -3339,7 +3308,7 @@ int security_task_fix_setgroups(struct cred *new, const struct cred *old) */ int security_task_setpgid(struct task_struct *p, pid_t pgid) { - return call_int_hook(task_setpgid, 0, p, pgid); + return call_int_hook(task_setpgid, p, pgid); } /** @@ -3353,7 +3322,7 @@ int security_task_setpgid(struct task_struct *p, pid_t pgid) */ int security_task_getpgid(struct task_struct *p) { - return call_int_hook(task_getpgid, 0, p); + return call_int_hook(task_getpgid, p); } /** @@ -3366,7 +3335,7 @@ int security_task_getpgid(struct task_struct *p) */ int security_task_getsid(struct task_struct *p) { - return call_int_hook(task_getsid, 0, p); + return call_int_hook(task_getsid, p); } /** @@ -3409,7 +3378,7 @@ EXPORT_SYMBOL(security_task_getsecid_obj); */ int security_task_setnice(struct task_struct *p, int nice) { - return call_int_hook(task_setnice, 0, p, nice); + return call_int_hook(task_setnice, p, nice); } /** @@ -3423,7 +3392,7 @@ int security_task_setnice(struct task_struct *p, int nice) */ int security_task_setioprio(struct task_struct *p, int ioprio) { - return call_int_hook(task_setioprio, 0, p, ioprio); + return call_int_hook(task_setioprio, p, ioprio); } /** @@ -3436,7 +3405,7 @@ int security_task_setioprio(struct task_struct *p, int ioprio) */ int security_task_getioprio(struct task_struct *p) { - return call_int_hook(task_getioprio, 0, p); + return call_int_hook(task_getioprio, p); } /** @@ -3453,7 +3422,7 @@ int security_task_getioprio(struct task_struct *p) int security_task_prlimit(const struct cred *cred, const struct cred *tcred, unsigned int flags) { - return call_int_hook(task_prlimit, 0, cred, tcred, flags); + return call_int_hook(task_prlimit, cred, tcred, flags); } /** @@ -3471,7 +3440,7 @@ int security_task_prlimit(const struct cred *cred, const struct cred *tcred, int security_task_setrlimit(struct task_struct *p, unsigned int resource, struct rlimit *new_rlim) { - return call_int_hook(task_setrlimit, 0, p, resource, new_rlim); + return call_int_hook(task_setrlimit, p, resource, new_rlim); } /** @@ -3485,7 +3454,7 @@ int security_task_setrlimit(struct task_struct *p, unsigned int resource, */ int security_task_setscheduler(struct task_struct *p) { - return call_int_hook(task_setscheduler, 0, p); + return call_int_hook(task_setscheduler, p); } /** @@ -3498,7 +3467,7 @@ int security_task_setscheduler(struct task_struct *p) */ int security_task_getscheduler(struct task_struct *p) { - return call_int_hook(task_getscheduler, 0, p); + return call_int_hook(task_getscheduler, p); } /** @@ -3511,7 +3480,7 @@ int security_task_getscheduler(struct task_struct *p) */ int security_task_movememory(struct task_struct *p) { - return call_int_hook(task_movememory, 0, p); + return call_int_hook(task_movememory, p); } /** @@ -3532,7 +3501,7 @@ int security_task_movememory(struct task_struct *p) int security_task_kill(struct task_struct *p, struct kernel_siginfo *info, int sig, const struct cred *cred) { - return call_int_hook(task_kill, 0, p, info, sig, cred); + return call_int_hook(task_kill, p, info, sig, cred); } /** @@ -3590,7 +3559,7 @@ void security_task_to_inode(struct task_struct *p, struct inode *inode) */ int security_create_user_ns(const struct cred *cred) { - return call_int_hook(userns_create, 0, cred); + return call_int_hook(userns_create, cred); } /** @@ -3604,7 +3573,7 @@ int security_create_user_ns(const struct cred *cred) */ int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag) { - return call_int_hook(ipc_permission, 0, ipcp, flag); + return call_int_hook(ipc_permission, ipcp, flag); } /** @@ -3636,7 +3605,7 @@ int security_msg_msg_alloc(struct msg_msg *msg) if (unlikely(rc)) return rc; - rc = call_int_hook(msg_msg_alloc_security, 0, msg); + rc = call_int_hook(msg_msg_alloc_security, msg); if (unlikely(rc)) security_msg_msg_free(msg); return rc; @@ -3670,7 +3639,7 @@ int security_msg_queue_alloc(struct kern_ipc_perm *msq) if (unlikely(rc)) return rc; - rc = call_int_hook(msg_queue_alloc_security, 0, msq); + rc = call_int_hook(msg_queue_alloc_security, msq); if (unlikely(rc)) security_msg_queue_free(msq); return rc; @@ -3702,7 +3671,7 @@ void security_msg_queue_free(struct kern_ipc_perm *msq) */ int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg) { - return call_int_hook(msg_queue_associate, 0, msq, msqflg); + return call_int_hook(msg_queue_associate, msq, msqflg); } /** @@ -3717,7 +3686,7 @@ int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg) */ int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd) { - return call_int_hook(msg_queue_msgctl, 0, msq, cmd); + return call_int_hook(msg_queue_msgctl, msq, cmd); } /** @@ -3734,7 +3703,7 @@ int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd) int security_msg_queue_msgsnd(struct kern_ipc_perm *msq, struct msg_msg *msg, int msqflg) { - return call_int_hook(msg_queue_msgsnd, 0, msq, msg, msqflg); + return call_int_hook(msg_queue_msgsnd, msq, msg, msqflg); } /** @@ -3755,7 +3724,7 @@ int security_msg_queue_msgsnd(struct kern_ipc_perm *msq, int security_msg_queue_msgrcv(struct kern_ipc_perm *msq, struct msg_msg *msg, struct task_struct *target, long type, int mode) { - return call_int_hook(msg_queue_msgrcv, 0, msq, msg, target, type, mode); + return call_int_hook(msg_queue_msgrcv, msq, msg, target, type, mode); } /** @@ -3773,7 +3742,7 @@ int security_shm_alloc(struct kern_ipc_perm *shp) if (unlikely(rc)) return rc; - rc = call_int_hook(shm_alloc_security, 0, shp); + rc = call_int_hook(shm_alloc_security, shp); if (unlikely(rc)) security_shm_free(shp); return rc; @@ -3806,7 +3775,7 @@ void security_shm_free(struct kern_ipc_perm *shp) */ int security_shm_associate(struct kern_ipc_perm *shp, int shmflg) { - return call_int_hook(shm_associate, 0, shp, shmflg); + return call_int_hook(shm_associate, shp, shmflg); } /** @@ -3821,7 +3790,7 @@ int security_shm_associate(struct kern_ipc_perm *shp, int shmflg) */ int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd) { - return call_int_hook(shm_shmctl, 0, shp, cmd); + return call_int_hook(shm_shmctl, shp, cmd); } /** @@ -3839,7 +3808,7 @@ int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd) int security_shm_shmat(struct kern_ipc_perm *shp, char __user *shmaddr, int shmflg) { - return call_int_hook(shm_shmat, 0, shp, shmaddr, shmflg); + return call_int_hook(shm_shmat, shp, shmaddr, shmflg); } /** @@ -3857,7 +3826,7 @@ int security_sem_alloc(struct kern_ipc_perm *sma) if (unlikely(rc)) return rc; - rc = call_int_hook(sem_alloc_security, 0, sma); + rc = call_int_hook(sem_alloc_security, sma); if (unlikely(rc)) security_sem_free(sma); return rc; @@ -3889,7 +3858,7 @@ void security_sem_free(struct kern_ipc_perm *sma) */ int security_sem_associate(struct kern_ipc_perm *sma, int semflg) { - return call_int_hook(sem_associate, 0, sma, semflg); + return call_int_hook(sem_associate, sma, semflg); } /** @@ -3904,7 +3873,7 @@ int security_sem_associate(struct kern_ipc_perm *sma, int semflg) */ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd) { - return call_int_hook(sem_semctl, 0, sma, cmd); + return call_int_hook(sem_semctl, sma, cmd); } /** @@ -3922,7 +3891,7 @@ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd) int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops, unsigned nsops, int alter) { - return call_int_hook(sem_semop, 0, sma, sops, nsops, alter); + return call_int_hook(sem_semop, sma, sops, nsops, alter); } /** @@ -4151,7 +4120,7 @@ int security_setprocattr(int lsmid, const char *name, void *value, size_t size) */ int security_netlink_send(struct sock *sk, struct sk_buff *skb) { - return call_int_hook(netlink_send, 0, sk, skb); + return call_int_hook(netlink_send, sk, skb); } /** @@ -4164,7 +4133,7 @@ int security_netlink_send(struct sock *sk, struct sk_buff *skb) */ int security_ismaclabel(const char *name) { - return call_int_hook(ismaclabel, 0, name); + return call_int_hook(ismaclabel, name); } EXPORT_SYMBOL(security_ismaclabel); @@ -4183,20 +4152,7 @@ EXPORT_SYMBOL(security_ismaclabel); */ int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen) { - struct security_hook_list *hp; - int rc; - - /* - * Currently, only one LSM can implement secid_to_secctx (i.e this - * LSM hook is not "stackable"). - */ - hlist_for_each_entry(hp, &security_hook_heads.secid_to_secctx, list) { - rc = hp->hook.secid_to_secctx(secid, secdata, seclen); - if (rc != LSM_RET_DEFAULT(secid_to_secctx)) - return rc; - } - - return LSM_RET_DEFAULT(secid_to_secctx); + return call_int_hook(secid_to_secctx, secid, secdata, seclen); } EXPORT_SYMBOL(security_secid_to_secctx); @@ -4213,7 +4169,7 @@ EXPORT_SYMBOL(security_secid_to_secctx); int security_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid) { *secid = 0; - return call_int_hook(secctx_to_secid, 0, secdata, seclen, secid); + return call_int_hook(secctx_to_secid, secdata, seclen, secid); } EXPORT_SYMBOL(security_secctx_to_secid); @@ -4260,7 +4216,7 @@ EXPORT_SYMBOL(security_inode_invalidate_secctx); */ int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen) { - return call_int_hook(inode_notifysecctx, 0, inode, ctx, ctxlen); + return call_int_hook(inode_notifysecctx, inode, ctx, ctxlen); } EXPORT_SYMBOL(security_inode_notifysecctx); @@ -4282,7 +4238,7 @@ EXPORT_SYMBOL(security_inode_notifysecctx); */ int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen) { - return call_int_hook(inode_setsecctx, 0, dentry, ctx, ctxlen); + return call_int_hook(inode_setsecctx, dentry, ctx, ctxlen); } EXPORT_SYMBOL(security_inode_setsecctx); @@ -4299,19 +4255,7 @@ EXPORT_SYMBOL(security_inode_setsecctx); */ int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.inode_getsecctx, list) { - rc = hp->hook.inode_getsecctx(inode, ctx, ctxlen); - if (rc != LSM_RET_DEFAULT(inode_getsecctx)) - return rc; - } - - return LSM_RET_DEFAULT(inode_getsecctx); + return call_int_hook(inode_getsecctx, inode, ctx, ctxlen); } EXPORT_SYMBOL(security_inode_getsecctx); @@ -4330,7 +4274,7 @@ int security_post_notification(const struct cred *w_cred, const struct cred *cred, struct watch_notification *n) { - return call_int_hook(post_notification, 0, w_cred, cred, n); + return call_int_hook(post_notification, w_cred, cred, n); } #endif /* CONFIG_WATCH_QUEUE */ @@ -4346,7 +4290,7 @@ int security_post_notification(const struct cred *w_cred, */ int security_watch_key(struct key *key) { - return call_int_hook(watch_key, 0, key); + return call_int_hook(watch_key, key); } #endif /* CONFIG_KEY_NOTIFICATIONS */ @@ -4375,7 +4319,7 @@ int security_watch_key(struct key *key) int security_unix_stream_connect(struct sock *sock, struct sock *other, struct sock *newsk) { - return call_int_hook(unix_stream_connect, 0, sock, other, newsk); + return call_int_hook(unix_stream_connect, sock, other, newsk); } EXPORT_SYMBOL(security_unix_stream_connect); @@ -4401,7 +4345,7 @@ EXPORT_SYMBOL(security_unix_stream_connect); */ int security_unix_may_send(struct socket *sock, struct socket *other) { - return call_int_hook(unix_may_send, 0, sock, other); + return call_int_hook(unix_may_send, sock, other); } EXPORT_SYMBOL(security_unix_may_send); @@ -4418,7 +4362,7 @@ EXPORT_SYMBOL(security_unix_may_send); */ int security_socket_create(int family, int type, int protocol, int kern) { - return call_int_hook(socket_create, 0, family, type, protocol, kern); + return call_int_hook(socket_create, family, type, protocol, kern); } /** @@ -4442,7 +4386,7 @@ int security_socket_create(int family, int type, int protocol, int kern) int security_socket_post_create(struct socket *sock, int family, int type, int protocol, int kern) { - return call_int_hook(socket_post_create, 0, sock, family, type, + return call_int_hook(socket_post_create, sock, family, type, protocol, kern); } @@ -4458,7 +4402,7 @@ int security_socket_post_create(struct socket *sock, int family, */ int security_socket_socketpair(struct socket *socka, struct socket *sockb) { - return call_int_hook(socket_socketpair, 0, socka, sockb); + return call_int_hook(socket_socketpair, socka, sockb); } EXPORT_SYMBOL(security_socket_socketpair); @@ -4477,7 +4421,7 @@ EXPORT_SYMBOL(security_socket_socketpair); int security_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen) { - return call_int_hook(socket_bind, 0, sock, address, addrlen); + return call_int_hook(socket_bind, sock, address, addrlen); } /** @@ -4494,7 +4438,7 @@ int security_socket_bind(struct socket *sock, int security_socket_connect(struct socket *sock, struct sockaddr *address, int addrlen) { - return call_int_hook(socket_connect, 0, sock, address, addrlen); + return call_int_hook(socket_connect, sock, address, addrlen); } /** @@ -4508,7 +4452,7 @@ int security_socket_connect(struct socket *sock, */ int security_socket_listen(struct socket *sock, int backlog) { - return call_int_hook(socket_listen, 0, sock, backlog); + return call_int_hook(socket_listen, sock, backlog); } /** @@ -4524,7 +4468,7 @@ int security_socket_listen(struct socket *sock, int backlog) */ int security_socket_accept(struct socket *sock, struct socket *newsock) { - return call_int_hook(socket_accept, 0, sock, newsock); + return call_int_hook(socket_accept, sock, newsock); } /** @@ -4539,7 +4483,7 @@ int security_socket_accept(struct socket *sock, struct socket *newsock) */ int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) { - return call_int_hook(socket_sendmsg, 0, sock, msg, size); + return call_int_hook(socket_sendmsg, sock, msg, size); } /** @@ -4556,7 +4500,7 @@ int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) int security_socket_recvmsg(struct socket *sock, struct msghdr *msg, int size, int flags) { - return call_int_hook(socket_recvmsg, 0, sock, msg, size, flags); + return call_int_hook(socket_recvmsg, sock, msg, size, flags); } /** @@ -4570,7 +4514,7 @@ int security_socket_recvmsg(struct socket *sock, struct msghdr *msg, */ int security_socket_getsockname(struct socket *sock) { - return call_int_hook(socket_getsockname, 0, sock); + return call_int_hook(socket_getsockname, sock); } /** @@ -4583,7 +4527,7 @@ int security_socket_getsockname(struct socket *sock) */ int security_socket_getpeername(struct socket *sock) { - return call_int_hook(socket_getpeername, 0, sock); + return call_int_hook(socket_getpeername, sock); } /** @@ -4599,7 +4543,7 @@ int security_socket_getpeername(struct socket *sock) */ int security_socket_getsockopt(struct socket *sock, int level, int optname) { - return call_int_hook(socket_getsockopt, 0, sock, level, optname); + return call_int_hook(socket_getsockopt, sock, level, optname); } /** @@ -4614,7 +4558,7 @@ int security_socket_getsockopt(struct socket *sock, int level, int optname) */ int security_socket_setsockopt(struct socket *sock, int level, int optname) { - return call_int_hook(socket_setsockopt, 0, sock, level, optname); + return call_int_hook(socket_setsockopt, sock, level, optname); } /** @@ -4629,7 +4573,7 @@ int security_socket_setsockopt(struct socket *sock, int level, int optname) */ int security_socket_shutdown(struct socket *sock, int how) { - return call_int_hook(socket_shutdown, 0, sock, how); + return call_int_hook(socket_shutdown, sock, how); } /** @@ -4646,7 +4590,7 @@ int security_socket_shutdown(struct socket *sock, int how) */ int security_sock_rcv_skb(struct sock *sk, struct sk_buff *skb) { - return call_int_hook(socket_sock_rcv_skb, 0, sk, skb); + return call_int_hook(socket_sock_rcv_skb, sk, skb); } EXPORT_SYMBOL(security_sock_rcv_skb); @@ -4668,20 +4612,8 @@ EXPORT_SYMBOL(security_sock_rcv_skb); int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval, sockptr_t optlen, unsigned int len) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.socket_getpeersec_stream, - list) { - rc = hp->hook.socket_getpeersec_stream(sock, optval, optlen, - len); - if (rc != LSM_RET_DEFAULT(socket_getpeersec_stream)) - return rc; - } - return LSM_RET_DEFAULT(socket_getpeersec_stream); + return call_int_hook(socket_getpeersec_stream, sock, optval, optlen, + len); } /** @@ -4701,19 +4633,7 @@ int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval, int security_socket_getpeersec_dgram(struct socket *sock, struct sk_buff *skb, u32 *secid) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.socket_getpeersec_dgram, - list) { - rc = hp->hook.socket_getpeersec_dgram(sock, skb, secid); - if (rc != LSM_RET_DEFAULT(socket_getpeersec_dgram)) - return rc; - } - return LSM_RET_DEFAULT(socket_getpeersec_dgram); + return call_int_hook(socket_getpeersec_dgram, sock, skb, secid); } EXPORT_SYMBOL(security_socket_getpeersec_dgram); @@ -4730,7 +4650,7 @@ EXPORT_SYMBOL(security_socket_getpeersec_dgram); */ int security_sk_alloc(struct sock *sk, int family, gfp_t priority) { - return call_int_hook(sk_alloc_security, 0, sk, family, priority); + return call_int_hook(sk_alloc_security, sk, family, priority); } /** @@ -4811,7 +4731,7 @@ EXPORT_SYMBOL(security_sock_graft); int security_inet_conn_request(const struct sock *sk, struct sk_buff *skb, struct request_sock *req) { - return call_int_hook(inet_conn_request, 0, sk, skb, req); + return call_int_hook(inet_conn_request, sk, skb, req); } EXPORT_SYMBOL(security_inet_conn_request); @@ -4852,7 +4772,7 @@ EXPORT_SYMBOL(security_inet_conn_established); */ int security_secmark_relabel_packet(u32 secid) { - return call_int_hook(secmark_relabel_packet, 0, secid); + return call_int_hook(secmark_relabel_packet, secid); } EXPORT_SYMBOL(security_secmark_relabel_packet); @@ -4889,7 +4809,7 @@ EXPORT_SYMBOL(security_secmark_refcount_dec); */ int security_tun_dev_alloc_security(void **security) { - return call_int_hook(tun_dev_alloc_security, 0, security); + return call_int_hook(tun_dev_alloc_security, security); } EXPORT_SYMBOL(security_tun_dev_alloc_security); @@ -4914,7 +4834,7 @@ EXPORT_SYMBOL(security_tun_dev_free_security); */ int security_tun_dev_create(void) { - return call_int_hook(tun_dev_create, 0); + return call_int_hook(tun_dev_create); } EXPORT_SYMBOL(security_tun_dev_create); @@ -4928,7 +4848,7 @@ EXPORT_SYMBOL(security_tun_dev_create); */ int security_tun_dev_attach_queue(void *security) { - return call_int_hook(tun_dev_attach_queue, 0, security); + return call_int_hook(tun_dev_attach_queue, security); } EXPORT_SYMBOL(security_tun_dev_attach_queue); @@ -4944,7 +4864,7 @@ EXPORT_SYMBOL(security_tun_dev_attach_queue); */ int security_tun_dev_attach(struct sock *sk, void *security) { - return call_int_hook(tun_dev_attach, 0, sk, security); + return call_int_hook(tun_dev_attach, sk, security); } EXPORT_SYMBOL(security_tun_dev_attach); @@ -4959,7 +4879,7 @@ EXPORT_SYMBOL(security_tun_dev_attach); */ int security_tun_dev_open(void *security) { - return call_int_hook(tun_dev_open, 0, security); + return call_int_hook(tun_dev_open, security); } EXPORT_SYMBOL(security_tun_dev_open); @@ -4975,7 +4895,7 @@ EXPORT_SYMBOL(security_tun_dev_open); int security_sctp_assoc_request(struct sctp_association *asoc, struct sk_buff *skb) { - return call_int_hook(sctp_assoc_request, 0, asoc, skb); + return call_int_hook(sctp_assoc_request, asoc, skb); } EXPORT_SYMBOL(security_sctp_assoc_request); @@ -4996,8 +4916,7 @@ EXPORT_SYMBOL(security_sctp_assoc_request); int security_sctp_bind_connect(struct sock *sk, int optname, struct sockaddr *address, int addrlen) { - return call_int_hook(sctp_bind_connect, 0, sk, optname, - address, addrlen); + return call_int_hook(sctp_bind_connect, sk, optname, address, addrlen); } EXPORT_SYMBOL(security_sctp_bind_connect); @@ -5031,7 +4950,7 @@ EXPORT_SYMBOL(security_sctp_sk_clone); int security_sctp_assoc_established(struct sctp_association *asoc, struct sk_buff *skb) { - return call_int_hook(sctp_assoc_established, 0, asoc, skb); + return call_int_hook(sctp_assoc_established, asoc, skb); } EXPORT_SYMBOL(security_sctp_assoc_established); @@ -5049,7 +4968,7 @@ EXPORT_SYMBOL(security_sctp_assoc_established); */ int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk) { - return call_int_hook(mptcp_add_subflow, 0, sk, ssk); + return call_int_hook(mptcp_add_subflow, sk, ssk); } #endif /* CONFIG_SECURITY_NETWORK */ @@ -5067,7 +4986,7 @@ int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk) */ int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey) { - return call_int_hook(ib_pkey_access, 0, sec, subnet_prefix, pkey); + return call_int_hook(ib_pkey_access, sec, subnet_prefix, pkey); } EXPORT_SYMBOL(security_ib_pkey_access); @@ -5084,8 +5003,7 @@ EXPORT_SYMBOL(security_ib_pkey_access); int security_ib_endport_manage_subnet(void *sec, const char *dev_name, u8 port_num) { - return call_int_hook(ib_endport_manage_subnet, 0, sec, - dev_name, port_num); + return call_int_hook(ib_endport_manage_subnet, sec, dev_name, port_num); } EXPORT_SYMBOL(security_ib_endport_manage_subnet); @@ -5099,7 +5017,7 @@ EXPORT_SYMBOL(security_ib_endport_manage_subnet); */ int security_ib_alloc_security(void **sec) { - return call_int_hook(ib_alloc_security, 0, sec); + return call_int_hook(ib_alloc_security, sec); } EXPORT_SYMBOL(security_ib_alloc_security); @@ -5132,7 +5050,7 @@ int security_xfrm_policy_alloc(struct xfrm_sec_ctx **ctxp, struct xfrm_user_sec_ctx *sec_ctx, gfp_t gfp) { - return call_int_hook(xfrm_policy_alloc_security, 0, ctxp, sec_ctx, gfp); + return call_int_hook(xfrm_policy_alloc_security, ctxp, sec_ctx, gfp); } EXPORT_SYMBOL(security_xfrm_policy_alloc); @@ -5149,7 +5067,7 @@ EXPORT_SYMBOL(security_xfrm_policy_alloc); int security_xfrm_policy_clone(struct xfrm_sec_ctx *old_ctx, struct xfrm_sec_ctx **new_ctxp) { - return call_int_hook(xfrm_policy_clone_security, 0, old_ctx, new_ctxp); + return call_int_hook(xfrm_policy_clone_security, old_ctx, new_ctxp); } /** @@ -5174,7 +5092,7 @@ EXPORT_SYMBOL(security_xfrm_policy_free); */ int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx) { - return call_int_hook(xfrm_policy_delete_security, 0, ctx); + return call_int_hook(xfrm_policy_delete_security, ctx); } /** @@ -5191,7 +5109,7 @@ int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx) int security_xfrm_state_alloc(struct xfrm_state *x, struct xfrm_user_sec_ctx *sec_ctx) { - return call_int_hook(xfrm_state_alloc, 0, x, sec_ctx); + return call_int_hook(xfrm_state_alloc, x, sec_ctx); } EXPORT_SYMBOL(security_xfrm_state_alloc); @@ -5210,7 +5128,7 @@ EXPORT_SYMBOL(security_xfrm_state_alloc); int security_xfrm_state_alloc_acquire(struct xfrm_state *x, struct xfrm_sec_ctx *polsec, u32 secid) { - return call_int_hook(xfrm_state_alloc_acquire, 0, x, polsec, secid); + return call_int_hook(xfrm_state_alloc_acquire, x, polsec, secid); } /** @@ -5223,7 +5141,7 @@ int security_xfrm_state_alloc_acquire(struct xfrm_state *x, */ int security_xfrm_state_delete(struct xfrm_state *x) { - return call_int_hook(xfrm_state_delete_security, 0, x); + return call_int_hook(xfrm_state_delete_security, x); } EXPORT_SYMBOL(security_xfrm_state_delete); @@ -5252,7 +5170,7 @@ void security_xfrm_state_free(struct xfrm_state *x) */ int security_xfrm_policy_lookup(struct xfrm_sec_ctx *ctx, u32 fl_secid) { - return call_int_hook(xfrm_policy_lookup, 0, ctx, fl_secid); + return call_int_hook(xfrm_policy_lookup, ctx, fl_secid); } /** @@ -5300,12 +5218,12 @@ int security_xfrm_state_pol_flow_match(struct xfrm_state *x, */ int security_xfrm_decode_session(struct sk_buff *skb, u32 *secid) { - return call_int_hook(xfrm_decode_session, 0, skb, secid, 1); + return call_int_hook(xfrm_decode_session, skb, secid, 1); } void security_skb_classify_flow(struct sk_buff *skb, struct flowi_common *flic) { - int rc = call_int_hook(xfrm_decode_session, 0, skb, &flic->flowic_secid, + int rc = call_int_hook(xfrm_decode_session, skb, &flic->flowic_secid, 0); BUG_ON(rc); @@ -5328,7 +5246,7 @@ EXPORT_SYMBOL(security_skb_classify_flow); int security_key_alloc(struct key *key, const struct cred *cred, unsigned long flags) { - return call_int_hook(key_alloc, 0, key, cred, flags); + return call_int_hook(key_alloc, key, cred, flags); } /** @@ -5355,7 +5273,7 @@ void security_key_free(struct key *key) int security_key_permission(key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm) { - return call_int_hook(key_permission, 0, key_ref, cred, need_perm); + return call_int_hook(key_permission, key_ref, cred, need_perm); } /** @@ -5374,7 +5292,7 @@ int security_key_permission(key_ref_t key_ref, const struct cred *cred, int security_key_getsecurity(struct key *key, char **buffer) { *buffer = NULL; - return call_int_hook(key_getsecurity, 0, key, buffer); + return call_int_hook(key_getsecurity, key, buffer); } /** @@ -5412,7 +5330,7 @@ void security_key_post_create_or_update(struct key *keyring, struct key *key, */ int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule) { - return call_int_hook(audit_rule_init, 0, field, op, rulestr, lsmrule); + return call_int_hook(audit_rule_init, field, op, rulestr, lsmrule); } /** @@ -5426,7 +5344,7 @@ int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule) */ int security_audit_rule_known(struct audit_krule *krule) { - return call_int_hook(audit_rule_known, 0, krule); + return call_int_hook(audit_rule_known, krule); } /** @@ -5456,7 +5374,7 @@ void security_audit_rule_free(void *lsmrule) */ int security_audit_rule_match(u32 secid, u32 field, u32 op, void *lsmrule) { - return call_int_hook(audit_rule_match, 0, secid, field, op, lsmrule); + return call_int_hook(audit_rule_match, secid, field, op, lsmrule); } #endif /* CONFIG_AUDIT */ @@ -5475,7 +5393,7 @@ int security_audit_rule_match(u32 secid, u32 field, u32 op, void *lsmrule) */ int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) { - return call_int_hook(bpf, 0, cmd, attr, size); + return call_int_hook(bpf, cmd, attr, size); } /** @@ -5490,7 +5408,7 @@ int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) */ int security_bpf_map(struct bpf_map *map, fmode_t fmode) { - return call_int_hook(bpf_map, 0, map, fmode); + return call_int_hook(bpf_map, map, fmode); } /** @@ -5504,7 +5422,7 @@ int security_bpf_map(struct bpf_map *map, fmode_t fmode) */ int security_bpf_prog(struct bpf_prog *prog) { - return call_int_hook(bpf_prog, 0, prog); + return call_int_hook(bpf_prog, prog); } /** @@ -5517,7 +5435,7 @@ int security_bpf_prog(struct bpf_prog *prog) */ int security_bpf_map_alloc(struct bpf_map *map) { - return call_int_hook(bpf_map_alloc_security, 0, map); + return call_int_hook(bpf_map_alloc_security, map); } /** @@ -5530,7 +5448,7 @@ int security_bpf_map_alloc(struct bpf_map *map) */ int security_bpf_prog_alloc(struct bpf_prog_aux *aux) { - return call_int_hook(bpf_prog_alloc_security, 0, aux); + return call_int_hook(bpf_prog_alloc_security, aux); } /** @@ -5567,7 +5485,7 @@ void security_bpf_prog_free(struct bpf_prog_aux *aux) */ int security_locked_down(enum lockdown_reason what) { - return call_int_hook(locked_down, 0, what); + return call_int_hook(locked_down, what); } EXPORT_SYMBOL(security_locked_down); @@ -5583,7 +5501,7 @@ EXPORT_SYMBOL(security_locked_down); */ int security_perf_event_open(struct perf_event_attr *attr, int type) { - return call_int_hook(perf_event_open, 0, attr, type); + return call_int_hook(perf_event_open, attr, type); } /** @@ -5596,7 +5514,7 @@ int security_perf_event_open(struct perf_event_attr *attr, int type) */ int security_perf_event_alloc(struct perf_event *event) { - return call_int_hook(perf_event_alloc, 0, event); + return call_int_hook(perf_event_alloc, event); } /** @@ -5620,7 +5538,7 @@ void security_perf_event_free(struct perf_event *event) */ int security_perf_event_read(struct perf_event *event) { - return call_int_hook(perf_event_read, 0, event); + return call_int_hook(perf_event_read, event); } /** @@ -5633,7 +5551,7 @@ int security_perf_event_read(struct perf_event *event) */ int security_perf_event_write(struct perf_event *event) { - return call_int_hook(perf_event_write, 0, event); + return call_int_hook(perf_event_write, event); } #endif /* CONFIG_PERF_EVENTS */ @@ -5649,7 +5567,7 @@ int security_perf_event_write(struct perf_event *event) */ int security_uring_override_creds(const struct cred *new) { - return call_int_hook(uring_override_creds, 0, new); + return call_int_hook(uring_override_creds, new); } /** @@ -5662,7 +5580,7 @@ int security_uring_override_creds(const struct cred *new) */ int security_uring_sqpoll(void) { - return call_int_hook(uring_sqpoll, 0); + return call_int_hook(uring_sqpoll); } /** @@ -5675,6 +5593,6 @@ int security_uring_sqpoll(void) */ int security_uring_cmd(struct io_uring_cmd *ioucmd) { - return call_int_hook(uring_cmd, 0, ioucmd); + return call_int_hook(uring_cmd, ioucmd); } #endif /* CONFIG_IO_URING */ From 9b0a62758665e4d76269bba61eb63e5b8d18e499 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 14:52:01 +0100 Subject: [PATCH 0720/1038] thermal: core: Store zone trips table in struct thermal_zone_device The current code expects thermal zone creators to pass a pointer to a writable trips table to thermal_zone_device_register_with_trips() and that trips table is then used by the thermal core going forward. Consequently, the callers of thermal_zone_device_register_with_trips() are required to hold on to the trips table passed to it until the given thermal zone is unregistered, at which point the trips table can be freed, but at the same time they are not expected to access that table directly. This is both error prone and confusing. To address it, turn the trips table pointer in struct thermal_zone_device into a flex array (counted by its num_trips field), allocate it during thermal zone device allocation and copy the contents of the trips table supplied by the zone creator (which can be const now) into it, which will allow the callers of thermal_zone_device_register_with_trips() to drop their trip tables right after the zone registration. This requires the imx thermal driver to be adjusted to store the new temperature in its internal trips table in imx_set_trip_temp(), because it will be separate from the core's trips table now and it has to be explicitly kept in sync with the latter. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Daniel Lezcano --- drivers/thermal/imx_thermal.c | 1 + drivers/thermal/thermal_core.c | 18 +++++++++--------- drivers/thermal/thermal_of.c | 4 ++-- drivers/thermal/thermal_trip.c | 2 +- include/linux/thermal.h | 10 +++++----- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 7019c4fdd549..499eff858f3f 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -354,6 +354,7 @@ static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip_id, return -EINVAL; imx_set_alarm_temp(data, temp); + trips[IMX_TRIP_PASSIVE].temperature = temp; pm_runtime_put(data->dev); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index dfaa6341694a..8bdece8b1536 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1227,9 +1227,6 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp) if (tz->ops->get_crit_temp) return tz->ops->get_crit_temp(tz, temp); - if (!tz->trips) - return -EINVAL; - mutex_lock(&tz->lock); for (i = 0; i < tz->num_trips; i++) { @@ -1272,10 +1269,13 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); * IS_ERR*() helpers. */ struct thermal_zone_device * -thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *trips, int num_trips, int mask, - void *devdata, struct thermal_zone_device_ops *ops, - const struct thermal_zone_params *tzp, int passive_delay, - int polling_delay) +thermal_zone_device_register_with_trips(const char *type, + const struct thermal_trip *trips, + int num_trips, int mask, + void *devdata, + struct thermal_zone_device_ops *ops, + const struct thermal_zone_params *tzp, + int passive_delay, int polling_delay) { struct thermal_zone_device *tz; int id; @@ -1322,7 +1322,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t if (!thermal_class) return ERR_PTR(-ENODEV); - tz = kzalloc(sizeof(*tz), GFP_KERNEL); + tz = kzalloc(struct_size(tz, trips, num_trips), GFP_KERNEL); if (!tz) return ERR_PTR(-ENOMEM); @@ -1354,7 +1354,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t tz->ops = ops; tz->device.class = thermal_class; tz->devdata = devdata; - tz->trips = trips; + memcpy(tz->trips, trips, num_trips * sizeof(*trips)); tz->num_trips = num_trips; thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 4d6c22e0ed85..20fa7d5ac536 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -438,12 +438,10 @@ static int thermal_of_unbind(struct thermal_zone_device *tz, */ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - struct thermal_trip *trips = tz->trips; struct thermal_zone_device_ops *ops = tz->ops; thermal_zone_device_disable(tz); thermal_zone_device_unregister(tz); - kfree(trips); kfree(ops); } @@ -526,6 +524,8 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * goto out_kfree_trips; } + kfree(trips); + ret = thermal_zone_device_enable(tz); if (ret) { pr_err("Failed to enabled thermal zone '%s', id=%d: %d\n", diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index c875a26d5adf..a80ca78f8473 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -122,7 +122,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) int __thermal_zone_get_trip(struct thermal_zone_device *tz, int trip_id, struct thermal_trip *trip) { - if (!tz || !tz->trips || trip_id < 0 || trip_id >= tz->num_trips || !trip) + if (!tz || trip_id < 0 || trip_id >= tz->num_trips || !trip) return -EINVAL; *trip = tz->trips[trip_id]; diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 65d8f92a9a0d..572a24f29a10 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -130,7 +130,6 @@ struct thermal_cooling_device { * @trip_hyst_attrs: attributes for trip points for sysfs: trip hysteresis * @mode: current mode of this thermal zone * @devdata: private pointer for device private data - * @trips: an array of struct thermal_trip * @num_trips: number of trip points the thermal zone supports * @passive_delay_jiffies: number of jiffies to wait between polls when * performing passive cooling. @@ -160,6 +159,7 @@ struct thermal_cooling_device { * @poll_queue: delayed work for polling * @notify_event: Last notification event * @suspended: thermal zone suspend indicator + * @trips: array of struct thermal_trip objects */ struct thermal_zone_device { int id; @@ -172,7 +172,6 @@ struct thermal_zone_device { struct thermal_attr *trip_hyst_attrs; enum thermal_device_mode mode; void *devdata; - struct thermal_trip *trips; int num_trips; unsigned long passive_delay_jiffies; unsigned long polling_delay_jiffies; @@ -193,10 +192,11 @@ struct thermal_zone_device { struct list_head node; struct delayed_work poll_queue; enum thermal_notify_event notify_event; + bool suspended; #ifdef CONFIG_THERMAL_DEBUGFS struct thermal_debugfs *debugfs; #endif - bool suspended; + struct thermal_trip trips[] __counted_by(num_trips); }; /** @@ -315,7 +315,7 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp); #ifdef CONFIG_THERMAL struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, - struct thermal_trip *trips, + const struct thermal_trip *trips, int num_trips, int mask, void *devdata, struct thermal_zone_device_ops *ops, @@ -375,7 +375,7 @@ void thermal_zone_device_critical(struct thermal_zone_device *tz); #else static inline struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, - struct thermal_trip *trips, + const struct thermal_trip *trips, int num_trips, int mask, void *devdata, struct thermal_zone_device_ops *ops, From 9686f04a7ba6366dbdbb5609dc26a6282803b888 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:15:39 +0100 Subject: [PATCH 0721/1038] thermal: ACPI: Discard trips table after zone registration Because the thermal core creates and uses its own copy of the trips table passed to thermal_zone_device_register_with_trips(), it is not necessary to hold on to a local copy of it any more after the given thermal zone has been registered. Accordingly, modify the ACPI thermal driver to store the trips table passed to thermal_zone_device_register_with_trips() in a local variable which is automatically discarded when acpi_thermal_add() returns to its caller. Also make some additional code simplifications unlocked by the above change. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Acked-by: Daniel Lezcano --- drivers/acpi/thermal.c | 57 +++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 4748e8061253..6c4ba00d60e2 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -47,6 +47,8 @@ #define ACPI_THERMAL_TRIP_PASSIVE (-1) +#define ACPI_THERMAL_MAX_NR_TRIPS (ACPI_THERMAL_MAX_ACTIVE + 3) + /* * This exception is thrown out in two cases: * 1.An invalid trip point becomes invalid or a valid trip point becomes invalid @@ -112,7 +114,6 @@ struct acpi_thermal { unsigned long polling_frequency; volatile u8 zombie; struct acpi_thermal_trips trips; - struct thermal_trip *trip_table; struct thermal_zone_device *thermal_zone; int kelvin_offset; /* in millidegrees */ struct work_struct thermal_check_work; @@ -451,26 +452,19 @@ fail: return false; } -static int acpi_thermal_get_trip_points(struct acpi_thermal *tz) +static void acpi_thermal_get_trip_points(struct acpi_thermal *tz) { - unsigned int count = 0; int i; - if (acpi_thermal_init_trip(tz, ACPI_THERMAL_TRIP_PASSIVE)) - count++; + acpi_thermal_init_trip(tz, ACPI_THERMAL_TRIP_PASSIVE); for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE; i++) { - if (acpi_thermal_init_trip(tz, i)) - count++; - else + if (!acpi_thermal_init_trip(tz, i)) break; - } while (++i < ACPI_THERMAL_MAX_ACTIVE) tz->trips.active[i].trip.temp_dk = THERMAL_TEMP_INVALID; - - return count; } /* sys I/F for generic thermal sysfs support */ @@ -662,13 +656,14 @@ static void acpi_thermal_zone_sysfs_remove(struct acpi_thermal *tz) } static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz, + const struct thermal_trip *trip_table, unsigned int trip_count, int passive_delay) { int result; tz->thermal_zone = thermal_zone_device_register_with_trips("acpitz", - tz->trip_table, + trip_table, trip_count, 0, tz, &acpi_thermal_zone_ops, @@ -823,10 +818,10 @@ static void acpi_thermal_free_thermal_zone(struct acpi_thermal *tz) static int acpi_thermal_add(struct acpi_device *device) { + struct thermal_trip trip_table[ACPI_THERMAL_MAX_NR_TRIPS] = { 0 }; struct acpi_thermal_trip *acpi_trip; struct thermal_trip *trip; struct acpi_thermal *tz; - unsigned int trip_count; int crit_temp, hot_temp; int passive_delay = 0; int result; @@ -848,21 +843,10 @@ static int acpi_thermal_add(struct acpi_device *device) acpi_thermal_aml_dependency_fix(tz); /* Get trip points [_CRT, _PSV, etc.] (required). */ - trip_count = acpi_thermal_get_trip_points(tz); + acpi_thermal_get_trip_points(tz); crit_temp = acpi_thermal_get_critical_trip(tz); - if (crit_temp != THERMAL_TEMP_INVALID) - trip_count++; - hot_temp = acpi_thermal_get_hot_trip(tz); - if (hot_temp != THERMAL_TEMP_INVALID) - trip_count++; - - if (!trip_count) { - pr_warn(FW_BUG "No valid trip points!\n"); - result = -ENODEV; - goto free_memory; - } /* Get temperature [_TMP] (required). */ result = acpi_thermal_get_temperature(tz); @@ -881,13 +865,7 @@ static int acpi_thermal_add(struct acpi_device *device) acpi_thermal_guess_offset(tz, crit_temp); - trip = kcalloc(trip_count, sizeof(*trip), GFP_KERNEL); - if (!trip) { - result = -ENOMEM; - goto free_memory; - } - - tz->trip_table = trip; + trip = trip_table; if (crit_temp != THERMAL_TEMP_INVALID) { trip->type = THERMAL_TRIP_CRITICAL; @@ -923,9 +901,17 @@ static int acpi_thermal_add(struct acpi_device *device) trip++; } - result = acpi_thermal_register_thermal_zone(tz, trip_count, passive_delay); + if (trip == trip_table) { + pr_warn(FW_BUG "No valid trip points!\n"); + result = -ENODEV; + goto free_memory; + } + + result = acpi_thermal_register_thermal_zone(tz, trip_table, + trip - trip_table, + passive_delay); if (result) - goto free_trips; + goto free_memory; refcount_set(&tz->thermal_check_count, 3); mutex_init(&tz->thermal_check_lock); @@ -944,8 +930,6 @@ static int acpi_thermal_add(struct acpi_device *device) flush_wq: flush_workqueue(acpi_thermal_pm_queue); acpi_thermal_unregister_thermal_zone(tz); -free_trips: - kfree(tz->trip_table); free_memory: acpi_thermal_free_thermal_zone(tz); @@ -966,7 +950,6 @@ static void acpi_thermal_remove(struct acpi_device *device) flush_workqueue(acpi_thermal_pm_queue); acpi_thermal_unregister_thermal_zone(tz); - kfree(tz->trip_table); acpi_thermal_free_thermal_zone(tz); } From fcbf8780008609380d2e5b407c5bb7950fff018c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 5 Feb 2024 22:16:59 +0100 Subject: [PATCH 0722/1038] thermal: intel: Discard trip tables after zone registration Because the thermal core creates and uses its own copy of the trips table passed to thermal_zone_device_register_with_trips(), it is not necessary to hold on to a local copy of it any more after the given thermal zone has been registered. Accordingly, modify Intel thermal drivers to discard the trips tables passed to thermal_zone_device_register_with_trips() after thermal zone registration, for example by storing them in local variables which are automatically discarded when the zone registration is complete. Also make some additional code simplifications unlocked by the above changes. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Acked-by: Daniel Lezcano --- .../int340x_thermal/int340x_thermal_zone.c | 6 +-- .../int340x_thermal/int340x_thermal_zone.h | 1 - .../processor_thermal_device_pci.c | 7 ++-- drivers/thermal/intel/intel_pch_thermal.c | 24 ++++++------ .../thermal/intel/intel_quark_dts_thermal.c | 12 +++--- drivers/thermal/intel/intel_soc_dts_iosf.c | 38 ++++++++----------- drivers/thermal/intel/intel_soc_dts_iosf.h | 1 - drivers/thermal/intel/x86_pkg_temp_thermal.c | 35 ++++++----------- 8 files changed, 51 insertions(+), 73 deletions(-) diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index 3e4bfe817fac..950e91ca5f42 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -179,8 +179,6 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, for (i = 0; i < trip_cnt; ++i) zone_trips[i].hysteresis = hyst; - int34x_zone->trips = zone_trips; - int34x_zone->lpat_table = acpi_lpat_get_conversion_table(adev->handle); int34x_zone->zone = thermal_zone_device_register_with_trips( @@ -190,6 +188,8 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->ops, &int340x_thermal_params, 0, 0); + kfree(zone_trips); + if (IS_ERR(int34x_zone->zone)) { ret = PTR_ERR(int34x_zone->zone); goto err_thermal_zone; @@ -203,7 +203,6 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, err_enable: thermal_zone_device_unregister(int34x_zone->zone); err_thermal_zone: - kfree(int34x_zone->trips); acpi_lpat_free_conversion_table(int34x_zone->lpat_table); err_trips_alloc: kfree(int34x_zone->ops); @@ -217,7 +216,6 @@ void int340x_thermal_zone_remove(struct int34x_thermal_zone *int34x_zone) { thermal_zone_device_unregister(int34x_zone->zone); acpi_lpat_free_conversion_table(int34x_zone->lpat_table); - kfree(int34x_zone->trips); kfree(int34x_zone->ops); kfree(int34x_zone); } diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h index e0df6271facc..91edc7f83d79 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h @@ -20,7 +20,6 @@ struct active_trip { struct int34x_thermal_zone { struct acpi_device *adev; - struct thermal_trip *trips; int aux_trip_nr; struct thermal_zone_device *zone; struct thermal_zone_device_ops *ops; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index d7495571dd5d..bf47f319e3c6 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -233,10 +233,6 @@ static int get_trip_temp(struct proc_thermal_pci *pci_info) return temp; } -static struct thermal_trip psv_trip = { - .type = THERMAL_TRIP_PASSIVE, -}; - static struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, @@ -251,6 +247,9 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ { struct proc_thermal_device *proc_priv; struct proc_thermal_pci *pci_info; + struct thermal_trip psv_trip = { + .type = THERMAL_TRIP_PASSIVE, + }; int irq_flag = 0, irq, ret; bool msi_irq = false; diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c index b3905e34c507..76a552870fe0 100644 --- a/drivers/thermal/intel/intel_pch_thermal.c +++ b/drivers/thermal/intel/intel_pch_thermal.c @@ -84,7 +84,6 @@ struct pch_thermal_device { void __iomem *hw_base; struct pci_dev *pdev; struct thermal_zone_device *tzd; - struct thermal_trip trips[PCH_MAX_TRIPS]; bool bios_enabled; }; @@ -94,7 +93,8 @@ struct pch_thermal_device { * passive trip temperature using _PSV method. There is no specific * passive temperature setting in MMIO interface of this PCI device. */ -static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip) +static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, + struct thermal_trip *trip) { struct acpi_device *adev; int temp; @@ -106,12 +106,13 @@ static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip) if (thermal_acpi_passive_trip_temp(adev, &temp) || temp <= 0) return 0; - ptd->trips[trip].type = THERMAL_TRIP_PASSIVE; - ptd->trips[trip].temperature = temp; + trip->type = THERMAL_TRIP_PASSIVE; + trip->temperature = temp; return 1; } #else -static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip) +static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, + struct thermal_trip *trip) { return 0; } @@ -159,6 +160,7 @@ static const char *board_names[] = { static int intel_pch_thermal_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct thermal_trip ptd_trips[PCH_MAX_TRIPS] = { 0 }; enum pch_board_ids board_id = id->driver_data; struct pch_thermal_device *ptd; int nr_trips = 0; @@ -220,21 +222,21 @@ read_trips: trip_temp = readw(ptd->hw_base + WPT_CTT); trip_temp &= 0x1FF; if (trip_temp) { - ptd->trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); - ptd->trips[nr_trips++].type = THERMAL_TRIP_CRITICAL; + ptd_trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); + ptd_trips[nr_trips++].type = THERMAL_TRIP_CRITICAL; } trip_temp = readw(ptd->hw_base + WPT_PHL); trip_temp &= 0x1FF; if (trip_temp) { - ptd->trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); - ptd->trips[nr_trips++].type = THERMAL_TRIP_HOT; + ptd_trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); + ptd_trips[nr_trips++].type = THERMAL_TRIP_HOT; } - nr_trips += pch_wpt_add_acpi_psv_trip(ptd, nr_trips); + nr_trips += pch_wpt_add_acpi_psv_trip(ptd, &ptd_trips[nr_trips]); ptd->tzd = thermal_zone_device_register_with_trips(board_names[board_id], - ptd->trips, nr_trips, + ptd_trips, nr_trips, 0, ptd, &tzd_ops, NULL, 0, 0); if (IS_ERR(ptd->tzd)) { diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c index 646ca8bd40a9..06f6d1468717 100644 --- a/drivers/thermal/intel/intel_quark_dts_thermal.c +++ b/drivers/thermal/intel/intel_quark_dts_thermal.c @@ -105,7 +105,6 @@ struct soc_sensor_entry { u32 store_ptps; u32 store_dts_enable; struct thermal_zone_device *tzone; - struct thermal_trip trips[QRK_MAX_DTS_TRIPS]; }; static struct soc_sensor_entry *soc_dts; @@ -320,6 +319,7 @@ static void free_soc_dts(struct soc_sensor_entry *aux_entry) static struct soc_sensor_entry *alloc_soc_dts(void) { + struct thermal_trip trips[QRK_MAX_DTS_TRIPS] = { 0 }; struct soc_sensor_entry *aux_entry; int err; u32 out; @@ -362,14 +362,14 @@ static struct soc_sensor_entry *alloc_soc_dts(void) goto err_ret; } - aux_entry->trips[QRK_DTS_ID_TP_CRITICAL].temperature = get_trip_temp(QRK_DTS_ID_TP_CRITICAL); - aux_entry->trips[QRK_DTS_ID_TP_CRITICAL].type = THERMAL_TRIP_CRITICAL; + trips[QRK_DTS_ID_TP_CRITICAL].temperature = get_trip_temp(QRK_DTS_ID_TP_CRITICAL); + trips[QRK_DTS_ID_TP_CRITICAL].type = THERMAL_TRIP_CRITICAL; - aux_entry->trips[QRK_DTS_ID_TP_HOT].temperature = get_trip_temp(QRK_DTS_ID_TP_HOT); - aux_entry->trips[QRK_DTS_ID_TP_HOT].type = THERMAL_TRIP_HOT; + trips[QRK_DTS_ID_TP_HOT].temperature = get_trip_temp(QRK_DTS_ID_TP_HOT); + trips[QRK_DTS_ID_TP_HOT].type = THERMAL_TRIP_HOT; aux_entry->tzone = thermal_zone_device_register_with_trips("quark_dts", - aux_entry->trips, + trips, QRK_MAX_DTS_TRIPS, wr_mask, aux_entry, &tzone_ops, diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c index d00def3c4703..096cdd3c3b1f 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.c +++ b/drivers/thermal/intel/intel_soc_dts_iosf.c @@ -129,22 +129,6 @@ err_restore_ptps: return status; } -static int configure_trip(struct intel_soc_dts_sensor_entry *dts, - int thres_index, enum thermal_trip_type trip_type, - int temp) -{ - int ret; - - ret = update_trip_temp(dts->sensors, thres_index, temp); - if (ret) - return ret; - - dts->trips[thres_index].temperature = temp; - dts->trips[thres_index].type = trip_type; - - return 0; -} - static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp) { @@ -218,6 +202,7 @@ static void remove_dts_thermal_zone(struct intel_soc_dts_sensor_entry *dts) } static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, + struct thermal_trip *trips, bool critical_trip) { int writable_trip_cnt = SOC_MAX_DTS_TRIPS; @@ -254,7 +239,7 @@ static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, } dts->trip_mask = trip_mask; snprintf(name, sizeof(name), "soc_dts%d", id); - dts->tzone = thermal_zone_device_register_with_trips(name, dts->trips, + dts->tzone = thermal_zone_device_register_with_trips(name, trips, SOC_MAX_DTS_TRIPS, trip_mask, dts, &tzone_ops, @@ -315,14 +300,15 @@ EXPORT_SYMBOL_GPL(intel_soc_dts_iosf_interrupt_handler); static void dts_trips_reset(struct intel_soc_dts_sensors *sensors, int dts_index) { - configure_trip(&sensors->soc_dts[dts_index], 0, 0, 0); - configure_trip(&sensors->soc_dts[dts_index], 1, 0, 0); + update_trip_temp(sensors, 0, 0); + update_trip_temp(sensors, 1, 0); } struct intel_soc_dts_sensors * intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, bool critical_trip, int crit_offset) { + struct thermal_trip trips[SOC_MAX_DTS_SENSORS][SOC_MAX_DTS_TRIPS] = { 0 }; struct intel_soc_dts_sensors *sensors; int tj_max; int ret; @@ -350,11 +336,13 @@ intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, sensors->soc_dts[i].sensors = sensors; - ret = configure_trip(&sensors->soc_dts[i], 0, - THERMAL_TRIP_PASSIVE, 0); + ret = update_trip_temp(sensors, 0, 0); if (ret) goto err_reset_trips; + trips[i][0].type = THERMAL_TRIP_PASSIVE; + trips[i][0].temperature = 0; + if (critical_trip) { trip_type = THERMAL_TRIP_CRITICAL; temp = sensors->tj_max - crit_offset; @@ -362,13 +350,17 @@ intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, trip_type = THERMAL_TRIP_PASSIVE; temp = 0; } - ret = configure_trip(&sensors->soc_dts[i], 1, trip_type, temp); + ret = update_trip_temp(sensors, 1, temp); if (ret) goto err_reset_trips; + + trips[i][1].type = trip_type; + trips[i][1].temperature = temp; } for (i = 0; i < SOC_MAX_DTS_SENSORS; ++i) { - ret = add_dts_thermal_zone(i, &sensors->soc_dts[i], critical_trip); + ret = add_dts_thermal_zone(i, &sensors->soc_dts[i], trips[i], + critical_trip); if (ret) goto err_remove_zone; } diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.h b/drivers/thermal/intel/intel_soc_dts_iosf.h index 162841df0ebe..da59d719eb9d 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.h +++ b/drivers/thermal/intel/intel_soc_dts_iosf.h @@ -29,7 +29,6 @@ struct intel_soc_dts_sensor_entry { int id; u32 store_status; u32 trip_mask; - struct thermal_trip trips[SOC_MAX_DTS_TRIPS]; struct thermal_zone_device *tzone; struct intel_soc_dts_sensors *sensors; }; diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 61c3d450ee60..dafd28a39bcc 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -53,7 +53,6 @@ struct zone_device { u32 msr_pkg_therm_high; struct delayed_work work; struct thermal_zone_device *tzone; - struct thermal_trip *trips; struct cpumask cpumask; }; @@ -268,17 +267,13 @@ static int pkg_thermal_notify(u64 msr_val) return 0; } -static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int num_trips) +static int pkg_temp_thermal_trips_init(int cpu, int tj_max, + struct thermal_trip *trips, int num_trips) { - struct thermal_trip *trips; unsigned long thres_reg_value; u32 mask, shift, eax, edx; int ret, i; - trips = kzalloc(sizeof(*trips) * num_trips, GFP_KERNEL); - if (!trips) - return ERR_PTR(-ENOMEM); - for (i = 0; i < num_trips; i++) { if (i) { @@ -291,10 +286,8 @@ static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &eax, &edx); - if (ret < 0) { - kfree(trips); - return ERR_PTR(ret); - } + if (ret < 0) + return ret; thres_reg_value = (eax & mask) >> shift; @@ -307,11 +300,12 @@ static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int __func__, cpu, i, trips[i].temperature); } - return trips; + return 0; } static int pkg_temp_thermal_device_add(unsigned int cpu) { + struct thermal_trip trips[MAX_NUMBER_OF_TRIPS] = { 0 }; int id = topology_logical_die_id(cpu); u32 eax, ebx, ecx, edx; struct zone_device *zonedev; @@ -336,21 +330,19 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) if (!zonedev) return -ENOMEM; - zonedev->trips = pkg_temp_thermal_trips_init(cpu, tj_max, thres_count); - if (IS_ERR(zonedev->trips)) { - err = PTR_ERR(zonedev->trips); + err = pkg_temp_thermal_trips_init(cpu, tj_max, trips, thres_count); + if (err) goto out_kfree_zonedev; - } INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn); zonedev->cpu = cpu; zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp", - zonedev->trips, thres_count, + trips, thres_count, (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01, zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0); if (IS_ERR(zonedev->tzone)) { err = PTR_ERR(zonedev->tzone); - goto out_kfree_trips; + goto out_kfree_zonedev; } err = thermal_zone_device_enable(zonedev->tzone); if (err) @@ -369,8 +361,6 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) out_unregister_tz: thermal_zone_device_unregister(zonedev->tzone); -out_kfree_trips: - kfree(zonedev->trips); out_kfree_zonedev: kfree(zonedev); return err; @@ -457,10 +447,9 @@ static int pkg_thermal_cpu_offline(unsigned int cpu) raw_spin_unlock_irq(&pkg_temp_lock); /* Final cleanup if this is the last cpu */ - if (lastcpu) { - kfree(zonedev->trips); + if (lastcpu) kfree(zonedev); - } + return 0; } From 698a1eb1f75eb6ac957d2af7721a3b1a94281e5d Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:18:01 +0100 Subject: [PATCH 0723/1038] thermal: core: Store zone ops in struct thermal_zone_device The current code requires thermal zone creators to pass pointers to writable ops structures to thermal_zone_device_register_with_trips() which needs to modify the target struct thermal_zone_device_ops object if the "critical" operation in it is NULL. Moreover, the callers of thermal_zone_device_register_with_trips() are required to hold on to the struct thermal_zone_device_ops object passed to it until the given thermal zone is unregistered. Both of these requirements are quite inconvenient, so modify struct thermal_zone_device to contain struct thermal_zone_device_ops as field and make thermal_zone_device_register_with_trips() copy the contents of the struct thermal_zone_device_ops passed to it via a pointer (which can be const now) to that field. Also adjust the code using thermal zone ops accordingly and modify thermal_of_zone_register() to use a local ops variable during thermal zone registration so ops do not need to be freed in thermal_of_zone_unregister() any more. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_core.c | 40 +++++++++++++++---------------- drivers/thermal/thermal_helpers.c | 10 ++++---- drivers/thermal/thermal_hwmon.c | 4 ++-- drivers/thermal/thermal_of.c | 27 +++++++-------------- drivers/thermal/thermal_sysfs.c | 14 +++++------ drivers/thermal/thermal_trip.c | 4 ++-- include/linux/thermal.h | 8 +++---- 7 files changed, 48 insertions(+), 59 deletions(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 8bdece8b1536..bb21f78b4bfa 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -356,9 +356,9 @@ static void handle_critical_trips(struct thermal_zone_device *tz, trace_thermal_zone_trip(tz, thermal_zone_trip_id(tz, trip), trip->type); if (trip->type == THERMAL_TRIP_CRITICAL) - tz->ops->critical(tz); - else if (tz->ops->hot) - tz->ops->hot(tz); + tz->ops.critical(tz); + else if (tz->ops.hot) + tz->ops.hot(tz); } static void handle_thermal_trip(struct thermal_zone_device *tz, @@ -493,8 +493,8 @@ static int thermal_zone_device_set_mode(struct thermal_zone_device *tz, return ret; } - if (tz->ops->change_mode) - ret = tz->ops->change_mode(tz, mode); + if (tz->ops.change_mode) + ret = tz->ops.change_mode(tz, mode); if (!ret) tz->mode = mode; @@ -867,8 +867,8 @@ static void bind_cdev(struct thermal_cooling_device *cdev) struct thermal_zone_device *pos = NULL; list_for_each_entry(pos, &thermal_tz_list, node) { - if (pos->ops->bind) { - ret = pos->ops->bind(pos, cdev); + if (pos->ops.bind) { + ret = pos->ops.bind(pos, cdev); if (ret) print_bind_err_msg(pos, cdev, ret); } @@ -1184,8 +1184,8 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev) /* Unbind all thermal zones associated with 'this' cdev */ list_for_each_entry(tz, &thermal_tz_list, node) { - if (tz->ops->unbind) - tz->ops->unbind(tz, cdev); + if (tz->ops.unbind) + tz->ops.unbind(tz, cdev); } mutex_unlock(&thermal_list_lock); @@ -1199,13 +1199,13 @@ static void bind_tz(struct thermal_zone_device *tz) int ret; struct thermal_cooling_device *pos = NULL; - if (!tz->ops->bind) + if (!tz->ops.bind) return; mutex_lock(&thermal_list_lock); list_for_each_entry(pos, &thermal_cdev_list, node) { - ret = tz->ops->bind(tz, pos); + ret = tz->ops.bind(tz, pos); if (ret) print_bind_err_msg(tz, pos, ret); } @@ -1224,8 +1224,8 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp) { int i, ret = -EINVAL; - if (tz->ops->get_crit_temp) - return tz->ops->get_crit_temp(tz, temp); + if (tz->ops.get_crit_temp) + return tz->ops.get_crit_temp(tz, temp); mutex_lock(&tz->lock); @@ -1273,7 +1273,7 @@ thermal_zone_device_register_with_trips(const char *type, const struct thermal_trip *trips, int num_trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { @@ -1348,10 +1348,10 @@ thermal_zone_device_register_with_trips(const char *type, tz->id = id; strscpy(tz->type, type, sizeof(tz->type)); - if (!ops->critical) - ops->critical = thermal_zone_device_critical; + tz->ops = *ops; + if (!tz->ops.critical) + tz->ops.critical = thermal_zone_device_critical; - tz->ops = ops; tz->device.class = thermal_class; tz->devdata = devdata; memcpy(tz->trips, trips, num_trips * sizeof(*trips)); @@ -1437,7 +1437,7 @@ EXPORT_SYMBOL_GPL(thermal_zone_device_register_with_trips); struct thermal_zone_device *thermal_tripless_zone_device_register( const char *type, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp) { return thermal_zone_device_register_with_trips(type, NULL, 0, 0, devdata, @@ -1499,8 +1499,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) /* Unbind all cdevs associated with 'this' thermal zone */ list_for_each_entry(cdev, &thermal_cdev_list, node) - if (tz->ops->unbind) - tz->ops->unbind(tz, cdev); + if (tz->ops.unbind) + tz->ops.unbind(tz, cdev); mutex_unlock(&thermal_list_lock); diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index 0329f4a71b02..c5a057b59c42 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -26,8 +26,8 @@ int get_tz_trend(struct thermal_zone_device *tz, const struct thermal_trip *trip { enum thermal_trend trend; - if (tz->emul_temperature || !tz->ops->get_trend || - tz->ops->get_trend(tz, trip, &trend)) { + if (tz->emul_temperature || !tz->ops.get_trend || + tz->ops.get_trend(tz, trip, &trend)) { if (tz->temperature > tz->last_temperature) trend = THERMAL_TREND_RAISING; else if (tz->temperature < tz->last_temperature) @@ -75,7 +75,7 @@ EXPORT_SYMBOL(get_thermal_instance); * temperature and fill @temp. * * Both tz and tz->ops must be valid pointers when calling this function, - * and the tz->ops->get_temp callback must be provided. + * and the tz->ops.get_temp callback must be provided. * The function must be called under tz->lock. * * Return: On success returns 0, an error code otherwise @@ -88,7 +88,7 @@ int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) lockdep_assert_held(&tz->lock); - ret = tz->ops->get_temp(tz, temp); + ret = tz->ops.get_temp(tz, temp); if (IS_ENABLED(CONFIG_THERMAL_EMULATION) && tz->emul_temperature) { for_each_trip(tz, trip) { @@ -132,7 +132,7 @@ int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) mutex_lock(&tz->lock); - if (!tz->ops->get_temp) { + if (!tz->ops.get_temp) { ret = -EINVAL; goto unlock; } diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index 252116f1e535..f0e504fd866a 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -80,7 +80,7 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf) mutex_lock(&tz->lock); - ret = tz->ops->get_crit_temp(tz, &temperature); + ret = tz->ops.get_crit_temp(tz, &temperature); mutex_unlock(&tz->lock); @@ -132,7 +132,7 @@ thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon, static bool thermal_zone_crit_temp_valid(struct thermal_zone_device *tz) { int temp; - return tz->ops->get_crit_temp && !tz->ops->get_crit_temp(tz, &temp); + return tz->ops.get_crit_temp && !tz->ops.get_crit_temp(tz, &temp); } int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 20fa7d5ac536..405743e90531 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -438,11 +438,8 @@ static int thermal_of_unbind(struct thermal_zone_device *tz, */ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - struct thermal_zone_device_ops *ops = tz->ops; - thermal_zone_device_disable(tz); thermal_zone_device_unregister(tz); - kfree(ops); } /** @@ -468,33 +465,27 @@ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) static struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data, const struct thermal_zone_device_ops *ops) { + struct thermal_zone_device_ops of_ops = *ops; struct thermal_zone_device *tz; struct thermal_trip *trips; struct thermal_zone_params tzp = {}; - struct thermal_zone_device_ops *of_ops; struct device_node *np; const char *action; int delay, pdelay; int ntrips, mask; int ret; - of_ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL); - if (!of_ops) - return ERR_PTR(-ENOMEM); - np = of_thermal_zone_find(sensor, id); if (IS_ERR(np)) { if (PTR_ERR(np) != -ENODEV) pr_err("Failed to find thermal zone for %pOFn id=%d\n", sensor, id); - ret = PTR_ERR(np); - goto out_kfree_of_ops; + return ERR_CAST(np); } trips = thermal_of_trips_init(np, &ntrips); if (IS_ERR(trips)) { pr_err("Failed to find trip points for %pOFn id=%d\n", sensor, id); - ret = PTR_ERR(trips); - goto out_kfree_of_ops; + return ERR_CAST(trips); } ret = thermal_of_monitor_init(np, &delay, &pdelay); @@ -505,18 +496,18 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * thermal_of_parameters_init(np, &tzp); - of_ops->bind = thermal_of_bind; - of_ops->unbind = thermal_of_unbind; + of_ops.bind = thermal_of_bind; + of_ops.unbind = thermal_of_unbind; mask = GENMASK_ULL((ntrips) - 1, 0); ret = of_property_read_string(np, "critical-action", &action); if (!ret) - if (!of_ops->critical && !strcasecmp(action, "reboot")) - of_ops->critical = thermal_zone_device_critical_reboot; + if (!of_ops.critical && !strcasecmp(action, "reboot")) + of_ops.critical = thermal_zone_device_critical_reboot; tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, - mask, data, of_ops, &tzp, + mask, data, &of_ops, &tzp, pdelay, delay); if (IS_ERR(tz)) { ret = PTR_ERR(tz); @@ -538,8 +529,6 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * out_kfree_trips: kfree(trips); -out_kfree_of_ops: - kfree(of_ops); return ERR_PTR(ret); } diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index d55f9303afb5..2eace6b33b8c 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -123,8 +123,8 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (temp != trip->temperature) { - if (tz->ops->set_trip_temp) { - ret = tz->ops->set_trip_temp(tz, trip_id, temp); + if (tz->ops.set_trip_temp) { + ret = tz->ops.set_trip_temp(tz, trip_id, temp); if (ret) goto unlock; } @@ -174,8 +174,8 @@ trip_point_hyst_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (hyst != trip->hysteresis) { - if (tz->ops->set_trip_hyst) { - ret = tz->ops->set_trip_hyst(tz, trip_id, hyst); + if (tz->ops.set_trip_hyst) { + ret = tz->ops.set_trip_hyst(tz, trip_id, hyst); if (ret) goto unlock; } @@ -250,10 +250,10 @@ emul_temp_store(struct device *dev, struct device_attribute *attr, mutex_lock(&tz->lock); - if (!tz->ops->set_emul_temp) + if (!tz->ops.set_emul_temp) tz->emul_temperature = temperature; else - ret = tz->ops->set_emul_temp(tz, temperature); + ret = tz->ops.set_emul_temp(tz, temperature); if (!ret) __thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED); @@ -474,7 +474,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_hyst_attrs[indx].name; tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show; - if (tz->ops->set_trip_hyst) { + if (tz->ops.set_trip_hyst) { tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_hyst_attrs[indx].attr.store = trip_point_hyst_store; diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index a80ca78f8473..09f6050dd041 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -70,7 +70,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) lockdep_assert_held(&tz->lock); - if (!tz->ops->set_trips) + if (!tz->ops.set_trips) return; for_each_trip(tz, trip) { @@ -114,7 +114,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) * Set a temperature window. When this window is left the driver * must inform the thermal core via thermal_zone_device_update. */ - ret = tz->ops->set_trips(tz, low, high); + ret = tz->ops.set_trips(tz, low, high); if (ret) dev_err(&tz->device, "Failed to set trips: %d\n", ret); } diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 572a24f29a10..ec0559e98d6f 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -182,7 +182,7 @@ struct thermal_zone_device { int prev_low_trip; int prev_high_trip; atomic_t need_update; - struct thermal_zone_device_ops *ops; + struct thermal_zone_device_ops ops; struct thermal_zone_params *tzp; struct thermal_governor *governor; void *governor_data; @@ -318,14 +318,14 @@ struct thermal_zone_device *thermal_zone_device_register_with_trips( const struct thermal_trip *trips, int num_trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay); struct thermal_zone_device *thermal_tripless_zone_device_register( const char *type, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp); void thermal_zone_device_unregister(struct thermal_zone_device *tz); @@ -378,7 +378,7 @@ static inline struct thermal_zone_device *thermal_zone_device_register_with_trip const struct thermal_trip *trips, int num_trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { return ERR_PTR(-ENODEV); } From 75fb8714728460c4584c72e2f6410944a0b2cc44 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 14 Feb 2024 13:48:34 +0100 Subject: [PATCH 0724/1038] thermal: ACPI: Constify acpi_thermal_zone_ops Because thermal zone operations are now stored directly in struct thermal_zone_device, acpi_thermal_zone_ops need not be modified by the thermal core and so it can be const. Adjust the code accordingly. No functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Daniel Lezcano --- drivers/acpi/thermal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 6c4ba00d60e2..8042e96016e6 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -620,7 +620,7 @@ acpi_thermal_unbind_cooling_device(struct thermal_zone_device *thermal, return acpi_thermal_bind_unbind_cdev(thermal, cdev, false); } -static struct thermal_zone_device_ops acpi_thermal_zone_ops = { +static const struct thermal_zone_device_ops acpi_thermal_zone_ops = { .bind = acpi_thermal_bind_cooling_device, .unbind = acpi_thermal_unbind_cooling_device, .get_temp = thermal_get_temp, From 62dd17846d33e75e623965e281e246dff0e01309 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 5 Feb 2024 22:20:32 +0100 Subject: [PATCH 0725/1038] thermal: intel: Adjust ops handling during thermal zone registration Because thermal zone operations are now stored directly in struct thermal_zone_device, thermal zone creators can discard the operations structure after the zone registration is complete, or it can be made read-only. Accordingly, make int340x_thermal_zone_add() use a local variable to represent thermal zone operations, so it is freed automatically upon the function exit, and make the other Intel thermal drivers use const zone operations structures. Signed-off-by: Rafael J. Wysocki Reviewed-by: Stanislaw Gruszka Reviewed-by: Daniel Lezcano --- .../int340x_thermal/int340x_thermal_zone.c | 26 +++++-------------- .../int340x_thermal/int340x_thermal_zone.h | 1 - .../processor_thermal_device_pci.c | 2 +- drivers/thermal/intel/intel_pch_thermal.c | 2 +- .../thermal/intel/intel_quark_dts_thermal.c | 2 +- drivers/thermal/intel/intel_soc_dts_iosf.c | 2 +- drivers/thermal/intel/x86_pkg_temp_thermal.c | 2 +- 7 files changed, 11 insertions(+), 26 deletions(-) diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index 950e91ca5f42..1b0a9e9206c4 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -61,12 +61,6 @@ static void int340x_thermal_critical(struct thermal_zone_device *zone) dev_dbg(&zone->device, "%s: critical temperature reached\n", zone->type); } -static struct thermal_zone_device_ops int340x_thermal_zone_ops = { - .get_temp = int340x_thermal_get_zone_temp, - .set_trip_temp = int340x_thermal_set_trip_temp, - .critical = int340x_thermal_critical, -}; - static inline void *int_to_trip_priv(int i) { return (void *)(long)i; @@ -126,6 +120,11 @@ static struct thermal_zone_params int340x_thermal_params = { struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int (*get_temp) (struct thermal_zone_device *, int *)) { + const struct thermal_zone_device_ops zone_ops = { + .set_trip_temp = int340x_thermal_set_trip_temp, + .critical = int340x_thermal_critical, + .get_temp = get_temp ? get_temp : int340x_thermal_get_zone_temp, + }; struct int34x_thermal_zone *int34x_zone; struct thermal_trip *zone_trips; unsigned long long trip_cnt = 0; @@ -140,16 +139,6 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->adev = adev; - int34x_zone->ops = kmemdup(&int340x_thermal_zone_ops, - sizeof(int340x_thermal_zone_ops), GFP_KERNEL); - if (!int34x_zone->ops) { - ret = -ENOMEM; - goto err_ops_alloc; - } - - if (get_temp) - int34x_zone->ops->get_temp = get_temp; - status = acpi_evaluate_integer(adev->handle, "PATC", NULL, &trip_cnt); if (ACPI_SUCCESS(status)) { int34x_zone->aux_trip_nr = trip_cnt; @@ -185,7 +174,7 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, acpi_device_bid(adev), zone_trips, trip_cnt, trip_mask, int34x_zone, - int34x_zone->ops, + &zone_ops, &int340x_thermal_params, 0, 0); kfree(zone_trips); @@ -205,8 +194,6 @@ err_enable: err_thermal_zone: acpi_lpat_free_conversion_table(int34x_zone->lpat_table); err_trips_alloc: - kfree(int34x_zone->ops); -err_ops_alloc: kfree(int34x_zone); return ERR_PTR(ret); } @@ -216,7 +203,6 @@ void int340x_thermal_zone_remove(struct int34x_thermal_zone *int34x_zone) { thermal_zone_device_unregister(int34x_zone->zone); acpi_lpat_free_conversion_table(int34x_zone->lpat_table); - kfree(int34x_zone->ops); kfree(int34x_zone); } EXPORT_SYMBOL_GPL(int340x_thermal_zone_remove); diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h index 91edc7f83d79..d504e271009a 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h @@ -22,7 +22,6 @@ struct int34x_thermal_zone { struct acpi_device *adev; int aux_trip_nr; struct thermal_zone_device *zone; - struct thermal_zone_device_ops *ops; void *priv_data; struct acpi_lpat_conversion_table *lpat_table; }; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index bf47f319e3c6..8ff7e914ecf8 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -233,7 +233,7 @@ static int get_trip_temp(struct proc_thermal_pci *pci_info) return temp; } -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, }; diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c index 76a552870fe0..8bfb221e57e7 100644 --- a/drivers/thermal/intel/intel_pch_thermal.c +++ b/drivers/thermal/intel/intel_pch_thermal.c @@ -132,7 +132,7 @@ static void pch_critical(struct thermal_zone_device *tzd) thermal_zone_device_type(tzd)); } -static struct thermal_zone_device_ops tzd_ops = { +static const struct thermal_zone_device_ops tzd_ops = { .get_temp = pch_thermal_get_temp, .critical = pch_critical, }; diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c index 06f6d1468717..210b822bc988 100644 --- a/drivers/thermal/intel/intel_quark_dts_thermal.c +++ b/drivers/thermal/intel/intel_quark_dts_thermal.c @@ -292,7 +292,7 @@ static int sys_change_mode(struct thermal_zone_device *tzd, return ret; } -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, .change_mode = sys_change_mode, diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c index 096cdd3c3b1f..c008eb42d82e 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.c +++ b/drivers/thermal/intel/intel_soc_dts_iosf.c @@ -168,7 +168,7 @@ static int sys_get_curr_temp(struct thermal_zone_device *tzd, return 0; } -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, }; diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index dafd28a39bcc..20cb21cb97c1 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -166,7 +166,7 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp) } /* Thermal zone callback registry */ -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, }; From a85739c8c6894c3b9ff860e79e91db44cb59bd63 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 12 Feb 2024 19:26:25 +0100 Subject: [PATCH 0726/1038] thermal: Get rid of CONFIG_THERMAL_WRITABLE_TRIPS The only difference made by CONFIG_THERMAL_WRITABLE_TRIPS is whether or not the writable trips mask passed during thermal zone registration will take any effect, but whoever passes a non-zero writable trips mask to thermal_zone_device_register_with_trips() can be forgiven thinking that it will always work. Moreover, some thermal drivers expect user space to set trip temperature values, so they select CONFIG_THERMAL_WRITABLE_TRIPS, possibly overriding a manual choice to unset it and going against the design purportedly allowing system integrators to decide on the writability of trip points for the given kernel build. It is also set in one platform's defconfig. Forthermore, CONFIG_THERMAL_WRITABLE_TRIPS only affects trip temperature, because trip hysteresis is writable as long as the thermal zone provides a callback to update it, regardless of the CONFIG_THERMAL_WRITABLE_TRIPS value. The above means that the symbol in question is used inconsistently and its purpose is at least moot, so remove it and always take the writable trip mask passed to thermal_zone_device_register_with_trips() into account. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- arch/arm/configs/imx_v6_v7_defconfig | 1 - drivers/thermal/Kconfig | 11 ----------- drivers/thermal/intel/Kconfig | 2 -- drivers/thermal/thermal_sysfs.c | 3 +-- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig index 0a90583f9f01..ae20dc002dca 100644 --- a/arch/arm/configs/imx_v6_v7_defconfig +++ b/arch/arm/configs/imx_v6_v7_defconfig @@ -228,7 +228,6 @@ CONFIG_SENSORS_IIO_HWMON=y CONFIG_SENSORS_PWM_FAN=y CONFIG_SENSORS_SY7636A=y CONFIG_THERMAL_STATISTICS=y -CONFIG_THERMAL_WRITABLE_TRIPS=y CONFIG_CPU_THERMAL=y CONFIG_IMX_THERMAL=y CONFIG_WATCHDOG=y diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 17a8ae5e991d..204ed89a3ec9 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -83,17 +83,6 @@ config THERMAL_OF Say 'Y' here if you need to build thermal infrastructure based on device tree. -config THERMAL_WRITABLE_TRIPS - bool "Enable writable trip points" - help - This option allows the system integrator to choose whether - trip temperatures can be changed from userspace. The - writable trips need to be specified when setting up the - thermal zone but the choice here takes precedence. - - Say 'Y' here if you would like to allow userspace tools to - change trip temperatures. - choice prompt "Default Thermal governor" default THERMAL_DEFAULT_GOV_STEP_WISE diff --git a/drivers/thermal/intel/Kconfig b/drivers/thermal/intel/Kconfig index b43953b5539f..a31f2f32996a 100644 --- a/drivers/thermal/intel/Kconfig +++ b/drivers/thermal/intel/Kconfig @@ -23,7 +23,6 @@ config X86_PKG_TEMP_THERMAL tristate "X86 package temperature thermal driver" depends on X86_THERMAL_VECTOR select THERMAL_GOV_USER_SPACE - select THERMAL_WRITABLE_TRIPS select INTEL_TCC default m help @@ -47,7 +46,6 @@ config INTEL_SOC_DTS_THERMAL tristate "Intel SoCs DTS thermal driver" depends on X86 && PCI && ACPI select INTEL_SOC_DTS_IOSF_CORE - select THERMAL_WRITABLE_TRIPS help Enable this to register Intel SoCs (e.g. Bay Trail) platform digital temperature sensor (DTS). These SoCs have two additional DTSs in diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 2eace6b33b8c..83914aa2248b 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -458,8 +458,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_temp_attrs[indx].name; tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show; - if (IS_ENABLED(CONFIG_THERMAL_WRITABLE_TRIPS) && - mask & (1 << indx)) { + if (mask & (1 << indx)) { tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_temp_attrs[indx].attr.store = trip_point_temp_store; From edc6670233a333ccfd1ec0548f068bd121d209c8 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 23 Feb 2024 18:16:12 +0800 Subject: [PATCH 0727/1038] cred: Use KMEM_CACHE() instead of kmem_cache_create() Commit 0a31bd5f2bbb ("KMEM_CACHE(): simplify slab cache creation") introduces a new macro. Use the new KMEM_CACHE() macro instead of direct kmem_cache_create() to simplify the creation of SLAB caches. Signed-off-by: Kunwu Chan [PM: alignment fixes in both code and description] Signed-off-by: Paul Moore --- kernel/cred.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/cred.c b/kernel/cred.c index c033a201c808..075cfa7c896f 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -606,8 +606,8 @@ int set_cred_ucounts(struct cred *new) void __init cred_init(void) { /* allocate a slab in which we can store credentials */ - cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, - SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + cred_jar = KMEM_CACHE(cred, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); } /** From 240a8da623008eb9f4e32c7a19ce16a6605911dc Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Mon, 19 Feb 2024 18:26:06 -0800 Subject: [PATCH 0728/1038] cpufreq: intel_pstate: Allow model specific EPPs The current implementation allows model specific EPP override for balanced_performance. Add feature to allow model specific EPP for all predefined EPP strings. For example for some CPU models, even changing performance EPP has benefits Use a mask of EPPs as driver_data instead of just balanced_performance. Signed-off-by: Srinivas Pandruvada Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 41 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 5ad3542c0e1e..001f6d95453e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -3401,14 +3402,29 @@ static bool intel_pstate_hwp_is_enabled(void) return !!(value & 0x1); } -static const struct x86_cpu_id intel_epp_balance_perf[] = { +#define POWERSAVE_MASK GENMASK(7, 0) +#define BALANCE_POWER_MASK GENMASK(15, 8) +#define BALANCE_PERFORMANCE_MASK GENMASK(23, 16) +#define PERFORMANCE_MASK GENMASK(31, 24) + +#define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \ + (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\ + FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\ + FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\ + FIELD_PREP_CONST(PERFORMANCE_MASK, performance)) + +#define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \ + (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\ + balance_perf, HWP_EPP_PERFORMANCE)) + +static const struct x86_cpu_id intel_epp_default[] = { /* * Set EPP value as 102, this is the max suggested EPP * which can result in one core turbo frequency for * AlderLake Mobile CPUs. */ - X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, 102), - X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, 32), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)), + X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), {} }; @@ -3506,11 +3522,24 @@ hwp_cpu_matched: intel_pstate_sysfs_expose_params(); if (hwp_active) { - const struct x86_cpu_id *id = x86_match_cpu(intel_epp_balance_perf); + const struct x86_cpu_id *id = x86_match_cpu(intel_epp_default); const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor); - if (id) - epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = id->driver_data; + if (id) { + epp_values[EPP_INDEX_POWERSAVE] = + FIELD_GET(POWERSAVE_MASK, id->driver_data); + epp_values[EPP_INDEX_BALANCE_POWERSAVE] = + FIELD_GET(BALANCE_POWER_MASK, id->driver_data); + epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = + FIELD_GET(BALANCE_PERFORMANCE_MASK, id->driver_data); + epp_values[EPP_INDEX_PERFORMANCE] = + FIELD_GET(PERFORMANCE_MASK, id->driver_data); + pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n", + epp_values[EPP_INDEX_POWERSAVE], + epp_values[EPP_INDEX_BALANCE_POWERSAVE], + epp_values[EPP_INDEX_BALANCE_PERFORMANCE], + epp_values[EPP_INDEX_PERFORMANCE]); + } if (hybrid_id) { hybrid_scaling_factor = hybrid_id->driver_data; From 1f4b7fdd71e066aa7c01e3e26ceeb39b47dd5461 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Mon, 19 Feb 2024 18:26:07 -0800 Subject: [PATCH 0729/1038] cpufreq: intel_pstate: Update default EPPs for Meteor Lake Update default balanced_performance EPP to 115 and performance EPP to 16. Changing the balanced_performance EPP has better performance/watt compared to default powerup EPP value of 128. Changing the performance EPP to 0x10 shows reduced power for similar performance as EPP 0. On small form factor devices this is beneficial as lower power results in lower CPU and skin temperature. This results in reduced thermal throttling and higher performance. Signed-off-by: Srinivas Pandruvada Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 001f6d95453e..4d2a1aed6c39 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3425,6 +3425,8 @@ static const struct x86_cpu_id intel_epp_default[] = { */ X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, + HWP_EPP_BALANCE_POWERSAVE, 115, 16)), {} }; From 88debc69754f7fe5186954941bb1cc4d744f4f25 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 16:34:15 +0100 Subject: [PATCH 0730/1038] cpufreq: Remove references to 10ms min sampling rate A minimum sampling rate value of 10ms was introduced in: commit cef9615a853e ("[CPUFREQ] ondemand: Uncouple minimal sampling rate from HZ in NO_HZ case") The use of this value was removed in: commit ed4676e25463 ("cpufreq: Replace "max_transition_latency" with "dynamic_switching"") Remove: - a comment referencing this value - an unused macro associated to this value Signed-off-by: Pierre Gondois Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq_ondemand.c | 1 - include/linux/cpufreq.h | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index c52d19d67557..a7c38b8b3e78 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -22,7 +22,6 @@ #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (100000) #define MICRO_FREQUENCY_UP_THRESHOLD (95) -#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) #define MIN_FREQUENCY_UP_THRESHOLD (1) #define MAX_FREQUENCY_UP_THRESHOLD (100) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 9bebeec24abb..85908b3a2f24 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -569,9 +569,7 @@ static inline unsigned long cpufreq_scale(unsigned long old, u_int div, /* * The polling frequency depends on the capability of the processor. Default - * polling frequency is 1000 times the transition latency of the processor. The - * ondemand governor will work on any processor with transition latency <= 10ms, - * using appropriate sampling rate. + * polling frequency is 1000 times the transition latency of the processor. */ #define LATENCY_MULTIPLIER (1000) From cbc29538dbf7d7400f1ffc5dd5713e6a551463a0 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Mon, 29 Jan 2024 17:13:25 +0100 Subject: [PATCH 0731/1038] hwmon: Add driver for LTC4282 The LTC4282 hot swap controller allows a board to be safely inserted and removed from a live backplane. Using one or more external N-channel pass transistors, board supply voltage and inrush current are ramped up at an adjustable rate. An I2C interface and onboard ADC allows for monitoring of board current, voltage, power, energy and fault status. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240129-b4-ltc4282-support-v4-3-fe75798164cc@analog.com [groeck: clamp value range in ltc4282_write_voltage_byte_cached()] Signed-off-by: Guenter Roeck --- Documentation/hwmon/index.rst | 1 + Documentation/hwmon/ltc4282.rst | 133 +++ MAINTAINERS | 2 + drivers/hwmon/Kconfig | 11 + drivers/hwmon/Makefile | 1 + drivers/hwmon/ltc4282.c | 1782 +++++++++++++++++++++++++++++++ 6 files changed, 1930 insertions(+) create mode 100644 Documentation/hwmon/ltc4282.rst create mode 100644 drivers/hwmon/ltc4282.c diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index c7ed1f73ac06..f16c6dfaec7d 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -129,6 +129,7 @@ Hardware Monitoring Kernel Drivers ltc4245 ltc4260 ltc4261 + ltc4282 ltc4286 max127 max15301 diff --git a/Documentation/hwmon/ltc4282.rst b/Documentation/hwmon/ltc4282.rst new file mode 100644 index 000000000000..a87ec3564998 --- /dev/null +++ b/Documentation/hwmon/ltc4282.rst @@ -0,0 +1,133 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel drivers ltc4282 +========================================== + +Supported chips: + + * Analog Devices LTC4282 + + Prefix: 'ltc4282' + + Addresses scanned: - I2C 0x40 - 0x5A (7-bit) + Addresses scanned: - I2C 0x80 - 0xB4 with a step of 2 (8-bit) + + Datasheet: + + https://www.analog.com/media/en/technical-documentation/data-sheets/ltc4282.pdf + +Author: Nuno Sá + +Description +___________ + +The LTC4282 hot swap controller allows a board to be safely inserted and removed +from a live backplane. Using one or more external N-channel pass transistors, +board supply voltage and inrush current are ramped up at an adjustable rate. An +I2C interface and onboard ADC allows for monitoring of board current, voltage, +power, energy and fault status. The device features analog foldback current +limiting and supply monitoring for applications from 2.9V to 33V. Dual 12V gate +drive allows high power applications to either share safe operating area across +parallel MOSFETs or support a 2-stage start-up that first charges the load +capacitance followed by enabling a low on-resistance path to the load. The +LTC4282 is well suited to high power applications because the precise monitoring +capability and accurate current limiting reduce the extremes in which both loads +and power supplies must safely operate. Non-volatile configuration allows for +flexibility in the autonomous generation of alerts and response to faults. + +Sysfs entries +_____________ + +The following attributes are supported. Limits are read-write and all the other +attributes are read-only. Note that in0 and in1 are mutually exclusive. Enabling +one disables the other and disabling one enables the other. + +======================= ========================================== +in0_input Output voltage (mV). +in0_min Undervoltage threshold +in0_max Overvoltage threshold +in0_lowest Lowest measured voltage +in0_highest Highest measured voltage +in0_reset_history Write 1 to reset in0 history. + Also clears fet bad and short fault logs. +in0_min_alarm Undervoltage alarm +in0_max_alarm Overvoltage alarm +in0_enable Enable/Disable VSOURCE monitoring +in0_fault Failure in the MOSFETs. Either bad or shorted FET. +in0_label Channel label (VSOURCE) + +in1_input Input voltage (mV). +in1_min Undervoltage threshold +in1_max Overvoltage threshold +in1_lowest Lowest measured voltage +in1_highest Highest measured voltage +in1_reset_history Write 1 to reset in1 history. + Also clears over/undervoltage fault logs. +in1_min_alarm Undervoltage alarm +in1_max_alarm Overvoltage alarm +in1_lcrit_alarm Critical Undervoltage alarm +in1_crit_alarm Critical Overvoltage alarm +in1_enable Enable/Disable VDD monitoring +in1_label Channel label (VDD) + +in2_input GPIO voltage (mV) +in2_min Undervoltage threshold +in2_max Overvoltage threshold +in2_lowest Lowest measured voltage +in2_highest Highest measured voltage +in2_reset_history Write 1 to reset in2 history +in2_min_alarm Undervoltage alarm +in2_max_alarm Overvoltage alarm +in2_label Channel label (VGPIO) + +curr1_input Sense current (mA) +curr1_min Undercurrent threshold +curr1_max Overcurrent threshold +curr1_lowest Lowest measured current +curr1_highest Highest measured current +curr1_reset_history Write 1 to reset curr1 history. + Also clears overcurrent fault logs. +curr1_min_alarm Undercurrent alarm +curr1_max_alarm Overcurrent alarm +curr1_crit_alarm Critical Overcurrent alarm +curr1_label Channel label (ISENSE) + +power1_input Power (in uW) +power1_min Low power threshold +power1_max High power threshold +power1_input_lowest Historical minimum power use +power1_input_highest Historical maximum power use +power1_reset_history Write 1 to reset power1 history. + Also clears power bad fault logs. +power1_min_alarm Low power alarm +power1_max_alarm High power alarm +power1_label Channel label (Power) + +energy1_input Measured energy over time (in microJoule) +energy1_enable Enable/Disable Energy accumulation +======================= ========================================== + +DebugFs entries +_______________ + +The chip also has a fault log register where failures can be logged. Hence, +as these are logging events, we give access to them in debugfs. Note that +even if some failure is detected in these logs, it does necessarily mean +that the failure is still present. As mentioned in the proper Sysfs entries, +these logs can be cleared by writing in the proper reset_history attribute. + +.. warning:: The debugfs interface is subject to change without notice + and is only available when the kernel is compiled with + ``CONFIG_DEBUG_FS`` defined. + +``/sys/kernel/debug/ltc4282-hwmon[X]/`` +contains the following attributes: + +======================= ========================================== +power1_bad_fault_log Set to 1 by a power1 bad fault occurring. +in0_fet_short_fault_log Set to 1 when the ADC detects a FET-short fault. +in0_fet_bad_fault_log Set to 1 when a FET-BAD fault occurs. +in1_crit_fault_log Set to 1 by a VDD overvoltage fault occurring. +in1_lcrit_fault_log Set to 1 by a VDD undervoltage fault occurring. +curr1_crit_fault_log Set to 1 by an overcurrent fault occurring. +======================= ========================================== diff --git a/MAINTAINERS b/MAINTAINERS index 3625754d7d1b..df8ea2d94a0c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12765,6 +12765,8 @@ M: Nuno Sa L: linux-hwmon@vger.kernel.org S: Supported F: Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml +F: Documentation/hwmon/ltc4282.rst +F: drivers/hwmon/ltc4282.c LTC4286 HARDWARE MONITOR DRIVER M: Delphine CC Chiu diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index a608264da87d..f6160cc70077 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1038,6 +1038,17 @@ config SENSORS_LTC4261 This driver can also be built as a module. If so, the module will be called ltc4261. +config SENSORS_LTC4282 + tristate "Analog Devices LTC4282" + depends on I2C + select REGMAP_I2C + help + If you say yes here you get support for Analog Devices LTC4282 + High Current Hot Swap Controller I2C interface. + + This driver can also be built as a module. If so, the module will + be called ltc4282. + config SENSORS_LTQ_CPUTEMP bool "Lantiq cpu temperature sensor driver" depends on SOC_XWAY diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 47be39af5c03..8bfc422a29e5 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -136,6 +136,7 @@ obj-$(CONFIG_SENSORS_LTC4222) += ltc4222.o obj-$(CONFIG_SENSORS_LTC4245) += ltc4245.o obj-$(CONFIG_SENSORS_LTC4260) += ltc4260.o obj-$(CONFIG_SENSORS_LTC4261) += ltc4261.o +obj-$(CONFIG_SENSORS_LTC4282) += ltc4282.o obj-$(CONFIG_SENSORS_LTQ_CPUTEMP) += ltq-cputemp.o obj-$(CONFIG_SENSORS_MAX1111) += max1111.o obj-$(CONFIG_SENSORS_MAX127) += max127.o diff --git a/drivers/hwmon/ltc4282.c b/drivers/hwmon/ltc4282.c new file mode 100644 index 000000000000..4f608a3790fb --- /dev/null +++ b/drivers/hwmon/ltc4282.c @@ -0,0 +1,1782 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Analog Devices LTC4282 I2C High Current Hot Swap Controller over I2C + * + * Copyright 2023 Analog Devices Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LTC4282_CTRL_LSB 0x00 + #define LTC4282_CTRL_OV_RETRY_MASK BIT(0) + #define LTC4282_CTRL_UV_RETRY_MASK BIT(1) + #define LTC4282_CTRL_OC_RETRY_MASK BIT(2) + #define LTC4282_CTRL_ON_ACTIVE_LOW_MASK BIT(5) + #define LTC4282_CTRL_ON_DELAY_MASK BIT(6) +#define LTC4282_CTRL_MSB 0x01 + #define LTC4282_CTRL_VIN_MODE_MASK GENMASK(1, 0) + #define LTC4282_CTRL_OV_MODE_MASK GENMASK(3, 2) + #define LTC4282_CTRL_UV_MODE_MASK GENMASK(5, 4) +#define LTC4282_FAULT_LOG 0x04 + #define LTC4282_OV_FAULT_MASK BIT(0) + #define LTC4282_UV_FAULT_MASK BIT(1) + #define LTC4282_VDD_FAULT_MASK \ + (LTC4282_OV_FAULT_MASK | LTC4282_UV_FAULT_MASK) + #define LTC4282_OC_FAULT_MASK BIT(2) + #define LTC4282_POWER_BAD_FAULT_MASK BIT(3) + #define LTC4282_FET_SHORT_FAULT_MASK BIT(5) + #define LTC4282_FET_BAD_FAULT_MASK BIT(6) + #define LTC4282_FET_FAILURE_FAULT_MASK \ + (LTC4282_FET_SHORT_FAULT_MASK | LTC4282_FET_BAD_FAULT_MASK) +#define LTC4282_ADC_ALERT_LOG 0x05 + #define LTC4282_GPIO_ALARM_L_MASK BIT(0) + #define LTC4282_GPIO_ALARM_H_MASK BIT(1) + #define LTC4282_VSOURCE_ALARM_L_MASK BIT(2) + #define LTC4282_VSOURCE_ALARM_H_MASK BIT(3) + #define LTC4282_VSENSE_ALARM_L_MASK BIT(4) + #define LTC4282_VSENSE_ALARM_H_MASK BIT(5) + #define LTC4282_POWER_ALARM_L_MASK BIT(6) + #define LTC4282_POWER_ALARM_H_MASK BIT(7) +#define LTC4282_FET_BAD_FAULT_TIMEOUT 0x06 + #define LTC4282_FET_BAD_MAX_TIMEOUT 255 +#define LTC4282_GPIO_CONFIG 0x07 + #define LTC4282_GPIO_2_FET_STRESS_MASK BIT(1) + #define LTC4282_GPIO_1_CONFIG_MASK GENMASK(5, 4) +#define LTC4282_VGPIO_MIN 0x08 +#define LTC4282_VGPIO_MAX 0x09 +#define LTC4282_VSOURCE_MIN 0x0a +#define LTC4282_VSOURCE_MAX 0x0b +#define LTC4282_VSENSE_MIN 0x0c +#define LTC4282_VSENSE_MAX 0x0d +#define LTC4282_POWER_MIN 0x0e +#define LTC4282_POWER_MAX 0x0f +#define LTC4282_CLK_DIV 0x10 + #define LTC4282_CLK_DIV_MASK GENMASK(4, 0) + #define LTC4282_CLKOUT_MASK GENMASK(6, 5) +#define LTC4282_ILIM_ADJUST 0x11 + #define LTC4282_GPIO_MODE_MASK BIT(1) + #define LTC4282_VDD_MONITOR_MASK BIT(2) + #define LTC4282_FOLDBACK_MODE_MASK GENMASK(4, 3) + #define LTC4282_ILIM_ADJUST_MASK GENMASK(7, 5) +#define LTC4282_ENERGY 0x12 +#define LTC4282_TIME_COUNTER 0x18 +#define LTC4282_ALERT_CTRL 0x1c + #define LTC4282_ALERT_OUT_MASK BIT(6) +#define LTC4282_ADC_CTRL 0x1d + #define LTC4282_FAULT_LOG_EN_MASK BIT(2) + #define LTC4282_METER_HALT_MASK BIT(5) + #define LTC4282_METER_RESET_MASK BIT(6) + #define LTC4282_RESET_MASK BIT(7) +#define LTC4282_STATUS_LSB 0x1e + #define LTC4282_OV_STATUS_MASK BIT(0) + #define LTC4282_UV_STATUS_MASK BIT(1) + #define LTC4282_VDD_STATUS_MASK \ + (LTC4282_OV_STATUS_MASK | LTC4282_UV_STATUS_MASK) + #define LTC4282_OC_STATUS_MASK BIT(2) + #define LTC4282_POWER_GOOD_MASK BIT(3) + #define LTC4282_FET_FAILURE_MASK GENMASK(6, 5) +#define LTC4282_STATUS_MSB 0x1f +#define LTC4282_RESERVED_1 0x32 +#define LTC4282_RESERVED_2 0x33 +#define LTC4282_VGPIO 0x34 +#define LTC4282_VGPIO_LOWEST 0x36 +#define LTC4282_VGPIO_HIGHEST 0x38 +#define LTC4282_VSOURCE 0x3a +#define LTC4282_VSOURCE_LOWEST 0x3c +#define LTC4282_VSOURCE_HIGHEST 0x3e +#define LTC4282_VSENSE 0x40 +#define LTC4282_VSENSE_LOWEST 0x42 +#define LTC4282_VSENSE_HIGHEST 0x44 +#define LTC4282_POWER 0x46 +#define LTC4282_POWER_LOWEST 0x48 +#define LTC4282_POWER_HIGHEST 0x4a +#define LTC4282_RESERVED_3 0x50 + +#define LTC4282_CLKIN_MIN (250 * KILO) +#define LTC4282_CLKIN_MAX (15500 * KILO) +#define LTC4282_CLKIN_RANGE (LTC4282_CLKIN_MAX - LTC4282_CLKIN_MIN + 1) +#define LTC4282_CLKOUT_SYSTEM (250 * KILO) +#define LTC4282_CLKOUT_CNV 15 + +enum { + LTC4282_CHAN_VSOURCE, + LTC4282_CHAN_VDD, + LTC4282_CHAN_VGPIO, +}; + +struct ltc4282_cache { + u32 in_max_raw; + u32 in_min_raw; + long in_highest; + long in_lowest; + bool en; +}; + +struct ltc4282_state { + struct regmap *map; + /* Protect against multiple accesses to the device registers */ + struct mutex lock; + struct clk_hw clk_hw; + /* + * Used to cache values for VDD/VSOURCE depending which will be used + * when hwmon is not enabled for that channel. Needed because they share + * the same registers. + */ + struct ltc4282_cache in0_1_cache[LTC4282_CHAN_VGPIO]; + u32 vsense_max; + long power_max; + u32 rsense; + u16 vdd; + u16 vfs_out; + bool energy_en; +}; + +enum { + LTC4282_CLKOUT_NONE, + LTC4282_CLKOUT_INT, + LTC4282_CLKOUT_TICK, +}; + +static int ltc4282_set_rate(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate) +{ + struct ltc4282_state *st = container_of(hw, struct ltc4282_state, + clk_hw); + u32 val = LTC4282_CLKOUT_INT; + + if (rate == LTC4282_CLKOUT_CNV) + val = LTC4282_CLKOUT_TICK; + + return regmap_update_bits(st->map, LTC4282_CLK_DIV, LTC4282_CLKOUT_MASK, + FIELD_PREP(LTC4282_CLKOUT_MASK, val)); +} + +/* + * Note the 15HZ conversion rate assumes 12bit ADC which is what we are + * supporting for now. + */ +static const unsigned int ltc4282_out_rates[] = { + LTC4282_CLKOUT_CNV, LTC4282_CLKOUT_SYSTEM +}; + +static long ltc4282_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + int idx = find_closest(rate, ltc4282_out_rates, + ARRAY_SIZE(ltc4282_out_rates)); + + return ltc4282_out_rates[idx]; +} + +static unsigned long ltc4282_recalc_rate(struct clk_hw *hw, + unsigned long parent) +{ + struct ltc4282_state *st = container_of(hw, struct ltc4282_state, + clk_hw); + u32 clkdiv; + int ret; + + ret = regmap_read(st->map, LTC4282_CLK_DIV, &clkdiv); + if (ret) + return 0; + + clkdiv = FIELD_GET(LTC4282_CLKOUT_MASK, clkdiv); + if (!clkdiv) + return 0; + if (clkdiv == LTC4282_CLKOUT_INT) + return LTC4282_CLKOUT_SYSTEM; + + return LTC4282_CLKOUT_CNV; +} + +static void ltc4282_disable(struct clk_hw *clk_hw) +{ + struct ltc4282_state *st = container_of(clk_hw, struct ltc4282_state, + clk_hw); + + regmap_clear_bits(st->map, LTC4282_CLK_DIV, LTC4282_CLKOUT_MASK); +} + +static int ltc4282_read_voltage_word(const struct ltc4282_state *st, u32 reg, + u32 fs, long *val) +{ + __be16 in; + int ret; + + ret = regmap_bulk_read(st->map, reg, &in, sizeof(in)); + if (ret) + return ret; + + /* + * This is also used to calculate current in which case fs comes in + * 10 * uV. Hence the ULL usage. + */ + *val = DIV_ROUND_CLOSEST_ULL(be16_to_cpu(in) * (u64)fs, U16_MAX); + return 0; +} + +static int ltc4282_read_voltage_byte_cached(const struct ltc4282_state *st, + u32 reg, u32 fs, long *val, + u32 *cached_raw) +{ + int ret; + u32 in; + + if (cached_raw) { + in = *cached_raw; + } else { + ret = regmap_read(st->map, reg, &in); + if (ret) + return ret; + } + + *val = DIV_ROUND_CLOSEST(in * fs, U8_MAX); + return 0; +} + +static int ltc4282_read_voltage_byte(const struct ltc4282_state *st, u32 reg, + u32 fs, long *val) +{ + return ltc4282_read_voltage_byte_cached(st, reg, fs, val, NULL); +} + +static int __ltc4282_read_alarm(struct ltc4282_state *st, u32 reg, u32 mask, + long *val) +{ + u32 alarm; + int ret; + + ret = regmap_read(st->map, reg, &alarm); + if (ret) + return ret; + + *val = !!(alarm & mask); + + /* if not status/fault logs, clear the alarm after reading it */ + if (reg != LTC4282_STATUS_LSB && reg != LTC4282_FAULT_LOG) + return regmap_clear_bits(st->map, reg, mask); + + return 0; +} + +static int ltc4282_read_alarm(struct ltc4282_state *st, u32 reg, u32 mask, + long *val) +{ + guard(mutex)(&st->lock); + return __ltc4282_read_alarm(st, reg, mask, val); +} + +static int ltc4282_vdd_source_read_in(struct ltc4282_state *st, u32 channel, + long *val) +{ + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) + return -ENODATA; + + return ltc4282_read_voltage_word(st, LTC4282_VSOURCE, st->vfs_out, val); +} + +static int ltc4282_vdd_source_read_hist(struct ltc4282_state *st, u32 reg, + u32 channel, long *cached, long *val) +{ + int ret; + + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) { + *val = *cached; + return 0; + } + + ret = ltc4282_read_voltage_word(st, reg, st->vfs_out, val); + if (ret) + return ret; + + *cached = *val; + return 0; +} + +static int ltc4282_vdd_source_read_lim(struct ltc4282_state *st, u32 reg, + u32 channel, u32 *cached, long *val) +{ + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) + return ltc4282_read_voltage_byte_cached(st, reg, st->vfs_out, + val, cached); + + return ltc4282_read_voltage_byte(st, reg, st->vfs_out, val); +} + +static int ltc4282_vdd_source_read_alm(struct ltc4282_state *st, u32 mask, + u32 channel, long *val) +{ + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) { + /* + * Do this otherwise alarms can get confused because we clear + * them after reading them. So, if someone mistakenly reads + * VSOURCE right before VDD (or the other way around), we might + * get no alarm just because it was cleared when reading VSOURCE + * and had no time for a new conversion and thus having the + * alarm again. + */ + *val = 0; + return 0; + } + + return __ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, mask, val); +} + +static int ltc4282_read_in(struct ltc4282_state *st, u32 attr, long *val, + u32 channel) +{ + switch (attr) { + case hwmon_in_input: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_word(st, LTC4282_VGPIO, + 1280, val); + + return ltc4282_vdd_source_read_in(st, channel, val); + case hwmon_in_highest: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_word(st, + LTC4282_VGPIO_HIGHEST, + 1280, val); + + return ltc4282_vdd_source_read_hist(st, LTC4282_VSOURCE_HIGHEST, + channel, + &st->in0_1_cache[channel].in_highest, val); + case hwmon_in_lowest: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_word(st, LTC4282_VGPIO_LOWEST, + 1280, val); + + return ltc4282_vdd_source_read_hist(st, LTC4282_VSOURCE_LOWEST, + channel, + &st->in0_1_cache[channel].in_lowest, val); + case hwmon_in_max_alarm: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_GPIO_ALARM_H_MASK, + val); + + return ltc4282_vdd_source_read_alm(st, + LTC4282_VSOURCE_ALARM_H_MASK, + channel, val); + case hwmon_in_min_alarm: + if (channel == LTC4282_CHAN_VGPIO) + ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_GPIO_ALARM_L_MASK, val); + + return ltc4282_vdd_source_read_alm(st, + LTC4282_VSOURCE_ALARM_L_MASK, + channel, val); + case hwmon_in_crit_alarm: + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_OV_STATUS_MASK, val); + case hwmon_in_lcrit_alarm: + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_UV_STATUS_MASK, val); + case hwmon_in_max: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_byte(st, LTC4282_VGPIO_MAX, + 1280, val); + + return ltc4282_vdd_source_read_lim(st, LTC4282_VSOURCE_MAX, + channel, + &st->in0_1_cache[channel].in_max_raw, val); + case hwmon_in_min: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_byte(st, LTC4282_VGPIO_MIN, + 1280, val); + + return ltc4282_vdd_source_read_lim(st, LTC4282_VSOURCE_MIN, + channel, + &st->in0_1_cache[channel].in_min_raw, val); + case hwmon_in_enable: + scoped_guard(mutex, &st->lock) { + *val = st->in0_1_cache[channel].en; + } + return 0; + case hwmon_in_fault: + /* + * We report failure if we detect either a fer_bad or a + * fet_short in the status register. + */ + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_FET_FAILURE_MASK, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_read_current_word(const struct ltc4282_state *st, u32 reg, + long *val) +{ + long in; + int ret; + + /* + * We pass in full scale in 10 * micro (note that 40 is already + * millivolt) so we have better approximations to calculate current. + */ + ret = ltc4282_read_voltage_word(st, reg, DECA * 40 * MILLI, &in); + if (ret) + return ret; + + *val = DIV_ROUND_CLOSEST(in * MILLI, st->rsense); + + return 0; +} + +static int ltc4282_read_current_byte(const struct ltc4282_state *st, u32 reg, + long *val) +{ + long in; + int ret; + + ret = ltc4282_read_voltage_byte(st, reg, DECA * 40 * MILLI, &in); + if (ret) + return ret; + + *val = DIV_ROUND_CLOSEST(in * MILLI, st->rsense); + + return 0; +} + +static int ltc4282_read_curr(struct ltc4282_state *st, const u32 attr, + long *val) +{ + switch (attr) { + case hwmon_curr_input: + return ltc4282_read_current_word(st, LTC4282_VSENSE, val); + case hwmon_curr_highest: + return ltc4282_read_current_word(st, LTC4282_VSENSE_HIGHEST, + val); + case hwmon_curr_lowest: + return ltc4282_read_current_word(st, LTC4282_VSENSE_LOWEST, + val); + case hwmon_curr_max: + return ltc4282_read_current_byte(st, LTC4282_VSENSE_MAX, val); + case hwmon_curr_min: + return ltc4282_read_current_byte(st, LTC4282_VSENSE_MIN, val); + case hwmon_curr_max_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_VSENSE_ALARM_H_MASK, val); + case hwmon_curr_min_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_VSENSE_ALARM_L_MASK, val); + case hwmon_curr_crit_alarm: + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_OC_STATUS_MASK, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_read_power_word(const struct ltc4282_state *st, u32 reg, + long *val) +{ + u64 temp = DECA * 40ULL * st->vfs_out * BIT(16), temp_2; + __be16 raw; + u16 power; + int ret; + + ret = regmap_bulk_read(st->map, reg, &raw, sizeof(raw)); + if (ret) + return ret; + + power = be16_to_cpu(raw); + /* + * Power is given by: + * P = CODE(16b) * 0.040 * Vfs(out) * 2^16 / ((2^16 - 1)^2 * Rsense) + */ + if (check_mul_overflow(power * temp, MICRO, &temp_2)) { + temp = DIV_ROUND_CLOSEST_ULL(power * temp, U16_MAX); + *val = DIV64_U64_ROUND_CLOSEST(temp * MICRO, + U16_MAX * (u64)st->rsense); + return 0; + } + + *val = DIV64_U64_ROUND_CLOSEST(temp_2, + st->rsense * int_pow(U16_MAX, 2)); + + return 0; +} + +static int ltc4282_read_power_byte(const struct ltc4282_state *st, u32 reg, + long *val) +{ + u32 power; + u64 temp; + int ret; + + ret = regmap_read(st->map, reg, &power); + if (ret) + return ret; + + temp = power * 40 * DECA * st->vfs_out * BIT_ULL(8); + *val = DIV64_U64_ROUND_CLOSEST(temp * MICRO, + int_pow(U8_MAX, 2) * st->rsense); + + return 0; +} + +static int ltc4282_read_energy(const struct ltc4282_state *st, u64 *val) +{ + u64 temp, energy; + __be64 raw; + int ret; + + ret = regmap_bulk_read(st->map, LTC4282_ENERGY, &raw, 6); + if (ret) + return ret; + + energy = be64_to_cpu(raw) >> 16; + /* + * The formula for energy is given by: + * E = CODE(48b) * 0.040 * Vfs(out) * Tconv * 256 / + * ((2^16 - 1)^2 * Rsense) + * + * Since we only support 12bit ADC, Tconv = 0.065535s. Passing Vfs(out) + * and 0.040 to mV and Tconv to us, we can simplify the formula to: + * E = CODE(48b) * 40 * Vfs(out) * 256 / (U16_MAX * Rsense) + * + * As Rsense can have tenths of micro-ohm resolution, we need to + * multiply by DECA to get microujoule. + */ + if (check_mul_overflow(DECA * st->vfs_out * 40 * BIT(8), energy, &temp)) { + temp = DIV_ROUND_CLOSEST(DECA * st->vfs_out * 40 * BIT(8), U16_MAX); + *val = DIV_ROUND_CLOSEST_ULL(temp * energy, st->rsense); + return 0; + } + + *val = DIV64_U64_ROUND_CLOSEST(temp, U16_MAX * (u64)st->rsense); + + return 0; +} + +static int ltc4282_read_power(struct ltc4282_state *st, const u32 attr, + long *val) +{ + switch (attr) { + case hwmon_power_input: + return ltc4282_read_power_word(st, LTC4282_POWER, val); + case hwmon_power_input_highest: + return ltc4282_read_power_word(st, LTC4282_POWER_HIGHEST, val); + case hwmon_power_input_lowest: + return ltc4282_read_power_word(st, LTC4282_POWER_LOWEST, val); + case hwmon_power_max_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_POWER_ALARM_H_MASK, val); + case hwmon_power_min_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_POWER_ALARM_L_MASK, val); + case hwmon_power_max: + return ltc4282_read_power_byte(st, LTC4282_POWER_MAX, val); + case hwmon_power_min: + return ltc4282_read_power_byte(st, LTC4282_POWER_MIN, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct ltc4282_state *st = dev_get_drvdata(dev); + + switch (type) { + case hwmon_in: + return ltc4282_read_in(st, attr, val, channel); + case hwmon_curr: + return ltc4282_read_curr(st, attr, val); + case hwmon_power: + return ltc4282_read_power(st, attr, val); + case hwmon_energy: + scoped_guard(mutex, &st->lock) { + *val = st->energy_en; + } + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_write_power_byte(const struct ltc4282_state *st, u32 reg, + long val) +{ + u32 power; + u64 temp; + + if (val > st->power_max) + val = st->power_max; + + temp = val * int_pow(U8_MAX, 2) * st->rsense; + power = DIV64_U64_ROUND_CLOSEST(temp, + MICRO * DECA * 256ULL * st->vfs_out * 40); + + return regmap_write(st->map, reg, power); +} + +static int ltc4282_write_power_word(const struct ltc4282_state *st, u32 reg, + long val) +{ + u64 temp = int_pow(U16_MAX, 2) * st->rsense, temp_2; + __be16 __raw; + u16 code; + + if (check_mul_overflow(temp, val, &temp_2)) { + temp = DIV_ROUND_CLOSEST_ULL(temp, DECA * MICRO); + code = DIV64_U64_ROUND_CLOSEST(temp * val, + 40ULL * BIT(16) * st->vfs_out); + } else { + temp = DECA * MICRO * 40ULL * BIT(16) * st->vfs_out; + code = DIV64_U64_ROUND_CLOSEST(temp_2, temp); + } + + __raw = cpu_to_be16(code); + return regmap_bulk_write(st->map, reg, &__raw, sizeof(__raw)); +} + +static int __ltc4282_in_write_history(const struct ltc4282_state *st, u32 reg, + long lowest, long highest, u32 fs) +{ + __be16 __raw; + u16 tmp; + int ret; + + tmp = DIV_ROUND_CLOSEST(U16_MAX * lowest, fs); + + __raw = cpu_to_be16(tmp); + + ret = regmap_bulk_write(st->map, reg, &__raw, 2); + if (ret) + return ret; + + tmp = DIV_ROUND_CLOSEST(U16_MAX * highest, fs); + + __raw = cpu_to_be16(tmp); + + return regmap_bulk_write(st->map, reg + 2, &__raw, 2); +} + +static int ltc4282_in_write_history(struct ltc4282_state *st, u32 reg, + long lowest, long highest, u32 fs) +{ + guard(mutex)(&st->lock); + return __ltc4282_in_write_history(st, reg, lowest, highest, fs); +} + +static int ltc4282_power_reset_hist(struct ltc4282_state *st) +{ + int ret; + + guard(mutex)(&st->lock); + + ret = ltc4282_write_power_word(st, LTC4282_POWER_LOWEST, + st->power_max); + if (ret) + return ret; + + ret = ltc4282_write_power_word(st, LTC4282_POWER_HIGHEST, 0); + if (ret) + return ret; + + /* now, let's also clear possible power_bad fault logs */ + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_POWER_BAD_FAULT_MASK); +} + +static int ltc4282_write_power(struct ltc4282_state *st, u32 attr, + long val) +{ + switch (attr) { + case hwmon_power_max: + return ltc4282_write_power_byte(st, LTC4282_POWER_MAX, val); + case hwmon_power_min: + return ltc4282_write_power_byte(st, LTC4282_POWER_MIN, val); + case hwmon_power_reset_history: + return ltc4282_power_reset_hist(st); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_write_voltage_byte_cached(const struct ltc4282_state *st, + u32 reg, u32 fs, long val, + u32 *cache_raw) +{ + u32 in; + + val = clamp_val(val, 0, fs); + in = DIV_ROUND_CLOSEST(val * U8_MAX, fs); + + if (cache_raw) { + *cache_raw = in; + return 0; + } + + return regmap_write(st->map, reg, in); +} + +static int ltc4282_write_voltage_byte(const struct ltc4282_state *st, u32 reg, + u32 fs, long val) +{ + return ltc4282_write_voltage_byte_cached(st, reg, fs, val, NULL); +} + +static int ltc4282_cache_history(struct ltc4282_state *st, u32 channel) +{ + long val; + int ret; + + ret = ltc4282_read_voltage_word(st, LTC4282_VSOURCE_LOWEST, st->vfs_out, + &val); + if (ret) + return ret; + + st->in0_1_cache[channel].in_lowest = val; + + ret = ltc4282_read_voltage_word(st, LTC4282_VSOURCE_HIGHEST, + st->vfs_out, &val); + if (ret) + return ret; + + st->in0_1_cache[channel].in_highest = val; + + ret = regmap_read(st->map, LTC4282_VSOURCE_MIN, + &st->in0_1_cache[channel].in_min_raw); + if (ret) + return ret; + + return regmap_read(st->map, LTC4282_VSOURCE_MAX, + &st->in0_1_cache[channel].in_max_raw); +} + +static int ltc4282_cache_sync(struct ltc4282_state *st, u32 channel) +{ + int ret; + + ret = __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + st->in0_1_cache[channel].in_lowest, + st->in0_1_cache[channel].in_highest, + st->vfs_out); + if (ret) + return ret; + + ret = regmap_write(st->map, LTC4282_VSOURCE_MIN, + st->in0_1_cache[channel].in_min_raw); + if (ret) + return ret; + + return regmap_write(st->map, LTC4282_VSOURCE_MAX, + st->in0_1_cache[channel].in_max_raw); +} + +static int ltc4282_vdd_source_write_lim(struct ltc4282_state *st, u32 reg, + int channel, u32 *cache, long val) +{ + int ret; + + guard(mutex)(&st->lock); + if (st->in0_1_cache[channel].en) + ret = ltc4282_write_voltage_byte(st, reg, st->vfs_out, val); + else + ret = ltc4282_write_voltage_byte_cached(st, reg, st->vfs_out, + val, cache); + + return ret; +} + +static int ltc4282_vdd_source_reset_hist(struct ltc4282_state *st, int channel) +{ + long lowest = st->vfs_out; + int ret; + + if (channel == LTC4282_CHAN_VDD) + lowest = st->vdd; + + guard(mutex)(&st->lock); + if (st->in0_1_cache[channel].en) { + ret = __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + lowest, 0, st->vfs_out); + if (ret) + return ret; + } + + st->in0_1_cache[channel].in_lowest = lowest; + st->in0_1_cache[channel].in_highest = 0; + + /* + * We are also clearing possible fault logs in reset_history. Clearing + * the logs might be important when the auto retry bits are not enabled + * as the chip only enables the output again after having these logs + * cleared. As some of these logs are related to limits, it makes sense + * to clear them in here. For VDD, we need to clear under/over voltage + * events. For VSOURCE, fet_short and fet_bad... + */ + if (channel == LTC4282_CHAN_VSOURCE) + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_FET_FAILURE_FAULT_MASK); + + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_VDD_FAULT_MASK); +} + +/* + * We need to mux between VSOURCE and VDD which means they are mutually + * exclusive. Moreover, we can't really disable both VDD and VSOURCE as the ADC + * is continuously running (we cannot independently halt it without also + * stopping VGPIO). Hence, the logic is that disabling or enabling VDD will + * automatically have the reverse effect on VSOURCE and vice-versa. + */ +static int ltc4282_vdd_source_enable(struct ltc4282_state *st, int channel, + long val) +{ + int ret, other_chan = ~channel & 0x1; + u8 __val = val; + + guard(mutex)(&st->lock); + if (st->in0_1_cache[channel].en == !!val) + return 0; + + /* clearing the bit makes the ADC to monitor VDD */ + if (channel == LTC4282_CHAN_VDD) + __val = !__val; + + ret = regmap_update_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_VDD_MONITOR_MASK, + FIELD_PREP(LTC4282_VDD_MONITOR_MASK, !!__val)); + if (ret) + return ret; + + st->in0_1_cache[channel].en = !!val; + st->in0_1_cache[other_chan].en = !val; + + if (st->in0_1_cache[channel].en) { + /* + * Then, we are disabling @other_chan. Let's save it's current + * history. + */ + ret = ltc4282_cache_history(st, other_chan); + if (ret) + return ret; + + return ltc4282_cache_sync(st, channel); + } + /* + * Then, we are enabling @other_chan. We need to do the opposite from + * above. + */ + ret = ltc4282_cache_history(st, channel); + if (ret) + return ret; + + return ltc4282_cache_sync(st, other_chan); +} + +static int ltc4282_write_in(struct ltc4282_state *st, u32 attr, long val, + int channel) +{ + switch (attr) { + case hwmon_in_max: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_write_voltage_byte(st, LTC4282_VGPIO_MAX, + 1280, val); + + return ltc4282_vdd_source_write_lim(st, LTC4282_VSOURCE_MAX, + channel, + &st->in0_1_cache[channel].in_max_raw, val); + case hwmon_in_min: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_write_voltage_byte(st, LTC4282_VGPIO_MIN, + 1280, val); + + return ltc4282_vdd_source_write_lim(st, LTC4282_VSOURCE_MIN, + channel, + &st->in0_1_cache[channel].in_min_raw, val); + case hwmon_in_reset_history: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_in_write_history(st, + LTC4282_VGPIO_LOWEST, + 1280, 0, 1280); + + return ltc4282_vdd_source_reset_hist(st, channel); + case hwmon_in_enable: + return ltc4282_vdd_source_enable(st, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_curr_reset_hist(struct ltc4282_state *st) +{ + int ret; + + guard(mutex)(&st->lock); + + ret = __ltc4282_in_write_history(st, LTC4282_VSENSE_LOWEST, + st->vsense_max, 0, 40 * MILLI); + if (ret) + return ret; + + /* now, let's also clear possible overcurrent fault logs */ + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_OC_FAULT_MASK); +} + +static int ltc4282_write_curr(struct ltc4282_state *st, u32 attr, + long val) +{ + /* need to pass it in millivolt */ + u32 in = DIV_ROUND_CLOSEST_ULL((u64)val * st->rsense, DECA * MICRO); + + switch (attr) { + case hwmon_curr_max: + return ltc4282_write_voltage_byte(st, LTC4282_VSENSE_MAX, 40, + in); + case hwmon_curr_min: + return ltc4282_write_voltage_byte(st, LTC4282_VSENSE_MIN, 40, + in); + case hwmon_curr_reset_history: + return ltc4282_curr_reset_hist(st); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_energy_enable_set(struct ltc4282_state *st, long val) +{ + int ret; + + guard(mutex)(&st->lock); + /* setting the bit halts the meter */ + ret = regmap_update_bits(st->map, LTC4282_ADC_CTRL, + LTC4282_METER_HALT_MASK, + FIELD_PREP(LTC4282_METER_HALT_MASK, !val)); + if (ret) + return ret; + + st->energy_en = !!val; + + return 0; +} + +static int ltc4282_write(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + struct ltc4282_state *st = dev_get_drvdata(dev); + + switch (type) { + case hwmon_power: + return ltc4282_write_power(st, attr, val); + case hwmon_in: + return ltc4282_write_in(st, attr, val, channel); + case hwmon_curr: + return ltc4282_write_curr(st, attr, val); + case hwmon_energy: + return ltc4282_energy_enable_set(st, val); + default: + return -EOPNOTSUPP; + } +} + +static umode_t ltc4282_in_is_visible(const struct ltc4282_state *st, u32 attr) +{ + switch (attr) { + case hwmon_in_input: + case hwmon_in_highest: + case hwmon_in_lowest: + case hwmon_in_max_alarm: + case hwmon_in_min_alarm: + case hwmon_in_label: + case hwmon_in_lcrit_alarm: + case hwmon_in_crit_alarm: + case hwmon_in_fault: + return 0444; + case hwmon_in_max: + case hwmon_in_min: + case hwmon_in_enable: + case hwmon_in_reset_history: + return 0644; + default: + return 0; + } +} + +static umode_t ltc4282_curr_is_visible(u32 attr) +{ + switch (attr) { + case hwmon_curr_input: + case hwmon_curr_highest: + case hwmon_curr_lowest: + case hwmon_curr_max_alarm: + case hwmon_curr_min_alarm: + case hwmon_curr_crit_alarm: + case hwmon_curr_label: + return 0444; + case hwmon_curr_max: + case hwmon_curr_min: + case hwmon_curr_reset_history: + return 0644; + default: + return 0; + } +} + +static umode_t ltc4282_power_is_visible(u32 attr) +{ + switch (attr) { + case hwmon_power_input: + case hwmon_power_input_highest: + case hwmon_power_input_lowest: + case hwmon_power_label: + case hwmon_power_max_alarm: + case hwmon_power_min_alarm: + return 0444; + case hwmon_power_max: + case hwmon_power_min: + case hwmon_power_reset_history: + return 0644; + default: + return 0; + } +} + +static umode_t ltc4282_is_visible(const void *data, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + switch (type) { + case hwmon_in: + return ltc4282_in_is_visible(data, attr); + case hwmon_curr: + return ltc4282_curr_is_visible(attr); + case hwmon_power: + return ltc4282_power_is_visible(attr); + case hwmon_energy: + /* hwmon_energy_enable */ + return 0644; + default: + return 0; + } +} + +static const char * const ltc4282_in_strs[] = { + "VSOURCE", "VDD", "VGPIO" +}; + +static int ltc4282_read_labels(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + switch (type) { + case hwmon_in: + *str = ltc4282_in_strs[channel]; + return 0; + case hwmon_curr: + *str = "ISENSE"; + return 0; + case hwmon_power: + *str = "Power"; + return 0; + default: + return -EOPNOTSUPP; + } +} + +static ssize_t ltc4282_energy_show(struct device *dev, + struct device_attribute *da, char *buf) +{ + struct ltc4282_state *st = dev_get_drvdata(dev); + u64 energy; + int ret; + + guard(mutex)(&st->lock); + if (!st->energy_en) + return -ENODATA; + + ret = ltc4282_read_energy(st, &energy); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%llu\n", energy); +} + +static const struct clk_ops ltc4282_ops = { + .recalc_rate = ltc4282_recalc_rate, + .round_rate = ltc4282_round_rate, + .set_rate = ltc4282_set_rate, + .disable = ltc4282_disable, +}; + +static int ltc428_clk_provider_setup(struct ltc4282_state *st, + struct device *dev) +{ + struct clk_init_data init; + int ret; + + if (!IS_ENABLED(CONFIG_COMMON_CLK)) + return 0; + + init.name = devm_kasprintf(dev, GFP_KERNEL, "%s-clk", + fwnode_get_name(dev_fwnode(dev))); + if (!init.name) + return -ENOMEM; + + init.ops = <c4282_ops; + init.flags = CLK_GET_RATE_NOCACHE; + st->clk_hw.init = &init; + + ret = devm_clk_hw_register(dev, &st->clk_hw); + if (ret) + return ret; + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &st->clk_hw); +} + +static int ltc428_clks_setup(struct ltc4282_state *st, struct device *dev) +{ + unsigned long rate; + struct clk *clkin; + u32 val; + int ret; + + ret = ltc428_clk_provider_setup(st, dev); + if (ret) + return ret; + + clkin = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(clkin)) + return dev_err_probe(dev, PTR_ERR(clkin), + "Failed to get clkin"); + if (!clkin) + return 0; + + rate = clk_get_rate(clkin); + if (!in_range(rate, LTC4282_CLKIN_MIN, LTC4282_CLKIN_RANGE)) + return dev_err_probe(dev, -EINVAL, + "Invalid clkin range(%lu) [%lu %lu]\n", + rate, LTC4282_CLKIN_MIN, + LTC4282_CLKIN_MAX); + + /* + * Clocks faster than 250KHZ should be reduced to 250KHZ. The clock + * frequency is divided by twice the value in the register. + */ + val = rate / (2 * LTC4282_CLKIN_MIN); + + return regmap_update_bits(st->map, LTC4282_CLK_DIV, + LTC4282_CLK_DIV_MASK, + FIELD_PREP(LTC4282_CLK_DIV_MASK, val)); +} + +static const int ltc4282_curr_lim_uv[] = { + 12500, 15625, 18750, 21875, 25000, 28125, 31250, 34375 +}; + +static int ltc4282_get_defaults(struct ltc4282_state *st, u32 *vin_mode) +{ + u32 reg_val, ilm_adjust; + int ret; + + ret = regmap_read(st->map, LTC4282_ADC_CTRL, ®_val); + if (ret) + return ret; + + st->energy_en = !FIELD_GET(LTC4282_METER_HALT_MASK, reg_val); + + ret = regmap_read(st->map, LTC4282_CTRL_MSB, ®_val); + if (ret) + return ret; + + *vin_mode = FIELD_GET(LTC4282_CTRL_VIN_MODE_MASK, reg_val); + + ret = regmap_read(st->map, LTC4282_ILIM_ADJUST, ®_val); + if (ret) + return ret; + + ilm_adjust = FIELD_GET(LTC4282_ILIM_ADJUST_MASK, reg_val); + st->vsense_max = ltc4282_curr_lim_uv[ilm_adjust]; + + st->in0_1_cache[LTC4282_CHAN_VSOURCE].en = FIELD_GET(LTC4282_VDD_MONITOR_MASK, + ilm_adjust); + if (!st->in0_1_cache[LTC4282_CHAN_VSOURCE].en) { + st->in0_1_cache[LTC4282_CHAN_VDD].en = true; + return regmap_read(st->map, LTC4282_VSOURCE_MAX, + &st->in0_1_cache[LTC4282_CHAN_VSOURCE].in_max_raw); + } + + return regmap_read(st->map, LTC4282_VSOURCE_MAX, + &st->in0_1_cache[LTC4282_CHAN_VDD].in_max_raw); +} + +/* + * Set max limits for ISENSE and Power as that depends on the max voltage on + * rsense that is defined in ILIM_ADJUST. This is specially important for power + * because for some rsense and vfsout values, if we allow the default raw 255 + * value, that would overflow long in 32bit archs when reading back the max + * power limit. + * + * Also set meaningful historic values for VDD and VSOURCE + * (0 would not mean much). + */ +static int ltc4282_set_max_limits(struct ltc4282_state *st) +{ + int ret; + + ret = ltc4282_write_voltage_byte(st, LTC4282_VSENSE_MAX, 40 * MILLI, + st->vsense_max); + if (ret) + return ret; + + /* Power is given by ISENSE * Vout. */ + st->power_max = DIV_ROUND_CLOSEST(st->vsense_max * DECA * MILLI, st->rsense) * st->vfs_out; + ret = ltc4282_write_power_byte(st, LTC4282_POWER_MAX, st->power_max); + if (ret) + return ret; + + if (st->in0_1_cache[LTC4282_CHAN_VDD].en) { + st->in0_1_cache[LTC4282_CHAN_VSOURCE].in_lowest = st->vfs_out; + return __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + st->vdd, 0, st->vfs_out); + } + + st->in0_1_cache[LTC4282_CHAN_VDD].in_lowest = st->vdd; + return __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + st->vfs_out, 0, st->vfs_out); +} + +static const char * const ltc4282_gpio1_modes[] = { + "power_bad", "power_good" +}; + +static const char * const ltc4282_gpio2_modes[] = { + "adc_input", "stress_fet" +}; + +static int ltc4282_gpio_setup(struct ltc4282_state *st, struct device *dev) +{ + const char *func = NULL; + int ret; + + ret = device_property_read_string(dev, "adi,gpio1-mode", &func); + if (!ret) { + ret = match_string(ltc4282_gpio1_modes, + ARRAY_SIZE(ltc4282_gpio1_modes), func); + if (ret < 0) + return dev_err_probe(dev, ret, + "Invalid func(%s) for gpio1\n", + func); + + ret = regmap_update_bits(st->map, LTC4282_GPIO_CONFIG, + LTC4282_GPIO_1_CONFIG_MASK, + FIELD_PREP(LTC4282_GPIO_1_CONFIG_MASK, ret)); + if (ret) + return ret; + } + + ret = device_property_read_string(dev, "adi,gpio2-mode", &func); + if (!ret) { + ret = match_string(ltc4282_gpio2_modes, + ARRAY_SIZE(ltc4282_gpio2_modes), func); + if (ret < 0) + return dev_err_probe(dev, ret, + "Invalid func(%s) for gpio2\n", + func); + if (!ret) { + /* setting the bit to 1 so the ADC to monitors GPIO2 */ + ret = regmap_set_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_GPIO_MODE_MASK); + } else { + ret = regmap_update_bits(st->map, LTC4282_GPIO_CONFIG, + LTC4282_GPIO_2_FET_STRESS_MASK, + FIELD_PREP(LTC4282_GPIO_2_FET_STRESS_MASK, 1)); + } + + if (ret) + return ret; + } + + if (!device_property_read_bool(dev, "adi,gpio3-monitor-enable")) + return 0; + + if (func && !strcmp(func, "adc_input")) + return dev_err_probe(dev, -EINVAL, + "Cannot have both gpio2 and gpio3 muxed into the ADC"); + + return regmap_clear_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_GPIO_MODE_MASK); +} + +static const char * const ltc4282_dividers[] = { + "external", "vdd_5_percent", "vdd_10_percent", "vdd_15_percent" +}; + +/* This maps the Vout full scale for the given Vin mode */ +static const u16 ltc4282_vfs_milli[] = { 5540, 8320, 16640, 33280 }; + +static const u16 ltc4282_vdd_milli[] = { 3300, 5000, 12000, 24000 }; + +enum { + LTC4282_VIN_3_3V, + LTC4282_VIN_5V, + LTC4282_VIN_12V, + LTC4282_VIN_24V, +}; + +static int ltc4282_setup(struct ltc4282_state *st, struct device *dev) +{ + const char *divider; + u32 val, vin_mode; + int ret; + + /* The part has an eeprom so let's get the needed defaults from it */ + ret = ltc4282_get_defaults(st, &vin_mode); + if (ret) + return ret; + + ret = device_property_read_u32(dev, "adi,rsense-nano-ohms", + &st->rsense); + if (ret) + return dev_err_probe(dev, ret, + "Failed to read adi,rsense-nano-ohms\n"); + if (st->rsense < CENTI) + return dev_err_probe(dev, -EINVAL, + "adi,rsense-nano-ohms too small (< %lu)\n", + CENTI); + + /* + * The resolution for rsense is tenths of micro (eg: 62.5 uOhm) which + * means we need nano in the bindings. However, to make things easier to + * handle (with respect to overflows) we divide it by 100 as we don't + * really need the last two digits. + */ + st->rsense /= CENTI; + + val = vin_mode; + ret = device_property_read_u32(dev, "adi,vin-mode-microvolt", &val); + if (!ret) { + switch (val) { + case 3300000: + val = LTC4282_VIN_3_3V; + break; + case 5000000: + val = LTC4282_VIN_5V; + break; + case 12000000: + val = LTC4282_VIN_12V; + break; + case 24000000: + val = LTC4282_VIN_24V; + break; + default: + return dev_err_probe(dev, -EINVAL, + "Invalid val(%u) for vin-mode-microvolt\n", + val); + } + + ret = regmap_update_bits(st->map, LTC4282_CTRL_MSB, + LTC4282_CTRL_VIN_MODE_MASK, + FIELD_PREP(LTC4282_CTRL_VIN_MODE_MASK, val)); + if (ret) + return ret; + + /* Foldback mode should also be set to the input voltage */ + ret = regmap_update_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_FOLDBACK_MODE_MASK, + FIELD_PREP(LTC4282_FOLDBACK_MODE_MASK, val)); + if (ret) + return ret; + } + + st->vfs_out = ltc4282_vfs_milli[val]; + st->vdd = ltc4282_vdd_milli[val]; + + ret = device_property_read_u32(dev, "adi,current-limit-sense-microvolt", + &st->vsense_max); + if (!ret) { + int reg_val; + + switch (val) { + case 12500: + reg_val = 0; + break; + case 15625: + reg_val = 1; + break; + case 18750: + reg_val = 2; + break; + case 21875: + reg_val = 3; + break; + case 25000: + reg_val = 4; + break; + case 28125: + reg_val = 5; + break; + case 31250: + reg_val = 6; + break; + case 34375: + reg_val = 7; + break; + default: + return dev_err_probe(dev, -EINVAL, + "Invalid val(%u) for adi,current-limit-microvolt\n", + st->vsense_max); + } + + ret = regmap_update_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_ILIM_ADJUST_MASK, + FIELD_PREP(LTC4282_ILIM_ADJUST_MASK, reg_val)); + if (ret) + return ret; + } + + ret = ltc4282_set_max_limits(st); + if (ret) + return ret; + + ret = device_property_read_string(dev, "adi,overvoltage-dividers", + ÷r); + if (!ret) { + int div = match_string(ltc4282_dividers, + ARRAY_SIZE(ltc4282_dividers), divider); + if (div < 0) + return dev_err_probe(dev, -EINVAL, + "Invalid val(%s) for adi,overvoltage-divider\n", + divider); + + ret = regmap_update_bits(st->map, LTC4282_CTRL_MSB, + LTC4282_CTRL_OV_MODE_MASK, + FIELD_PREP(LTC4282_CTRL_OV_MODE_MASK, div)); + } + + ret = device_property_read_string(dev, "adi,undervoltage-dividers", + ÷r); + if (!ret) { + int div = match_string(ltc4282_dividers, + ARRAY_SIZE(ltc4282_dividers), divider); + if (div < 0) + return dev_err_probe(dev, -EINVAL, + "Invalid val(%s) for adi,undervoltage-divider\n", + divider); + + ret = regmap_update_bits(st->map, LTC4282_CTRL_MSB, + LTC4282_CTRL_UV_MODE_MASK, + FIELD_PREP(LTC4282_CTRL_UV_MODE_MASK, div)); + } + + if (device_property_read_bool(dev, "adi,overcurrent-retry")) { + ret = regmap_set_bits(st->map, LTC4282_CTRL_LSB, + LTC4282_CTRL_OC_RETRY_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,overvoltage-retry-disable")) { + ret = regmap_clear_bits(st->map, LTC4282_CTRL_LSB, + LTC4282_CTRL_OV_RETRY_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,undervoltage-retry-disable")) { + ret = regmap_clear_bits(st->map, LTC4282_CTRL_LSB, + LTC4282_CTRL_UV_RETRY_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,fault-log-enable")) { + ret = regmap_set_bits(st->map, LTC4282_ADC_CTRL, + LTC4282_FAULT_LOG_EN_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,fault-log-enable")) { + ret = regmap_set_bits(st->map, LTC4282_ADC_CTRL, LTC4282_FAULT_LOG_EN_MASK); + if (ret) + return ret; + } + + ret = device_property_read_u32(dev, "adi,fet-bad-timeout-ms", &val); + if (!ret) { + if (val > LTC4282_FET_BAD_MAX_TIMEOUT) + return dev_err_probe(dev, -EINVAL, + "Invalid value(%u) for adi,fet-bad-timeout-ms", + val); + + ret = regmap_write(st->map, LTC4282_FET_BAD_FAULT_TIMEOUT, val); + if (ret) + return ret; + } + + return ltc4282_gpio_setup(st, dev); +} + +static bool ltc4282_readable_reg(struct device *dev, unsigned int reg) +{ + if (reg == LTC4282_RESERVED_1 || reg == LTC4282_RESERVED_2) + return false; + + return true; +} + +static bool ltc4282_writable_reg(struct device *dev, unsigned int reg) +{ + if (reg == LTC4282_STATUS_LSB || reg == LTC4282_STATUS_MSB) + return false; + if (reg == LTC4282_RESERVED_1 || reg == LTC4282_RESERVED_2) + return false; + + return true; +} + +static const struct regmap_config ltc4282_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = LTC4282_RESERVED_3, + .readable_reg = ltc4282_readable_reg, + .writeable_reg = ltc4282_writable_reg, +}; + +static const struct hwmon_channel_info * const ltc4282_info[] = { + HWMON_CHANNEL_INFO(in, + HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST | + HWMON_I_MAX | HWMON_I_MIN | HWMON_I_MIN_ALARM | + HWMON_I_MAX_ALARM | HWMON_I_ENABLE | + HWMON_I_RESET_HISTORY | HWMON_I_FAULT | + HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST | + HWMON_I_MAX | HWMON_I_MIN | HWMON_I_MIN_ALARM | + HWMON_I_MAX_ALARM | HWMON_I_LCRIT_ALARM | + HWMON_I_CRIT_ALARM | HWMON_I_ENABLE | + HWMON_I_RESET_HISTORY | HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST | + HWMON_I_MAX | HWMON_I_MIN | HWMON_I_MIN_ALARM | + HWMON_I_RESET_HISTORY | HWMON_I_MAX_ALARM | + HWMON_I_LABEL), + HWMON_CHANNEL_INFO(curr, + HWMON_C_INPUT | HWMON_C_LOWEST | HWMON_C_HIGHEST | + HWMON_C_MAX | HWMON_C_MIN | HWMON_C_MIN_ALARM | + HWMON_C_MAX_ALARM | HWMON_C_CRIT_ALARM | + HWMON_C_RESET_HISTORY | HWMON_C_LABEL), + HWMON_CHANNEL_INFO(power, + HWMON_P_INPUT | HWMON_P_INPUT_LOWEST | + HWMON_P_INPUT_HIGHEST | HWMON_P_MAX | HWMON_P_MIN | + HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM | + HWMON_P_RESET_HISTORY | HWMON_P_LABEL), + HWMON_CHANNEL_INFO(energy, + HWMON_E_ENABLE), + NULL +}; + +static const struct hwmon_ops ltc4282_hwmon_ops = { + .read = ltc4282_read, + .write = ltc4282_write, + .is_visible = ltc4282_is_visible, + .read_string = ltc4282_read_labels, +}; + +static const struct hwmon_chip_info ltc2947_chip_info = { + .ops = <c4282_hwmon_ops, + .info = ltc4282_info, +}; + +/* energy attributes are 6bytes wide so we need u64 */ +static SENSOR_DEVICE_ATTR_RO(energy1_input, ltc4282_energy, 0); + +static struct attribute *ltc4282_attrs[] = { + &sensor_dev_attr_energy1_input.dev_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(ltc4282); + +static int ltc4282_show_fault_log(void *arg, u64 *val, u32 mask) +{ + struct ltc4282_state *st = arg; + long alarm; + int ret; + + ret = ltc4282_read_alarm(st, LTC4282_FAULT_LOG, mask, &alarm); + if (ret) + return ret; + + *val = alarm; + + return 0; +} + +static int ltc4282_show_curr1_crit_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_OC_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_curr1_crit_fault_log, + ltc4282_show_curr1_crit_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_in1_lcrit_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_UV_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_in1_lcrit_fault_log, + ltc4282_show_in1_lcrit_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_in1_crit_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_OV_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_in1_crit_fault_log, + ltc4282_show_in1_crit_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_fet_bad_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_FET_BAD_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_fet_bad_fault_log, + ltc4282_show_fet_bad_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_fet_short_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_FET_SHORT_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_fet_short_fault_log, + ltc4282_show_fet_short_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_power1_bad_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_POWER_BAD_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_power1_bad_fault_log, + ltc4282_show_power1_bad_fault_log, NULL, "%llu\n"); + +static void ltc4282_debugfs_remove(void *dir) +{ + debugfs_remove_recursive(dir); +} + +static void ltc4282_debugfs_init(struct ltc4282_state *st, + struct i2c_client *i2c, + const struct device *hwmon) +{ + const char *debugfs_name; + struct dentry *dentry; + int ret; + + if (!IS_ENABLED(CONFIG_DEBUG_FS)) + return; + + debugfs_name = devm_kasprintf(&i2c->dev, GFP_KERNEL, "ltc4282-%s", + dev_name(hwmon)); + if (!debugfs_name) + return; + + dentry = debugfs_create_dir(debugfs_name, NULL); + if (IS_ERR(dentry)) + return; + + ret = devm_add_action_or_reset(&i2c->dev, ltc4282_debugfs_remove, + dentry); + if (ret) + return; + + debugfs_create_file_unsafe("power1_bad_fault_log", 0400, dentry, st, + <c4282_power1_bad_fault_log); + debugfs_create_file_unsafe("in0_fet_short_fault_log", 0400, dentry, st, + <c4282_fet_short_fault_log); + debugfs_create_file_unsafe("in0_fet_bad_fault_log", 0400, dentry, st, + <c4282_fet_bad_fault_log); + debugfs_create_file_unsafe("in1_crit_fault_log", 0400, dentry, st, + <c4282_in1_crit_fault_log); + debugfs_create_file_unsafe("in1_lcrit_fault_log", 0400, dentry, st, + <c4282_in1_lcrit_fault_log); + debugfs_create_file_unsafe("curr1_crit_fault_log", 0400, dentry, st, + <c4282_curr1_crit_fault_log); +} + +static int ltc4282_probe(struct i2c_client *i2c) +{ + struct device *dev = &i2c->dev, *hwmon; + struct ltc4282_state *st; + int ret; + + st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL); + if (!st) + return dev_err_probe(dev, -ENOMEM, + "Failed to allocate memory\n"); + + st->map = devm_regmap_init_i2c(i2c, <c4282_regmap_config); + if (IS_ERR(st->map)) + return dev_err_probe(dev, PTR_ERR(st->map), + "failed regmap init\n"); + + /* Soft reset */ + ret = regmap_set_bits(st->map, LTC4282_ADC_CTRL, LTC4282_RESET_MASK); + if (ret) + return ret; + + /* Yes, it's big but it is as specified in the datasheet */ + msleep(3200); + + ret = ltc428_clks_setup(st, dev); + if (ret) + return ret; + + ret = ltc4282_setup(st, dev); + if (ret) + return ret; + + mutex_init(&st->lock); + hwmon = devm_hwmon_device_register_with_info(dev, "ltc4282", st, + <c2947_chip_info, + ltc4282_groups); + if (IS_ERR(hwmon)) + return PTR_ERR(hwmon); + + ltc4282_debugfs_init(st, i2c, hwmon); + + return 0; +} + +static const struct of_device_id ltc4282_of_match[] = { + { .compatible = "adi,ltc4282" }, + {} +}; +MODULE_DEVICE_TABLE(of, ltc4282_of_match); + +static struct i2c_driver ltc4282_driver = { + .driver = { + .name = "ltc4282", + .of_match_table = ltc4282_of_match, + }, + .probe = ltc4282_probe, +}; +module_i2c_driver(ltc4282_driver); + +MODULE_AUTHOR("Nuno Sa "); +MODULE_DESCRIPTION("LTC4282 I2C High Current Hot Swap Controller"); +MODULE_LICENSE("GPL"); From d612bf839f618b090672356889e3bbdac67aebd5 Mon Sep 17 00:00:00 2001 From: Ivor Wanders Date: Tue, 30 Jan 2024 19:58:55 -0500 Subject: [PATCH 0732/1038] hwmon: add fan speed monitoring driver for Surface devices Adds a driver that provides read only access to the fan speed for Microsoft Surface Pro devices. The fan speed is always regulated by the EC and cannot be influenced directly. Signed-off-by: Ivor Wanders Link: https://github.com/linux-surface/kernel/pull/144 Reviewed-by: Maximilian Luz Reviewed-by: Armin Wolf Link: https://lore.kernel.org/r/20240131005856.10180-2-ivor@iwanders.net [groeck: - Declare surface_fan_hwmon_is_visible() static - Add dependency on SURFACE_AGGREGATOR_BUS ] Signed-off-by: Guenter Roeck --- Documentation/hwmon/index.rst | 1 + Documentation/hwmon/surface_fan.rst | 25 ++++++++ MAINTAINERS | 8 +++ drivers/hwmon/Kconfig | 14 +++++ drivers/hwmon/Makefile | 1 + drivers/hwmon/surface_fan.c | 91 +++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+) create mode 100644 Documentation/hwmon/surface_fan.rst create mode 100644 drivers/hwmon/surface_fan.c diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index f16c6dfaec7d..8f73badecba1 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -209,6 +209,7 @@ Hardware Monitoring Kernel Drivers smsc47m1 sparx5-temp stpddc60 + surface_fan sy7636a-hwmon tc654 tc74 diff --git a/Documentation/hwmon/surface_fan.rst b/Documentation/hwmon/surface_fan.rst new file mode 100644 index 000000000000..07942574c4f0 --- /dev/null +++ b/Documentation/hwmon/surface_fan.rst @@ -0,0 +1,25 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver surface_fan +========================= + +Supported Devices: + + * Microsoft Surface Pro 9 + +Author: Ivor Wanders + +Description +----------- + +This provides monitoring of the fan found in some Microsoft Surface Pro devices, +like the Surface Pro 9. The fan is always controlled by the onboard controller. + +Sysfs interface +--------------- + +======================= ======= ========================================= +Name Perm Description +======================= ======= ========================================= +``fan1_input`` RO Current fan speed in RPM. +======================= ======= ========================================= diff --git a/MAINTAINERS b/MAINTAINERS index df8ea2d94a0c..dd69696f3c0e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14549,6 +14549,14 @@ F: Documentation/driver-api/surface_aggregator/clients/dtx.rst F: drivers/platform/surface/surface_dtx.c F: include/uapi/linux/surface_aggregator/dtx.h +MICROSOFT SURFACE SENSOR FAN DRIVER +M: Maximilian Luz +M: Ivor Wanders +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/surface_fan.rst +F: drivers/hwmon/surface_fan.c + MICROSOFT SURFACE GPE LID SUPPORT DRIVER M: Maximilian Luz L: platform-driver-x86@vger.kernel.org diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index f6160cc70077..cdf228fe50e2 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -2005,6 +2005,20 @@ config SENSORS_SFCTEMP This driver can also be built as a module. If so, the module will be called sfctemp. +config SENSORS_SURFACE_FAN + tristate "Surface Fan Driver" + depends on SURFACE_AGGREGATOR + depends on SURFACE_AGGREGATOR_BUS + help + Driver that provides monitoring of the fan on Surface Pro devices that + have a fan, like the Surface Pro 9. + + This makes the fan's current speed accessible through the hwmon + system. It does not provide control over the fan, the firmware is + responsible for that, this driver merely provides monitoring. + + Select M or Y here, if you want to be able to read the fan's speed. + config SENSORS_ADC128D818 tristate "Texas Instruments ADC128D818" depends on I2C diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 8bfc422a29e5..a49704cd48a4 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -202,6 +202,7 @@ obj-$(CONFIG_SENSORS_SMSC47M1) += smsc47m1.o obj-$(CONFIG_SENSORS_SMSC47M192)+= smsc47m192.o obj-$(CONFIG_SENSORS_SPARX5) += sparx5-temp.o obj-$(CONFIG_SENSORS_STTS751) += stts751.o +obj-$(CONFIG_SENSORS_SURFACE_FAN)+= surface_fan.o obj-$(CONFIG_SENSORS_SY7636A) += sy7636a-hwmon.o obj-$(CONFIG_SENSORS_AMC6821) += amc6821.o obj-$(CONFIG_SENSORS_TC74) += tc74.o diff --git a/drivers/hwmon/surface_fan.c b/drivers/hwmon/surface_fan.c new file mode 100644 index 000000000000..de3c5a2409c6 --- /dev/null +++ b/drivers/hwmon/surface_fan.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Surface Fan driver for Surface System Aggregator Module. It provides access + * to the fan's rpm through the hwmon system. + * + * Copyright (C) 2023 Ivor Wanders + */ + +#include +#include +#include +#include +#include + +// SSAM +SSAM_DEFINE_SYNC_REQUEST_CL_R(__ssam_fan_rpm_get, __le16, { + .target_category = SSAM_SSH_TC_FAN, + .command_id = 0x01, +}); + +// hwmon +static umode_t surface_fan_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, u32 attr, + int channel) +{ + return 0444; +} + +static int surface_fan_hwmon_read(struct device *dev, + enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct ssam_device *sdev = dev_get_drvdata(dev); + int ret; + __le16 value; + + ret = __ssam_fan_rpm_get(sdev, &value); + if (ret) + return ret; + + *val = le16_to_cpu(value); + + return 0; +} + +static const struct hwmon_channel_info *const surface_fan_info[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT), + NULL +}; + +static const struct hwmon_ops surface_fan_hwmon_ops = { + .is_visible = surface_fan_hwmon_is_visible, + .read = surface_fan_hwmon_read, +}; + +static const struct hwmon_chip_info surface_fan_chip_info = { + .ops = &surface_fan_hwmon_ops, + .info = surface_fan_info, +}; + +static int surface_fan_probe(struct ssam_device *sdev) +{ + struct device *hdev; + + hdev = devm_hwmon_device_register_with_info(&sdev->dev, + "surface_fan", sdev, + &surface_fan_chip_info, + NULL); + + return PTR_ERR_OR_ZERO(hdev); +} + +static const struct ssam_device_id ssam_fan_match[] = { + { SSAM_SDEV(FAN, SAM, 0x01, 0x01) }, + {}, +}; +MODULE_DEVICE_TABLE(ssam, ssam_fan_match); + +static struct ssam_device_driver surface_fan = { + .probe = surface_fan_probe, + .match_table = ssam_fan_match, + .driver = { + .name = "surface_fan", + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +module_ssam_device_driver(surface_fan); + +MODULE_AUTHOR("Ivor Wanders "); +MODULE_DESCRIPTION("Fan Driver for Surface System Aggregator Module"); +MODULE_LICENSE("GPL"); From 7e6707f7da317eb98c12de6a477d5cfb5057705f Mon Sep 17 00:00:00 2001 From: Charles Hsu Date: Wed, 31 Jan 2024 15:48:21 +0800 Subject: [PATCH 0733/1038] dt-bindings: Add MPQ8785 voltage regulator device Monolithic Power Systems, Inc. (MPS) synchronous step-down converter. Signed-off-by: Charles Hsu Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240131074822.2962078-1-ythsu0511@gmail.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/trivial-devices.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 79dcd92c4a43..088b23ed2ae6 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -129,6 +129,8 @@ properties: - mps,mp2975 # Monolithic Power Systems Inc. multi-phase hot-swap controller mp5990 - mps,mp5990 + # Monolithic Power Systems Inc. synchronous step-down converter mpq8785 + - mps,mpq8785 # Honeywell Humidicon HIH-6130 humidity/temperature sensor - honeywell,hi6130 # IBM Common Form Factor Power Supply Versions (all versions) From f20b4a931130cb574c40563cfda0fc2cb944b4df Mon Sep 17 00:00:00 2001 From: Charles Hsu Date: Wed, 31 Jan 2024 15:48:22 +0800 Subject: [PATCH 0734/1038] hwmon: Add driver for MPS MPQ8785 Synchronous Step-Down Converter Add support for mpq8785 device from Monolithic Power Systems, Inc. (MPS) vendor. This is synchronous step-down controller. Signed-off-by: Charles Hsu Link: https://lore.kernel.org/r/20240131074822.2962078-2-ythsu0511@gmail.com [groeck: probe_new --> probe; add MODULE_IMPORT_NS(PMBUS)] Signed-off-by: Guenter Roeck --- Documentation/hwmon/index.rst | 1 + Documentation/hwmon/mpq8785.rst | 94 +++++++++++++++++++++++++++++++++ drivers/hwmon/pmbus/Kconfig | 9 ++++ drivers/hwmon/pmbus/Makefile | 1 + drivers/hwmon/pmbus/mpq8785.c | 90 +++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+) create mode 100644 Documentation/hwmon/mpq8785.rst create mode 100644 drivers/hwmon/pmbus/mpq8785.c diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index 8f73badecba1..9ac087dd3e78 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -164,6 +164,7 @@ Hardware Monitoring Kernel Drivers mp2975 mp5023 mp5990 + mpq8785 nct6683 nct6775 nct7802 diff --git a/Documentation/hwmon/mpq8785.rst b/Documentation/hwmon/mpq8785.rst new file mode 100644 index 000000000000..bf8176b87086 --- /dev/null +++ b/Documentation/hwmon/mpq8785.rst @@ -0,0 +1,94 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel driver mpq8785 +======================= + +Supported chips: + + * MPS MPQ8785 + + Prefix: 'mpq8785' + +Author: Charles Hsu + +Description +----------- + +The MPQ8785 is a fully integrated, PMBus-compatible, high-frequency, synchronous +buck converter. The MPQ8785 offers a very compact solution that achieves up to +40A output current per phase, with excellent load and line regulation over a +wide input supply range. The MPQ8785 operates at high efficiency over a wide +output current load range. + +The PMBus interface provides converter configurations and key parameters +monitoring. + +The MPQ8785 adopts MPS's proprietary multi-phase digital constant-on-time (MCOT) +control, which provides fast transient response and eases loop stabilization. +The MCOT scheme also allows multiple MPQ8785 devices to be connected in parallel +with excellent current sharing and phase interleaving for high-current +applications. + +Fully integrated protection features include over-current protection (OCP), +over-voltage protection (OVP), under-voltage protection (UVP), and +over-temperature protection (OTP). + +The MPQ8785 requires a minimal number of readily available, standard external +components, and is available in a TLGA (5mmx6mm) package. + +Device compliant with: + +- PMBus rev 1.3 interface. + +The driver exports the following attributes via the 'sysfs' files +for input voltage: + +**in1_input** + +**in1_label** + +**in1_max** + +**in1_max_alarm** + +**in1_min** + +**in1_min_alarm** + +**in1_crit** + +**in1_crit_alarm** + +The driver provides the following attributes for output voltage: + +**in2_input** + +**in2_label** + +**in2_alarm** + +The driver provides the following attributes for output current: + +**curr1_input** + +**curr1_label** + +**curr1_max** + +**curr1_max_alarm** + +**curr1_crit** + +**curr1_crit_alarm** + +The driver provides the following attributes for temperature: + +**temp1_input** + +**temp1_max** + +**temp1_max_alarm** + +**temp1_crit** + +**temp1_crit_alarm** diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig index 294808f5240a..557ae0c414b0 100644 --- a/drivers/hwmon/pmbus/Kconfig +++ b/drivers/hwmon/pmbus/Kconfig @@ -377,6 +377,15 @@ config SENSORS_MPQ7932 This driver can also be built as a module. If so, the module will be called mpq7932. +config SENSORS_MPQ8785 + tristate "MPS MPQ8785" + help + If you say yes here you get hardware monitoring functionality support + for power management IC MPS MPQ8785. + + This driver can also be built as a module. If so, the module will + be called mpq8785. + config SENSORS_PIM4328 tristate "Flex PIM4328 and compatibles" help diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile index cf8a76744545..f14ecf03ad77 100644 --- a/drivers/hwmon/pmbus/Makefile +++ b/drivers/hwmon/pmbus/Makefile @@ -39,6 +39,7 @@ obj-$(CONFIG_SENSORS_MP2975) += mp2975.o obj-$(CONFIG_SENSORS_MP5023) += mp5023.o obj-$(CONFIG_SENSORS_MP5990) += mp5990.o obj-$(CONFIG_SENSORS_MPQ7932) += mpq7932.o +obj-$(CONFIG_SENSORS_MPQ8785) += mpq8785.o obj-$(CONFIG_SENSORS_PLI1209BC) += pli1209bc.o obj-$(CONFIG_SENSORS_PM6764TR) += pm6764tr.o obj-$(CONFIG_SENSORS_PXE1610) += pxe1610.o diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c new file mode 100644 index 000000000000..4e2549cc8120 --- /dev/null +++ b/drivers/hwmon/pmbus/mpq8785.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for MPS MPQ8785 Step-Down Converter + */ + +#include +#include +#include +#include "pmbus.h" + +static int mpq8785_identify(struct i2c_client *client, + struct pmbus_driver_info *info) +{ + int vout_mode; + + vout_mode = pmbus_read_byte_data(client, 0, PMBUS_VOUT_MODE); + if (vout_mode < 0 || vout_mode == 0xff) + return vout_mode < 0 ? vout_mode : -ENODEV; + switch (vout_mode >> 5) { + case 0: + info->format[PSC_VOLTAGE_OUT] = linear; + break; + case 1: + case 2: + info->format[PSC_VOLTAGE_OUT] = direct, + info->m[PSC_VOLTAGE_OUT] = 64; + info->b[PSC_VOLTAGE_OUT] = 0; + info->R[PSC_VOLTAGE_OUT] = 1; + break; + default: + return -ENODEV; + } + + return 0; +}; + +static struct pmbus_driver_info mpq8785_info = { + .pages = 1, + .format[PSC_VOLTAGE_IN] = direct, + .format[PSC_CURRENT_OUT] = direct, + .format[PSC_TEMPERATURE] = direct, + .m[PSC_VOLTAGE_IN] = 4, + .b[PSC_VOLTAGE_IN] = 0, + .R[PSC_VOLTAGE_IN] = 1, + .m[PSC_CURRENT_OUT] = 16, + .b[PSC_CURRENT_OUT] = 0, + .R[PSC_CURRENT_OUT] = 0, + .m[PSC_TEMPERATURE] = 1, + .b[PSC_TEMPERATURE] = 0, + .R[PSC_TEMPERATURE] = 0, + .func[0] = + PMBUS_HAVE_VIN | PMBUS_HAVE_STATUS_INPUT | + PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT | + PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT | + PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP, + .identify = mpq8785_identify, +}; + +static int mpq8785_probe(struct i2c_client *client) +{ + return pmbus_do_probe(client, &mpq8785_info); +}; + +static const struct i2c_device_id mpq8785_id[] = { + { "mpq8785", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(i2c, mpq8785_id); + +static const struct of_device_id __maybe_unused mpq8785_of_match[] = { + { .compatible = "mps,mpq8785" }, + {} +}; +MODULE_DEVICE_TABLE(of, mpq8785_of_match); + +static struct i2c_driver mpq8785_driver = { + .driver = { + .name = "mpq8785", + .of_match_table = of_match_ptr(mpq8785_of_match), + }, + .probe = mpq8785_probe, + .id_table = mpq8785_id, +}; + +module_i2c_driver(mpq8785_driver); + +MODULE_AUTHOR("Charles Hsu "); +MODULE_DESCRIPTION("PMBus driver for MPS MPQ8785"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(PMBUS); From 9d613d9b8a9e54cc21379a40de29294adc1e6594 Mon Sep 17 00:00:00 2001 From: Stefan Gloor Date: Wed, 31 Jan 2024 12:15:12 +0100 Subject: [PATCH 0735/1038] hwmon: (sht3x) read out sensor serial number The temperature/humidity sensors of the STS3x/SHT3x family are calibrated and factory-programmed with a unique serial number. For some sensors, this serial number can be used to obtain a calibration certificate via an API provided by the manufacturer (Sensirion). Expose the serial number via debugfs. Tested with: 2x STS31, 1x STS32, 1x SHT31 Signed-off-by: Stefan Gloor Link: https://lore.kernel.org/r/20240131111512.25321-2-code@stefan-gloor.ch Signed-off-by: Guenter Roeck --- Documentation/hwmon/sht3x.rst | 11 ++++++ drivers/hwmon/sht3x.c | 66 ++++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/Documentation/hwmon/sht3x.rst b/Documentation/hwmon/sht3x.rst index 957c854f5d08..9585fa7c5a5d 100644 --- a/Documentation/hwmon/sht3x.rst +++ b/Documentation/hwmon/sht3x.rst @@ -65,6 +65,10 @@ When the temperature and humidity readings move back between the hysteresis values, the alert bit is set to 0 and the alert pin on the sensor is set to low. +The serial number exposed to debugfs allows for unique identification of the +sensors. For sts32, sts33 and sht33, the manufacturer provides calibration +certificates through an API. + sysfs-Interface --------------- @@ -99,3 +103,10 @@ repeatability: write or read repeatability, higher repeatability means - 1: medium repeatability - 2: high repeatability =================== ============================================================ + +debugfs-Interface +----------------- + +=================== ============================================================ +serial_number: unique serial number of the sensor in decimal +=================== ============================================================ diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c index 79657910b79e..c0d02fbcdb76 100644 --- a/drivers/hwmon/sht3x.c +++ b/drivers/hwmon/sht3x.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,9 @@ static const unsigned char sht3x_cmd_heater_off[] = { 0x30, 0x66 }; /* other commands */ static const unsigned char sht3x_cmd_read_status_reg[] = { 0xf3, 0x2d }; static const unsigned char sht3x_cmd_clear_status_reg[] = { 0x30, 0x41 }; +static const unsigned char sht3x_cmd_read_serial_number[] = { 0x37, 0x80 }; + +static struct dentry *debugfs; /* delays for single-shot mode i2c commands, both in us */ #define SHT3X_SINGLE_WAIT_TIME_HPM 15000 @@ -163,12 +167,14 @@ struct sht3x_data { enum sht3x_chips chip_id; struct mutex i2c_lock; /* lock for sending i2c commands */ struct mutex data_lock; /* lock for updating driver data */ + struct dentry *sensor_dir; u8 mode; const unsigned char *command; u32 wait_time; /* in us*/ unsigned long last_update; /* last update in periodic mode*/ enum sht3x_repeatability repeatability; + u32 serial_number; /* * cached values for temperature and humidity and limits @@ -831,6 +837,40 @@ static int sht3x_write(struct device *dev, enum hwmon_sensor_types type, } } +static void sht3x_debugfs_init(struct sht3x_data *data) +{ + char name[32]; + + snprintf(name, sizeof(name), "i2c%u-%02x", + data->client->adapter->nr, data->client->addr); + data->sensor_dir = debugfs_create_dir(name, debugfs); + debugfs_create_u32("serial_number", 0444, + data->sensor_dir, &data->serial_number); +} + +static void sht3x_debugfs_remove(void *sensor_dir) +{ + debugfs_remove_recursive(sensor_dir); +} + +static int sht3x_serial_number_read(struct sht3x_data *data) +{ + int ret; + char buffer[SHT3X_RESPONSE_LENGTH]; + struct i2c_client *client = data->client; + + ret = sht3x_read_from_command(client, data, + sht3x_cmd_read_serial_number, + buffer, + SHT3X_RESPONSE_LENGTH, 0); + if (ret) + return ret; + + data->serial_number = (buffer[0] << 24) | (buffer[1] << 16) | + (buffer[3] << 8) | buffer[4]; + return ret; +} + static const struct hwmon_ops sht3x_ops = { .is_visible = sht3x_is_visible, .read = sht3x_read, @@ -899,6 +939,18 @@ static int sht3x_probe(struct i2c_client *client) if (ret) return ret; + ret = sht3x_serial_number_read(data); + if (ret) { + dev_dbg(dev, "unable to read serial number\n"); + } else { + sht3x_debugfs_init(data); + ret = devm_add_action_or_reset(dev, + sht3x_debugfs_remove, + data->sensor_dir); + if (ret) + return ret; + } + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, data, @@ -917,7 +969,19 @@ static struct i2c_driver sht3x_i2c_driver = { .id_table = sht3x_ids, }; -module_i2c_driver(sht3x_i2c_driver); +static int __init sht3x_init(void) +{ + debugfs = debugfs_create_dir("sht3x", NULL); + return i2c_add_driver(&sht3x_i2c_driver); +} +module_init(sht3x_init); + +static void __exit sht3x_cleanup(void) +{ + debugfs_remove_recursive(debugfs); + i2c_del_driver(&sht3x_i2c_driver); +} +module_exit(sht3x_cleanup); MODULE_AUTHOR("David Frey "); MODULE_AUTHOR("Pascal Sachs "); From f5b75bde41f23de21d3e336e25b7c4deb47b4372 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:13:55 -0500 Subject: [PATCH 0736/1038] hwmon: (adt7x10) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202071355.40666-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/adt7310.c | 2 +- drivers/hwmon/adt7410.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/hwmon/adt7310.c b/drivers/hwmon/adt7310.c index 067865f4887a..25281739aa3b 100644 --- a/drivers/hwmon/adt7310.c +++ b/drivers/hwmon/adt7310.c @@ -124,7 +124,7 @@ static int adt7310_reg_write(void *context, unsigned int reg, unsigned int val) static const struct regmap_config adt7310_regmap_config = { .reg_bits = 8, .val_bits = 16, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = adt7310_regmap_is_volatile, .reg_read = adt7310_reg_read, .reg_write = adt7310_reg_write, diff --git a/drivers/hwmon/adt7410.c b/drivers/hwmon/adt7410.c index fd214d9b3a89..d15f64d4b6e7 100644 --- a/drivers/hwmon/adt7410.c +++ b/drivers/hwmon/adt7410.c @@ -69,7 +69,7 @@ static const struct regmap_config adt7410_regmap_config = { .reg_bits = 8, .val_bits = 16, .max_register = ADT7X10_ID, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = adt7410_regmap_is_volatile, .reg_read = adt7410_reg_read, .reg_write = adt7410_reg_write, From 9c440cf0c68fa30dc3072b8f1755f2d91863d7d6 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:14:52 -0500 Subject: [PATCH 0737/1038] hwmon: (emc1403) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202071452.40778-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/emc1403.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c index 1332e4ac078c..d370efd6f986 100644 --- a/drivers/hwmon/emc1403.c +++ b/drivers/hwmon/emc1403.c @@ -385,7 +385,7 @@ static bool emc1403_regmap_is_volatile(struct device *dev, unsigned int reg) static const struct regmap_config emc1403_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = emc1403_regmap_is_volatile, }; From ba468d4b0b1e81067405c9f68986828ca8ede876 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:15:38 -0500 Subject: [PATCH 0738/1038] hwmon: (ina3221) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202071538.40877-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/ina3221.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c index 5ffdc94db436..2c9530b6f192 100644 --- a/drivers/hwmon/ina3221.c +++ b/drivers/hwmon/ina3221.c @@ -762,7 +762,7 @@ static const struct regmap_config ina3221_regmap_config = { .reg_bits = 8, .val_bits = 16, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_table = &ina3221_volatile_table, }; From f8fec5f317d4aacdcea51de78fd622fad33cbba5 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:16:28 -0500 Subject: [PATCH 0739/1038] hwmon: (jc42) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202071628.40990-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/jc42.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c index f958e830b23c..75dc25df0f8b 100644 --- a/drivers/hwmon/jc42.c +++ b/drivers/hwmon/jc42.c @@ -497,7 +497,7 @@ static const struct regmap_config jc42_regmap_config = { .writeable_reg = jc42_writable_reg, .readable_reg = jc42_readable_reg, .volatile_reg = jc42_volatile_reg, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; static int jc42_probe(struct i2c_client *client) From 6c224da4d79fbb60d087423066c59e8bc8f0a44f Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:18:00 -0500 Subject: [PATCH 0740/1038] hwmon: (lm83) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202071800.41113-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/lm83.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/lm83.c b/drivers/hwmon/lm83.c index 5befedca6abb..b333c9bde4e6 100644 --- a/drivers/hwmon/lm83.c +++ b/drivers/hwmon/lm83.c @@ -165,7 +165,7 @@ static bool lm83_regmap_is_volatile(struct device *dev, unsigned int reg) static const struct regmap_config lm83_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = lm83_regmap_is_volatile, .reg_read = lm83_regmap_reg_read, .reg_write = lm83_regmap_reg_write, From 7a04f015d47535545a57ccf27cdc5390f17f10f0 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:19:27 -0500 Subject: [PATCH 0741/1038] hwmon: (max31760) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202071927.41213-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/max31760.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c index 1b6f71bc61cb..127e31ca3c87 100644 --- a/drivers/hwmon/max31760.c +++ b/drivers/hwmon/max31760.c @@ -60,7 +60,7 @@ static const struct regmap_config regmap_config = { .reg_bits = 8, .val_bits = 8, .max_register = 0x5B, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = max31760_volatile_reg, }; From 23c7029f376968dbc8d72fd55e60fe2e31c2629b Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:20:07 -0500 Subject: [PATCH 0742/1038] hwmon: (nct7802) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202072007.41316-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/nct7802.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c index 024cff151c36..a0e664d5ebfe 100644 --- a/drivers/hwmon/nct7802.c +++ b/drivers/hwmon/nct7802.c @@ -1051,7 +1051,7 @@ static bool nct7802_regmap_is_volatile(struct device *dev, unsigned int reg) static const struct regmap_config nct7802_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = nct7802_regmap_is_volatile, }; From 3d8e02bda1327d17363446bd7a3f536f244a5905 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:20:39 -0500 Subject: [PATCH 0743/1038] hwmon: (sch5627) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202072039.41419-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/sch5627.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/sch5627.c b/drivers/hwmon/sch5627.c index 1891d4d75aa9..33e997b5c1f5 100644 --- a/drivers/hwmon/sch5627.c +++ b/drivers/hwmon/sch5627.c @@ -116,7 +116,7 @@ static const struct regmap_config sch5627_regmap_config = { .val_bits = 8, .wr_table = &sch5627_tunables_table, .rd_table = &sch5627_tunables_table, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .use_single_read = true, .use_single_write = true, .can_sleep = true, From cac78418e2f54247ea1c141f0cfa1a3f7cbb8889 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 2 Feb 2024 02:22:35 -0500 Subject: [PATCH 0744/1038] hwmon: (tmp401) convert to use maple tree register cache The maple tree register cache is based on a much more modern data structure than the rbtree cache and makes optimisation choices which are probably more appropriate for modern systems than those made by the rbtree cache. Signed-off-by: Bo Liu Link: https://lore.kernel.org/r/20240202072235.41614-1-liubo03@inspur.com Signed-off-by: Guenter Roeck --- drivers/hwmon/tmp401.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c index 91f2314568cf..df1b45a62e80 100644 --- a/drivers/hwmon/tmp401.c +++ b/drivers/hwmon/tmp401.c @@ -256,7 +256,7 @@ static int tmp401_reg_write(void *context, unsigned int reg, unsigned int val) static const struct regmap_config tmp401_regmap_config = { .reg_bits = 8, .val_bits = 16, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = tmp401_regmap_is_volatile, .reg_read = tmp401_reg_read, .reg_write = tmp401_reg_write, From cc804e48fef1535150f7b300b81990230b8a10f0 Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jan 2024 22:06:44 +0100 Subject: [PATCH 0745/1038] dt-bindings: vendor-prefixes: add Amphenol Add vendor prefix for Amphenol (https://www.amphenol-sensors.com) Acked-by: Krzysztof Kozlowski Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240130-topic-chipcap2-v6-1-260bea05cf9b@gmail.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/vendor-prefixes.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 1a0dc04f1db4..25158559471c 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -107,6 +107,8 @@ patternProperties: description: Amlogic, Inc. "^ampere,.*": description: Ampere Computing LLC + "^amphenol,.*": + description: Amphenol Advanced Sensors "^ampire,.*": description: Ampire Co., Ltd. "^ams,.*": From 5f85c4d10ef46a5f457a9d337159ff620a0a2191 Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jan 2024 22:06:45 +0100 Subject: [PATCH 0746/1038] hwmon: (core) Add support for humidity min/max alarm Add min_alarm and max_alarm attributes for humidityX to support devices that can generate these alarms. Such attributes already exist for other magnitudes such as tempX. Tested with a ChipCap 2 temperature-humidity sensor. Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240130-topic-chipcap2-v6-2-260bea05cf9b@gmail.com Signed-off-by: Guenter Roeck --- drivers/hwmon/hwmon.c | 2 ++ include/linux/hwmon.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 18705049ad61..3b259c425ab7 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -587,6 +587,8 @@ static const char * const hwmon_humidity_attr_templates[] = { [hwmon_humidity_fault] = "humidity%d_fault", [hwmon_humidity_rated_min] = "humidity%d_rated_min", [hwmon_humidity_rated_max] = "humidity%d_rated_max", + [hwmon_humidity_min_alarm] = "humidity%d_min_alarm", + [hwmon_humidity_max_alarm] = "humidity%d_max_alarm", }; static const char * const hwmon_fan_attr_templates[] = { diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index c7885fdce88f..edf96f249eb5 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -295,6 +295,8 @@ enum hwmon_humidity_attributes { hwmon_humidity_fault, hwmon_humidity_rated_min, hwmon_humidity_rated_max, + hwmon_humidity_min_alarm, + hwmon_humidity_max_alarm, }; #define HWMON_H_ENABLE BIT(hwmon_humidity_enable) @@ -308,6 +310,8 @@ enum hwmon_humidity_attributes { #define HWMON_H_FAULT BIT(hwmon_humidity_fault) #define HWMON_H_RATED_MIN BIT(hwmon_humidity_rated_min) #define HWMON_H_RATED_MAX BIT(hwmon_humidity_rated_max) +#define HWMON_H_MIN_ALARM BIT(hwmon_humidity_min_alarm) +#define HWMON_H_MAX_ALARM BIT(hwmon_humidity_max_alarm) enum hwmon_fan_attributes { hwmon_fan_enable, From b86d76015376a89ff99acfcd783e9ae4f4b9360d Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jan 2024 22:06:46 +0100 Subject: [PATCH 0747/1038] ABI: sysfs-class-hwmon: add descriptions for humidity min/max alarms This attributes have been recently introduced and require the corresponding ABI documentation. Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240130-topic-chipcap2-v6-3-260bea05cf9b@gmail.com Signed-off-by: Guenter Roeck --- Documentation/ABI/testing/sysfs-class-hwmon | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon index 6c4e68ad4a83..cfd0d0bab483 100644 --- a/Documentation/ABI/testing/sysfs-class-hwmon +++ b/Documentation/ABI/testing/sysfs-class-hwmon @@ -977,6 +977,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/humidityY_max_alarm +Description: + Maximum humidity detection + + - 0: OK + - 1: Maximum humidity detected + + RO + What: /sys/class/hwmon/hwmonX/humidityY_max_hyst Description: Humidity hysteresis value for max limit. @@ -996,6 +1005,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/humidityY_min_alarm +Description: + Minimum humidity detection + + - 0: OK + - 1: Minimum humidity detected + + RO + What: /sys/class/hwmon/hwmonX/humidityY_min_hyst Description: Humidity hysteresis value for min limit. From 8f89ac2b9bdec95e4a05612a198b4255dce2ca8a Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jan 2024 22:06:47 +0100 Subject: [PATCH 0748/1038] dt-bindings: hwmon: Add Amphenol ChipCap 2 Add device tree bindings and an example for the ChipCap 2 humidity and temperature sensor. Reviewed-by: Conor Dooley Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240130-topic-chipcap2-v6-4-260bea05cf9b@gmail.com Signed-off-by: Guenter Roeck --- .../bindings/hwmon/amphenol,chipcap2.yaml | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml diff --git a/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml new file mode 100644 index 000000000000..17351fdbefce --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/amphenol,chipcap2.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ChipCap 2 humidity and temperature iio sensor + +maintainers: + - Javier Carrasco + +description: | + Relative humidity and temperature sensor on I2C bus. + + Datasheets: + https://www.amphenol-sensors.com/en/telaire/humidity/527-humidity-sensors/3095-chipcap-2 + +properties: + compatible: + oneOf: + - const: amphenol,cc2d23 + - items: + - enum: + - amphenol,cc2d23s + - amphenol,cc2d25 + - amphenol,cc2d25s + - amphenol,cc2d33 + - amphenol,cc2d33s + - amphenol,cc2d35 + - amphenol,cc2d35s + - const: amphenol,cc2d23 + + reg: + maxItems: 1 + + interrupts: + items: + - description: measurement ready indicator + - description: low humidity alarm + - description: high humidity alarm + + interrupt-names: + items: + - const: ready + - const: low + - const: high + + vdd-supply: + description: + Dedicated, controllable supply-regulator to reset the device and + enter in command mode. + +required: + - compatible + - reg + - vdd-supply + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + humidity@28 { + compatible = "amphenol,cc2d23s", "amphenol,cc2d23"; + reg = <0x28>; + interrupt-parent = <&gpio>; + interrupts = <4 IRQ_TYPE_EDGE_RISING>, + <5 IRQ_TYPE_EDGE_RISING>, + <6 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "ready", "low", "high"; + vdd-supply = <®_vdd>; + }; + }; From 3af350929e752fe72f0457033d448bff9aac695a Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jan 2024 22:06:48 +0100 Subject: [PATCH 0749/1038] hwmon: Add support for Amphenol ChipCap 2 The Amphenol ChipCap 2 is a capacitive polymer humidity and temperature sensor with an integrated EEPROM and minimum/maximum humidity alarms. All device variants offer an I2C interface and depending on the part number, two different output modes: - CC2D: digital output - CC2A: analog (PDM) output This driver adds support for the digital variant (CC2D part numbers), which includes the following part numbers: - non-sleep measurement mode (CC2D23, CC2D25, CC2D33, CC2D35) - sleep measurement mode (CC2D23S, CC2D25S, CC2D33S, CC2D35S) The Chipcap 2 EEPROM can be accessed to configure a series of parameters like the minimum/maximum humidity alarm threshold and hysteresis. The EEPROM is only accessible in the command window after a power-on reset. The default window lasts 10 ms if no Start_CM command is sent. After the command window is finished (either after the mentioned timeout of after a Start_NOM command is sent), the device enters the normal operation mode and makes a first measurement automatically. Unfortunately, the device does not provide any hardware or software reset and therefore the driver must trigger power cycles to enter the command mode. A dedicated, external regulator is required for that. This driver keeps the device off until a measurement or access to the EEPROM is required, making use of the first automatic measurement to avoid different code paths for sleep and non-sleep devices. The minimum and maximum humidity alarms are configured with two registers per alarm: one stores the alarm threshold and the other one keeps the value that turns off the alarm. The alarm signals are only updated when a measurement is carried out. Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240130-topic-chipcap2-v6-5-260bea05cf9b@gmail.com Signed-off-by: Guenter Roeck --- Documentation/hwmon/chipcap2.rst | 73 +++ Documentation/hwmon/index.rst | 1 + MAINTAINERS | 8 + drivers/hwmon/Kconfig | 10 + drivers/hwmon/Makefile | 1 + drivers/hwmon/chipcap2.c | 816 +++++++++++++++++++++++++++++++ 6 files changed, 909 insertions(+) create mode 100644 Documentation/hwmon/chipcap2.rst create mode 100644 drivers/hwmon/chipcap2.c diff --git a/Documentation/hwmon/chipcap2.rst b/Documentation/hwmon/chipcap2.rst new file mode 100644 index 000000000000..dc165becc64c --- /dev/null +++ b/Documentation/hwmon/chipcap2.rst @@ -0,0 +1,73 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver ChipCap2 +====================== + +Supported chips: + + * Amphenol CC2D23, CC2D23S, CC2D25, CC2D25S, CC2D33, CC2D33S, CC2D35, CC2D35S + + Prefix: 'chipcap2' + + Addresses scanned: - + + Datasheet: https://www.amphenol-sensors.com/en/telaire/humidity/527-humidity-sensors/3095-chipcap-2 + +Author: + + - Javier Carrasco + +Description +----------- + +This driver implements support for the Amphenol ChipCap 2, a humidity and +temperature chip family. Temperature is measured in milli degrees celsius, +relative humidity is expressed as a per cent mille. The measurement ranges +are the following: + + - Relative humidity: 0 to 100000 pcm (14-bit resolution) + - Temperature: -40000 to +125000 m°C (14-bit resolution) + +The device communicates with the I2C protocol and uses the I2C address 0x28 +by default. + +Depending on the hardware configuration, up to two humidity alarms to control +minimum and maximum values are provided. Their thresholds and hystersis can be +configured via sysfs. + +Thresholds and hysteris must be provided as a per cent mille. These values +might be truncated to match the 14-bit device resolution (6.1 pcm/LSB) + +Known Issues +------------ + +The driver does not support I2C address and command window length modification. + +sysfs-Interface +--------------- + +The following list includes the sysfs attributes that the driver always provides, +their permissions and a short description: + +=============================== ======= ======================================== +Name Perm Description +=============================== ======= ======================================== +temp1_input: RO temperature input +humidity1_input: RO humidity input +=============================== ======= ======================================== + +The following list includes the sysfs attributes that the driver may provide +depending on the hardware configuration: + +=============================== ======= ======================================== +Name Perm Description +=============================== ======= ======================================== +humidity1_min: RW humidity low limit. Measurements under + this limit trigger a humidity low alarm +humidity1_max: RW humidity high limit. Measurements above + this limit trigger a humidity high alarm +humidity1_min_hyst: RW humidity low hystersis +humidity1_max_hyst: RW humidity high hystersis +humidity1_min_alarm: RO humidity low alarm indicator +humidity1_max_alarm: RO humidity high alarm indicator +=============================== ======= ======================================== diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index 9ac087dd3e78..0d12254c0f9e 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -51,6 +51,7 @@ Hardware Monitoring Kernel Drivers bel-pfe bpa-rs600 bt1-pvt + chipcap2 coretemp corsair-cpro corsair-psu diff --git a/MAINTAINERS b/MAINTAINERS index dd69696f3c0e..b00657d2536f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1098,6 +1098,14 @@ F: Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml F: drivers/perf/amlogic/ F: include/soc/amlogic/ +AMPHENOL CHIPCAP 2 HUMIDITY-TEMPERATURE IIO DRIVER +M: Javier Carrasco +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml +F: Documentation/hwmon/chipcap2.rst +F: drivers/hwmon/chipcap2.c + AMPHION VPU CODEC V4L2 DRIVER M: Ming Qian M: Zhou Peng diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index cdf228fe50e2..5c85c976d795 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -452,6 +452,16 @@ config SENSORS_BT1_PVT_ALARMS the data conversion will be periodically performed and the data will be saved in the internal driver cache. +config SENSORS_CHIPCAP2 + tristate "Amphenol ChipCap 2 relative humidity and temperature sensor" + depends on I2C + help + Say yes here to build support for the Amphenol ChipCap 2 + relative humidity and temperature sensor. + + To compile this driver as a module, choose M here: the module + will be called chipcap2. + config SENSORS_CORSAIR_CPRO tristate "Corsair Commander Pro controller" depends on HID diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index a49704cd48a4..5da0c4ce881b 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -58,6 +58,7 @@ obj-$(CONFIG_SENSORS_ASPEED) += aspeed-pwm-tacho.o obj-$(CONFIG_SENSORS_ATXP1) += atxp1.o obj-$(CONFIG_SENSORS_AXI_FAN_CONTROL) += axi-fan-control.o obj-$(CONFIG_SENSORS_BT1_PVT) += bt1-pvt.o +obj-$(CONFIG_SENSORS_CHIPCAP2) += chipcap2.o obj-$(CONFIG_SENSORS_CORETEMP) += coretemp.o obj-$(CONFIG_SENSORS_CORSAIR_CPRO) += corsair-cpro.o obj-$(CONFIG_SENSORS_CORSAIR_PSU) += corsair-psu.o diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c new file mode 100644 index 000000000000..a62c507b1042 --- /dev/null +++ b/drivers/hwmon/chipcap2.c @@ -0,0 +1,816 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * cc2.c - Support for the Amphenol ChipCap 2 relative humidity, temperature sensor + * + * Part numbers supported: + * CC2D23, CC2D23S, CC2D25, CC2D25S, CC2D33, CC2D33S, CC2D35, CC2D35S + * + * Author: Javier Carrasco + * + * Datasheet and application notes: + * https://www.amphenol-sensors.com/en/telaire/humidity/527-humidity-sensors/3095-chipcap-2 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CC2_START_CM 0xA0 +#define CC2_START_NOM 0x80 +#define CC2_R_ALARM_H_ON 0x18 +#define CC2_R_ALARM_H_OFF 0x19 +#define CC2_R_ALARM_L_ON 0x1A +#define CC2_R_ALARM_L_OFF 0x1B +#define CC2_RW_OFFSET 0x40 +#define CC2_W_ALARM_H_ON (CC2_R_ALARM_H_ON + CC2_RW_OFFSET) +#define CC2_W_ALARM_H_OFF (CC2_R_ALARM_H_OFF + CC2_RW_OFFSET) +#define CC2_W_ALARM_L_ON (CC2_R_ALARM_L_ON + CC2_RW_OFFSET) +#define CC2_W_ALARM_L_OFF (CC2_R_ALARM_L_OFF + CC2_RW_OFFSET) + +#define CC2_STATUS_FIELD GENMASK(7, 6) +#define CC2_STATUS_VALID_DATA 0x00 +#define CC2_STATUS_STALE_DATA 0x01 +#define CC2_STATUS_CMD_MODE 0x02 + +#define CC2_RESPONSE_FIELD GENMASK(1, 0) +#define CC2_RESPONSE_BUSY 0x00 +#define CC2_RESPONSE_ACK 0x01 +#define CC2_RESPONSE_NACK 0x02 + +#define CC2_ERR_CORR_EEPROM BIT(2) +#define CC2_ERR_UNCORR_EEPROM BIT(3) +#define CC2_ERR_RAM_PARITY BIT(4) +#define CC2_ERR_CONFIG_LOAD BIT(5) + +#define CC2_EEPROM_SIZE 10 +#define CC2_EEPROM_DATA_LEN 3 +#define CC2_MEASUREMENT_DATA_LEN 4 + +#define CC2_RH_DATA_FIELD GENMASK(13, 0) + +/* ensure clean off -> on transitions */ +#define CC2_POWER_CYCLE_MS 80 + +#define CC2_STARTUP_TO_DATA_MS 55 +#define CC2_RESP_START_CM_US 100 +#define CC2_RESP_EEPROM_R_US 100 +#define CC2_RESP_EEPROM_W_MS 12 +#define CC2_STARTUP_TIME_US 1250 + +#define CC2_RH_MAX (100 * 1000U) + +#define CC2_CM_RETRIES 5 + +struct cc2_rh_alarm_info { + bool low_alarm; + bool high_alarm; + bool low_alarm_visible; + bool high_alarm_visible; +}; + +struct cc2_data { + struct cc2_rh_alarm_info rh_alarm; + struct completion complete; + struct device *hwmon; + struct i2c_client *client; + struct mutex dev_access_lock; /* device access lock */ + struct regulator *regulator; + const char *name; + int irq_ready; + int irq_low; + int irq_high; + bool process_irqs; +}; + +enum cc2_chan_addr { + CC2_CHAN_TEMP = 0, + CC2_CHAN_HUMIDITY, +}; + +/* %RH as a per cent mille from a register value */ +static long cc2_rh_convert(u16 data) +{ + unsigned long tmp = (data & CC2_RH_DATA_FIELD) * CC2_RH_MAX; + + return tmp / ((1 << 14) - 1); +} + +/* convert %RH to a register value */ +static u16 cc2_rh_to_reg(long data) +{ + return data * ((1 << 14) - 1) / CC2_RH_MAX; +} + +/* temperature in milli degrees celsius from a register value */ +static long cc2_temp_convert(u16 data) +{ + unsigned long tmp = ((data >> 2) * 165 * 1000U) / ((1 << 14) - 1); + + return tmp - 40 * 1000U; +} + +static int cc2_enable(struct cc2_data *data) +{ + int ret; + + /* exclusive regulator, check in case a disable failed */ + if (regulator_is_enabled(data->regulator)) + return 0; + + /* clear any pending completion */ + try_wait_for_completion(&data->complete); + + ret = regulator_enable(data->regulator); + if (ret < 0) + return ret; + + usleep_range(CC2_STARTUP_TIME_US, CC2_STARTUP_TIME_US + 125); + + data->process_irqs = true; + + return 0; +} + +static void cc2_disable(struct cc2_data *data) +{ + int err; + + /* ignore alarms triggered by voltage toggling when powering up */ + data->process_irqs = false; + + /* exclusive regulator, check in case an enable failed */ + if (regulator_is_enabled(data->regulator)) { + err = regulator_disable(data->regulator); + if (err) + dev_dbg(&data->client->dev, "Failed to disable device"); + } +} + +static int cc2_cmd_response_diagnostic(struct device *dev, u8 status) +{ + int resp; + + if (FIELD_GET(CC2_STATUS_FIELD, status) != CC2_STATUS_CMD_MODE) { + dev_dbg(dev, "Command sent out of command window\n"); + return -ETIMEDOUT; + } + + resp = FIELD_GET(CC2_RESPONSE_FIELD, status); + switch (resp) { + case CC2_RESPONSE_ACK: + return 0; + case CC2_RESPONSE_BUSY: + return -EBUSY; + case CC2_RESPONSE_NACK: + if (resp & CC2_ERR_CORR_EEPROM) + dev_dbg(dev, "Command failed: corrected EEPROM\n"); + if (resp & CC2_ERR_UNCORR_EEPROM) + dev_dbg(dev, "Command failed: uncorrected EEPROM\n"); + if (resp & CC2_ERR_RAM_PARITY) + dev_dbg(dev, "Command failed: RAM parity\n"); + if (resp & CC2_ERR_RAM_PARITY) + dev_dbg(dev, "Command failed: configuration error\n"); + return -ENODATA; + default: + dev_dbg(dev, "Unknown command reply\n"); + return -EINVAL; + } +} + +static int cc2_read_command_status(struct i2c_client *client) +{ + u8 status; + int ret; + + ret = i2c_master_recv(client, &status, 1); + if (ret != 1) { + ret = ret < 0 ? ret : -EIO; + return ret; + } + + return cc2_cmd_response_diagnostic(&client->dev, status); +} + +/* + * The command mode is only accessible after sending the START_CM command in the + * first 10 ms after power-up. Only in case the command window is missed, + * CC2_CM_RETRIES retries are attempted before giving up and returning an error. + */ +static int cc2_command_mode_start(struct cc2_data *data) +{ + unsigned long timeout; + int i, ret; + + for (i = 0; i < CC2_CM_RETRIES; i++) { + ret = cc2_enable(data); + if (ret < 0) + return ret; + + ret = i2c_smbus_write_word_data(data->client, CC2_START_CM, 0); + if (ret < 0) + return ret; + + if (data->irq_ready > 0) { + timeout = usecs_to_jiffies(2 * CC2_RESP_START_CM_US); + ret = wait_for_completion_timeout(&data->complete, + timeout); + if (!ret) + return -ETIMEDOUT; + } else { + usleep_range(CC2_RESP_START_CM_US, + 2 * CC2_RESP_START_CM_US); + } + ret = cc2_read_command_status(data->client); + if (ret != -ETIMEDOUT || i == CC2_CM_RETRIES) + break; + + /* command window missed, prepare for a retry */ + cc2_disable(data); + msleep(CC2_POWER_CYCLE_MS); + } + + return ret; +} + +/* Sending a Start_NOM command finishes the command mode immediately with no + * reply and the device enters normal operation mode + */ +static int cc2_command_mode_finish(struct cc2_data *data) +{ + int ret; + + ret = i2c_smbus_write_word_data(data->client, CC2_START_NOM, 0); + if (ret < 0) + return ret; + + return 0; +} + +static int cc2_write_reg(struct cc2_data *data, u8 reg, u16 val) +{ + unsigned long timeout; + int ret; + + ret = cc2_command_mode_start(data); + if (ret < 0) + goto disable; + + cpu_to_be16s(&val); + ret = i2c_smbus_write_word_data(data->client, reg, val); + if (ret < 0) + goto disable; + + if (data->irq_ready > 0) { + timeout = msecs_to_jiffies(2 * CC2_RESP_EEPROM_W_MS); + ret = wait_for_completion_timeout(&data->complete, timeout); + if (!ret) { + ret = -ETIMEDOUT; + goto disable; + } + } else { + msleep(CC2_RESP_EEPROM_W_MS); + } + + ret = cc2_read_command_status(data->client); + +disable: + cc2_disable(data); + + return ret; +} + +static int cc2_read_reg(struct cc2_data *data, u8 reg, u16 *val) +{ + u8 buf[CC2_EEPROM_DATA_LEN]; + unsigned long timeout; + int ret; + + ret = cc2_command_mode_start(data); + if (ret < 0) + return ret; + + ret = i2c_smbus_write_word_data(data->client, reg, 0); + if (ret < 0) + return ret; + + if (data->irq_ready > 0) { + timeout = usecs_to_jiffies(2 * CC2_RESP_EEPROM_R_US); + ret = wait_for_completion_timeout(&data->complete, timeout); + if (!ret) + return -ETIMEDOUT; + + } else { + usleep_range(CC2_RESP_EEPROM_R_US, CC2_RESP_EEPROM_R_US + 10); + } + ret = i2c_master_recv(data->client, buf, CC2_EEPROM_DATA_LEN); + if (ret != CC2_EEPROM_DATA_LEN) + return ret < 0 ? ret : -EIO; + + *val = be16_to_cpup((__be16 *)&buf[1]); + + return cc2_read_command_status(data->client); +} + +static int cc2_get_reg_val(struct cc2_data *data, u8 reg, long *val) +{ + u16 reg_val; + int ret; + + ret = cc2_read_reg(data, reg, ®_val); + *val = cc2_rh_convert(reg_val); + cc2_disable(data); + + return ret; +} + +static int cc2_data_fetch(struct i2c_client *client, + enum hwmon_sensor_types type, long *val) +{ + u8 data[CC2_MEASUREMENT_DATA_LEN]; + u8 status; + int ret; + + ret = i2c_master_recv(client, data, CC2_MEASUREMENT_DATA_LEN); + if (ret != CC2_MEASUREMENT_DATA_LEN) { + ret = ret < 0 ? ret : -EIO; + return ret; + } + status = FIELD_GET(CC2_STATUS_FIELD, data[0]); + if (status == CC2_STATUS_STALE_DATA) + return -EBUSY; + + if (status != CC2_STATUS_VALID_DATA) + return -EIO; + + switch (type) { + case hwmon_humidity: + *val = cc2_rh_convert(be16_to_cpup((__be16 *)&data[0])); + break; + case hwmon_temp: + *val = cc2_temp_convert(be16_to_cpup((__be16 *)&data[2])); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int cc2_read_measurement(struct cc2_data *data, + enum hwmon_sensor_types type, long *val) +{ + unsigned long timeout; + int ret; + + if (data->irq_ready > 0) { + timeout = msecs_to_jiffies(CC2_STARTUP_TO_DATA_MS * 2); + ret = wait_for_completion_timeout(&data->complete, timeout); + if (!ret) + return -ETIMEDOUT; + + } else { + msleep(CC2_STARTUP_TO_DATA_MS); + } + + ret = cc2_data_fetch(data->client, type, val); + + return ret; +} + +/* + * A measurement requires enabling the device, waiting for the automatic + * measurement to finish, reading the measurement data and disabling the device + * again. + */ +static int cc2_measurement(struct cc2_data *data, enum hwmon_sensor_types type, + long *val) +{ + int ret; + + ret = cc2_enable(data); + if (ret) + return ret; + + ret = cc2_read_measurement(data, type, val); + + cc2_disable(data); + + return ret; +} + +/* + * In order to check alarm status, the corresponding ALARM_OFF (hysteresis) + * register must be read and a new measurement must be carried out to trigger + * the alarm signals. Given that the device carries out a measurement after + * exiting the command mode, there is no need to force two power-up sequences. + * Instead, a NOM command is sent and the device is disabled after the + * measurement is read. + */ +static int cc2_read_hyst_and_measure(struct cc2_data *data, u8 reg, + long *hyst, long *measurement) +{ + u16 reg_val; + int ret; + + ret = cc2_read_reg(data, reg, ®_val); + if (ret) + goto disable; + + *hyst = cc2_rh_convert(reg_val); + + ret = cc2_command_mode_finish(data); + if (ret) + goto disable; + + ret = cc2_read_measurement(data, hwmon_humidity, measurement); + +disable: + cc2_disable(data); + + return ret; +} + +static umode_t cc2_is_visible(const void *data, enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct cc2_data *cc2 = data; + + switch (type) { + case hwmon_humidity: + switch (attr) { + case hwmon_humidity_input: + return 0444; + case hwmon_humidity_min_alarm: + return cc2->rh_alarm.low_alarm_visible ? 0444 : 0; + case hwmon_humidity_max_alarm: + return cc2->rh_alarm.high_alarm_visible ? 0444 : 0; + case hwmon_humidity_min: + case hwmon_humidity_min_hyst: + return cc2->rh_alarm.low_alarm_visible ? 0644 : 0; + case hwmon_humidity_max: + case hwmon_humidity_max_hyst: + return cc2->rh_alarm.high_alarm_visible ? 0644 : 0; + default: + return 0; + } + case hwmon_temp: + switch (attr) { + case hwmon_temp_input: + return 0444; + default: + return 0; + } + default: + break; + } + + return 0; +} + +static irqreturn_t cc2_ready_interrupt(int irq, void *data) +{ + struct cc2_data *cc2 = data; + + if (cc2->process_irqs) + complete(&cc2->complete); + + return IRQ_HANDLED; +} + +static irqreturn_t cc2_low_interrupt(int irq, void *data) +{ + struct cc2_data *cc2 = data; + + if (cc2->process_irqs) { + hwmon_notify_event(cc2->hwmon, hwmon_humidity, + hwmon_humidity_min_alarm, CC2_CHAN_HUMIDITY); + cc2->rh_alarm.low_alarm = true; + } + + return IRQ_HANDLED; +} + +static irqreturn_t cc2_high_interrupt(int irq, void *data) +{ + struct cc2_data *cc2 = data; + + if (cc2->process_irqs) { + hwmon_notify_event(cc2->hwmon, hwmon_humidity, + hwmon_humidity_max_alarm, CC2_CHAN_HUMIDITY); + cc2->rh_alarm.high_alarm = true; + } + + return IRQ_HANDLED; +} + +static int cc2_humidity_min_alarm_status(struct cc2_data *data, long *val) +{ + long measurement, min_hyst; + int ret; + + ret = cc2_read_hyst_and_measure(data, CC2_R_ALARM_L_OFF, &min_hyst, + &measurement); + if (ret < 0) + return ret; + + if (data->rh_alarm.low_alarm) { + *val = (measurement < min_hyst) ? 1 : 0; + data->rh_alarm.low_alarm = *val; + } else { + *val = 0; + } + + return 0; +} + +static int cc2_humidity_max_alarm_status(struct cc2_data *data, long *val) +{ + long measurement, max_hyst; + int ret; + + ret = cc2_read_hyst_and_measure(data, CC2_R_ALARM_H_OFF, &max_hyst, + &measurement); + if (ret < 0) + return ret; + + if (data->rh_alarm.high_alarm) { + *val = (measurement > max_hyst) ? 1 : 0; + data->rh_alarm.high_alarm = *val; + } else { + *val = 0; + } + + return 0; +} + +static int cc2_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct cc2_data *data = dev_get_drvdata(dev); + int ret = 0; + + mutex_lock(&data->dev_access_lock); + + switch (type) { + case hwmon_temp: + ret = cc2_measurement(data, type, val); + break; + case hwmon_humidity: + switch (attr) { + case hwmon_humidity_input: + ret = cc2_measurement(data, type, val); + break; + case hwmon_humidity_min: + ret = cc2_get_reg_val(data, CC2_R_ALARM_L_ON, val); + break; + case hwmon_humidity_min_hyst: + ret = cc2_get_reg_val(data, CC2_R_ALARM_L_OFF, val); + break; + case hwmon_humidity_max: + ret = cc2_get_reg_val(data, CC2_R_ALARM_H_ON, val); + break; + case hwmon_humidity_max_hyst: + ret = cc2_get_reg_val(data, CC2_R_ALARM_H_OFF, val); + break; + case hwmon_humidity_min_alarm: + ret = cc2_humidity_min_alarm_status(data, val); + break; + case hwmon_humidity_max_alarm: + ret = cc2_humidity_max_alarm_status(data, val); + break; + default: + ret = -EOPNOTSUPP; + } + break; + default: + ret = -EOPNOTSUPP; + } + + mutex_unlock(&data->dev_access_lock); + + return ret; +} + +static int cc2_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, long val) +{ + struct cc2_data *data = dev_get_drvdata(dev); + int ret; + u16 arg; + u8 cmd; + + if (type != hwmon_humidity) + return -EOPNOTSUPP; + + if (val < 0 || val > CC2_RH_MAX) + return -EINVAL; + + mutex_lock(&data->dev_access_lock); + + switch (attr) { + case hwmon_humidity_min: + cmd = CC2_W_ALARM_L_ON; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + case hwmon_humidity_min_hyst: + cmd = CC2_W_ALARM_L_OFF; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + case hwmon_humidity_max: + cmd = CC2_W_ALARM_H_ON; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + case hwmon_humidity_max_hyst: + cmd = CC2_W_ALARM_H_OFF; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + default: + ret = -EOPNOTSUPP; + break; + } + + mutex_unlock(&data->dev_access_lock); + + return ret; +} + +static int cc2_request_ready_irq(struct cc2_data *data, struct device *dev) +{ + int ret = 0; + + data->irq_ready = fwnode_irq_get_byname(dev_fwnode(dev), "ready"); + if (data->irq_ready > 0) { + init_completion(&data->complete); + ret = devm_request_threaded_irq(dev, data->irq_ready, NULL, + cc2_ready_interrupt, + IRQF_ONESHOT | + IRQF_TRIGGER_RISING, + dev_name(dev), data); + } + + return ret; +} + +static int cc2_request_alarm_irqs(struct cc2_data *data, struct device *dev) +{ + int ret; + + data->irq_low = fwnode_irq_get_byname(dev_fwnode(dev), "low"); + if (data->irq_low > 0) { + ret = devm_request_threaded_irq(dev, data->irq_low, NULL, + cc2_low_interrupt, + IRQF_ONESHOT | + IRQF_TRIGGER_RISING, + dev_name(dev), data); + if (!ret) + data->rh_alarm.low_alarm_visible = true; + } + + data->irq_high = fwnode_irq_get_byname(dev_fwnode(dev), "high"); + if (data->irq_high > 0) { + ret = devm_request_threaded_irq(dev, data->irq_high, NULL, + cc2_high_interrupt, + IRQF_ONESHOT | + IRQF_TRIGGER_RISING, + dev_name(dev), data); + if (!ret) + data->rh_alarm.high_alarm_visible = true; + } + + return ret; +} + +static const struct hwmon_channel_info *cc2_info[] = { + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + HWMON_CHANNEL_INFO(humidity, HWMON_H_INPUT | HWMON_H_MIN | HWMON_H_MAX | + HWMON_H_MIN_HYST | HWMON_H_MAX_HYST | + HWMON_H_MIN_ALARM | HWMON_H_MAX_ALARM), + NULL +}; + +static const struct hwmon_ops cc2_hwmon_ops = { + .is_visible = cc2_is_visible, + .read = cc2_read, + .write = cc2_write, +}; + +static const struct hwmon_chip_info cc2_chip_info = { + .ops = &cc2_hwmon_ops, + .info = cc2_info, +}; + +static int cc2_probe(struct i2c_client *client) +{ + struct cc2_data *data; + struct device *dev = &client->dev; + int ret; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -EOPNOTSUPP; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + i2c_set_clientdata(client, data); + + mutex_init(&data->dev_access_lock); + + data->client = client; + + data->regulator = devm_regulator_get_exclusive(dev, "vdd"); + if (IS_ERR(data->regulator)) { + dev_err_probe(dev, PTR_ERR(data->regulator), + "Failed to get regulator\n"); + return PTR_ERR(data->regulator); + } + + ret = cc2_request_ready_irq(data, dev); + if (ret) { + dev_err_probe(dev, ret, "Failed to request ready irq\n"); + return ret; + } + + ret = cc2_request_alarm_irqs(data, dev); + if (ret) { + dev_err_probe(dev, ret, "Failed to request alarm irqs\n"); + goto disable; + } + + data->hwmon = devm_hwmon_device_register_with_info(dev, client->name, + data, &cc2_chip_info, + NULL); + if (IS_ERR(data->hwmon)) { + dev_err_probe(dev, PTR_ERR(data->hwmon), + "Failed to register hwmon device\n"); + ret = PTR_ERR(data->hwmon); + } + +disable: + cc2_disable(data); + + return ret; +} + +static void cc2_remove(struct i2c_client *client) +{ + struct cc2_data *data = i2c_get_clientdata(client); + + cc2_disable(data); +} + +static const struct i2c_device_id cc2_id[] = { + { "cc2d23" }, + { "cc2d23s" }, + { "cc2d25" }, + { "cc2d25s" }, + { "cc2d33" }, + { "cc2d33s" }, + { "cc2d35" }, + { "cc2d35s" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, cc2_id); + +static const struct of_device_id cc2_of_match[] = { + { .compatible = "amphenol,cc2d23" }, + { .compatible = "amphenol,cc2d23s" }, + { .compatible = "amphenol,cc2d25" }, + { .compatible = "amphenol,cc2d25s" }, + { .compatible = "amphenol,cc2d33" }, + { .compatible = "amphenol,cc2d33s" }, + { .compatible = "amphenol,cc2d35" }, + { .compatible = "amphenol,cc2d35s" }, + { }, +}; +MODULE_DEVICE_TABLE(of, cc2_of_match); + +static struct i2c_driver cc2_driver = { + .driver = { + .name = "cc2d23", + .of_match_table = cc2_of_match, + }, + .probe = cc2_probe, + .remove = cc2_remove, + .id_table = cc2_id, +}; +module_i2c_driver(cc2_driver); + +MODULE_AUTHOR("Javier Carrasco "); +MODULE_DESCRIPTION("Amphenol ChipCap 2 humidity and temperature sensor driver"); +MODULE_LICENSE("GPL"); From 2948b88a5ba4e650e9b37705272e2a17014a6edb Mon Sep 17 00:00:00 2001 From: Cosmo Chou Date: Mon, 15 Jan 2024 18:05:16 +0800 Subject: [PATCH 0750/1038] dt-bindings: vendor-prefixes: add asteralabs Add vendor prefix for Astera Labs, Inc. https://www.asteralabs.com Signed-off-by: Cosmo Chou Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240115100518.2887549-2-chou.cosmo@gmail.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/vendor-prefixes.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 25158559471c..c734ea4de428 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -161,6 +161,8 @@ patternProperties: description: ASPEED Technology Inc. "^asrock,.*": description: ASRock Inc. + "^asteralabs,.*": + description: Astera Labs, Inc. "^asus,.*": description: AsusTek Computer Inc. "^atheros,.*": From 684a28759f4c8bf046532615a029075559d1f873 Mon Sep 17 00:00:00 2001 From: Cosmo Chou Date: Mon, 15 Jan 2024 18:05:17 +0800 Subject: [PATCH 0751/1038] dt-bindings: trivial-devices: add Astera Labs PT5161L Add dt-bindings for pt5161l temperature monitoring. Signed-off-by: Cosmo Chou Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240115100518.2887549-3-chou.cosmo@gmail.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/trivial-devices.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 088b23ed2ae6..842eb65e4c03 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -47,6 +47,8 @@ properties: - adi,lt7182s # AMS iAQ-Core VOC Sensor - ams,iaq-core + # Temperature monitoring of Astera Labs PT5161L PCIe retimer + - asteralabs,pt5161l # i2c serial eeprom (24cxx) - at,24c08 # ATSHA204 - i2c h/w symmetric crypto module From f3b4b146eb107bda47ee4a8b0927699f962e8a2f Mon Sep 17 00:00:00 2001 From: Aleksa Savic Date: Mon, 29 Jan 2024 12:19:28 +0100 Subject: [PATCH 0752/1038] hwmon: Add driver for NZXT Kraken X and Z series AIO CPU coolers This driver enables hardware monitoring support for NZXT Kraken X53/X63/X73 and Z53/Z63/Z73 all-in-one CPU liquid coolers. All models expose liquid temperature and pump speed (in RPM), as well as PWM control (natively only through a temp-PWM curve, but the driver also emulates fixed PWM control on top of that). The Z-series models additionally expose the speed and duty of an optionally connected fan, with the same PWM control capabilities. Pump and fan duty control mode can be set through pwm[1-2]_enable, where 1 is for the manual control mode and 2 is for the liquid temp to PWM curve mode. Writing a 0 disables control of the channel through the driver after setting its duty to 100%. As it is not possible to query the device for the active mode, the driver keeps track of it. The temperature of the curves relates to the fixed [20-59] C range, per device limitations, and correlating to the detected liquid temperature. Only PWM values (ranging from 0-255) can be set. The addressable RGB LEDs and LCD screen, included only on Z-series models, are not supported in this driver. Co-developed-by: Jonas Malaco Signed-off-by: Jonas Malaco Co-developed-by: Yury Zhuravlev Signed-off-by: Yury Zhuravlev Signed-off-by: Aleksa Savic Link: https://lore.kernel.org/r/20240129111932.368232-1-savicaleksa83@gmail.com Signed-off-by: Guenter Roeck --- Documentation/hwmon/index.rst | 1 + Documentation/hwmon/nzxt-kraken3.rst | 74 ++ MAINTAINERS | 8 + drivers/hwmon/Kconfig | 10 + drivers/hwmon/Makefile | 1 + drivers/hwmon/nzxt-kraken3.c | 1008 ++++++++++++++++++++++++++ 6 files changed, 1102 insertions(+) create mode 100644 Documentation/hwmon/nzxt-kraken3.rst create mode 100644 drivers/hwmon/nzxt-kraken3.c diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index 0d12254c0f9e..6f8a4a7524e8 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -174,6 +174,7 @@ Hardware Monitoring Kernel Drivers nsa320 ntc_thermistor nzxt-kraken2 + nzxt-kraken3 nzxt-smart2 occ oxp-sensors diff --git a/Documentation/hwmon/nzxt-kraken3.rst b/Documentation/hwmon/nzxt-kraken3.rst new file mode 100644 index 000000000000..90fd9dec15ff --- /dev/null +++ b/Documentation/hwmon/nzxt-kraken3.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver nzxt-kraken3 +========================== + +Supported devices: + +* NZXT Kraken X53 +* NZXT Kraken X63 +* NZXT Kraken X73 +* NZXT Kraken Z53 +* NZXT Kraken Z63 +* NZXT Kraken Z73 + +Author: Jonas Malaco, Aleksa Savic + +Description +----------- + +This driver enables hardware monitoring support for NZXT Kraken X53/X63/X73 and +Z53/Z63/Z73 all-in-one CPU liquid coolers. All models expose liquid temperature +and pump speed (in RPM), as well as PWM control (either as a fixed value +or through a temp-PWM curve). The Z-series models additionally expose the speed +and duty of an optionally connected fan, with the same PWM control capabilities. + +Pump and fan duty control mode can be set through pwm[1-2]_enable, where 1 is +for the manual control mode and 2 is for the liquid temp to PWM curve mode. +Writing a 0 disables control of the channel through the driver after setting its +duty to 100%. + +The temperature of the curves relates to the fixed [20-59] range, correlating to +the detected liquid temperature. Only PWM values (ranging from 0-255) can be set. +If in curve mode, setting point values should be done in moderation - the devices +require complete curves to be sent for each change; they can lock up or discard +the changes if they are too numerous at once. Suggestion is to set them while +in an another mode, and then apply them by switching to curve. + +The devices can report if they are faulty. The driver supports that situation +and will issue a warning. This can also happen when the USB cable is connected, +but SATA power is not. + +The addressable RGB LEDs and LCD screen (only on Z-series models) are not +supported in this driver, but can be controlled through existing userspace tools, +such as `liquidctl`_. + +.. _liquidctl: https://github.com/liquidctl/liquidctl + +Usage Notes +----------- + +As these are USB HIDs, the driver can be loaded automatically by the kernel and +supports hot swapping. + +Possible pwm_enable values are: + +====== ========================================================================== +0 Set fan to 100% +1 Direct PWM mode (applies value in corresponding PWM entry) +2 Curve control mode (applies the temp-PWM duty curve based on coolant temp) +====== ========================================================================== + +Sysfs entries +------------- + +============================== ================================================================ +fan1_input Pump speed (in rpm) +fan2_input Fan speed (in rpm) +temp1_input Coolant temperature (in millidegrees Celsius) +pwm1 Pump duty (value between 0-255) +pwm1_enable Pump duty control mode (0: disabled, 1: manual, 2: curve) +pwm2 Fan duty (value between 0-255) +pwm2_enable Fan duty control mode (0: disabled, 1: manual, 2: curve) +temp[1-2]_auto_point[1-40]_pwm Temp-PWM duty curves (for pump and fan), related to coolant temp +============================== ================================================================ diff --git a/MAINTAINERS b/MAINTAINERS index b00657d2536f..b9906f88c1b7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15840,6 +15840,14 @@ S: Maintained F: Documentation/hwmon/nzxt-kraken2.rst F: drivers/hwmon/nzxt-kraken2.c +NZXT-KRAKEN3 HARDWARE MONITORING DRIVER +M: Jonas Malaco +M: Aleksa Savic +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/nzxt-kraken3.rst +F: drivers/hwmon/nzxt-kraken3.c + NZXT-SMART2 HARDWARE MONITORING DRIVER M: Aleksandr Mezin L: linux-hwmon@vger.kernel.org diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 5c85c976d795..3904bb297d61 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1695,6 +1695,16 @@ config SENSORS_NZXT_KRAKEN2 This driver can also be built as a module. If so, the module will be called nzxt-kraken2. +config SENSORS_NZXT_KRAKEN3 + tristate "NZXT Kraken X53/X63/X73, Z53/Z63/Z73 coolers" + depends on USB_HID + help + If you say yes here you get support for hardware monitoring for the + NZXT Kraken X53/X63/X73, Z53/Z63/Z73 all-in-one CPU liquid coolers. + + This driver can also be built as a module. If so, the module + will be called nzxt-kraken3. + config SENSORS_NZXT_SMART2 tristate "NZXT RGB & Fan Controller/Smart Device v2" depends on USB_HID diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 5da0c4ce881b..76e6dfef9f24 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -175,6 +175,7 @@ obj-$(CONFIG_SENSORS_NPCM7XX) += npcm750-pwm-fan.o obj-$(CONFIG_SENSORS_NSA320) += nsa320-hwmon.o obj-$(CONFIG_SENSORS_NTC_THERMISTOR) += ntc_thermistor.o obj-$(CONFIG_SENSORS_NZXT_KRAKEN2) += nzxt-kraken2.o +obj-$(CONFIG_SENSORS_NZXT_KRAKEN3) += nzxt-kraken3.o obj-$(CONFIG_SENSORS_NZXT_SMART2) += nzxt-smart2.o obj-$(CONFIG_SENSORS_OXP) += oxp-sensors.o obj-$(CONFIG_SENSORS_PC87360) += pc87360.o diff --git a/drivers/hwmon/nzxt-kraken3.c b/drivers/hwmon/nzxt-kraken3.c new file mode 100644 index 000000000000..5806a3f32bcb --- /dev/null +++ b/drivers/hwmon/nzxt-kraken3.c @@ -0,0 +1,1008 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * hwmon driver for NZXT Kraken X53/X63/X73 and Z53/Z63/Z73 all in one coolers. + * X53 and Z53 in code refer to all models in their respective series (shortened + * for brevity). + * + * Copyright 2021 Jonas Malaco + * Copyright 2022 Aleksa Savic + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USB_VENDOR_ID_NZXT 0x1e71 +#define USB_PRODUCT_ID_X53 0x2007 +#define USB_PRODUCT_ID_X53_SECOND 0x2014 +#define USB_PRODUCT_ID_Z53 0x3008 + +enum kinds { X53, Z53 } __packed; +enum pwm_enable { off, manual, curve } __packed; + +static const char *const kraken3_device_names[] = { + [X53] = "x53", + [Z53] = "z53", +}; + +#define DRIVER_NAME "nzxt_kraken3" +#define STATUS_REPORT_ID 0x75 +#define FIRMWARE_REPORT_ID 0x11 +#define STATUS_VALIDITY 2000 /* In ms, equivalent to period of four status reports */ +#define CUSTOM_CURVE_POINTS 40 /* For temps from 20C to 59C (critical temp) */ +#define PUMP_DUTY_MIN 20 /* In percent */ + +/* Sensor report offsets for Kraken X53 and Z53 */ +#define TEMP_SENSOR_START_OFFSET 15 +#define TEMP_SENSOR_END_OFFSET 16 +#define PUMP_SPEED_OFFSET 17 +#define PUMP_DUTY_OFFSET 19 + +/* Firmware version report offset for Kraken X53 and Z53 */ +#define FIRMWARE_VERSION_OFFSET 17 + +/* Sensor report offsets for Kraken Z53 */ +#define Z53_FAN_SPEED_OFFSET 23 +#define Z53_FAN_DUTY_OFFSET 25 + +/* Report offsets for control commands for Kraken X53 and Z53 */ +#define SET_DUTY_ID_OFFSET 1 + +/* Control commands and their lengths for Kraken X53 and Z53 */ + +/* Last byte sets the report interval at 0.5s */ +static const u8 set_interval_cmd[] = { 0x70, 0x02, 0x01, 0xB8, 1 }; +static const u8 finish_init_cmd[] = { 0x70, 0x01 }; +static const u8 __maybe_unused get_fw_version_cmd[] = { 0x10, 0x01 }; +static const u8 set_pump_duty_cmd_header[] = { 0x72, 0x00, 0x00, 0x00 }; +static const u8 z53_get_status_cmd[] = { 0x74, 0x01 }; + +#define SET_INTERVAL_CMD_LENGTH 5 +#define FINISH_INIT_CMD_LENGTH 2 +#define GET_FW_VERSION_CMD_LENGTH 2 +#define MAX_REPORT_LENGTH 64 +#define MIN_REPORT_LENGTH 20 +#define SET_CURVE_DUTY_CMD_HEADER_LENGTH 4 +/* 4 byte header and 40 duty offsets */ +#define SET_CURVE_DUTY_CMD_LENGTH (4 + 40) +#define Z53_GET_STATUS_CMD_LENGTH 2 + +static const char *const kraken3_temp_label[] = { + "Coolant temp", +}; + +static const char *const kraken3_fan_label[] = { + "Pump speed", + "Fan speed" +}; + +struct kraken3_channel_info { + enum pwm_enable mode; + + /* Both values are PWM */ + u16 reported_duty; + u16 fixed_duty; /* Manually set fixed duty */ + + u8 pwm_points[CUSTOM_CURVE_POINTS]; +}; + +struct kraken3_data { + struct hid_device *hdev; + struct device *hwmon_dev; + struct dentry *debugfs; + struct mutex buffer_lock; /* For locking access to buffer */ + struct mutex z53_status_request_lock; + struct completion fw_version_processed; + /* + * For X53 devices, tracks whether an initial (one) sensor report was received to + * make fancontrol not bail outright. For Z53 devices, whether a status report + * was processed after requesting one. + */ + struct completion status_report_processed; + /* For locking the above completion */ + spinlock_t status_completion_lock; + + u8 *buffer; + struct kraken3_channel_info channel_info[2]; /* Pump and fan */ + bool is_device_faulty; + + /* Sensor values */ + s32 temp_input[1]; + u16 fan_input[2]; + + enum kinds kind; + u8 firmware_version[3]; + + unsigned long updated; /* jiffies */ +}; + +static umode_t kraken3_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, + int channel) +{ + const struct kraken3_data *priv = data; + + switch (type) { + case hwmon_temp: + if (channel < 1) + return 0444; + break; + case hwmon_fan: + switch (priv->kind) { + case X53: + /* Just the pump */ + if (channel < 1) + return 0444; + break; + case Z53: + /* Pump and fan */ + if (channel < 2) + return 0444; + break; + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + case hwmon_pwm_input: + switch (priv->kind) { + case X53: + /* Just the pump */ + if (channel < 1) + return 0644; + break; + case Z53: + /* Pump and fan */ + if (channel < 2) + return 0644; + break; + default: + break; + } + break; + default: + break; + } + break; + default: + break; + } + + return 0; +} + +/* + * Writes the command to the device with the rest of the report (up to 64 bytes) filled + * with zeroes. + */ +static int kraken3_write_expanded(struct kraken3_data *priv, const u8 *cmd, int cmd_length) +{ + int ret; + + mutex_lock(&priv->buffer_lock); + + memcpy_and_pad(priv->buffer, MAX_REPORT_LENGTH, cmd, cmd_length, 0x00); + ret = hid_hw_output_report(priv->hdev, priv->buffer, MAX_REPORT_LENGTH); + + mutex_unlock(&priv->buffer_lock); + return ret; +} + +static int kraken3_percent_to_pwm(long val) +{ + return DIV_ROUND_CLOSEST(val * 255, 100); +} + +static int kraken3_pwm_to_percent(long val, int channel) +{ + int percent_value; + + if (val < 0 || val > 255) + return -EINVAL; + + percent_value = DIV_ROUND_CLOSEST(val * 100, 255); + + /* Bring up pump duty to min value if needed */ + if (channel == 0 && percent_value < PUMP_DUTY_MIN) + percent_value = PUMP_DUTY_MIN; + + return percent_value; +} + +static int kraken3_read_x53(struct kraken3_data *priv) +{ + int ret; + + if (completion_done(&priv->status_report_processed)) + /* + * We're here because data is stale. This means that sensor reports haven't + * been received for some time in kraken3_raw_event(). On X-series sensor data + * can't be manually requested, so return an error. + */ + return -ENODATA; + + /* + * Data needs to be read, but a sensor report wasn't yet received. It's usually + * fancontrol that requests data this early and it exits if it reads an error code. + * So, wait for the first report to be parsed (but up to STATUS_VALIDITY). + * This does not concern the Z series devices, because they send a sensor report + * only when requested. + */ + ret = wait_for_completion_interruptible_timeout(&priv->status_report_processed, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + + /* The first sensor report was parsed on time and reading can continue */ + return 0; +} + +static int kraken3_read_z53(struct kraken3_data *priv) +{ + int ret = mutex_lock_interruptible(&priv->z53_status_request_lock); + + if (ret < 0) + return ret; + + if (!time_after(jiffies, priv->updated + msecs_to_jiffies(STATUS_VALIDITY))) { + /* Data is up to date */ + goto unlock_and_return; + } + + /* + * Disable interrupts for a moment to safely reinit the completion, + * as hidraw calls could have allowed one or more readers to complete. + */ + spin_lock_bh(&priv->status_completion_lock); + reinit_completion(&priv->status_report_processed); + spin_unlock_bh(&priv->status_completion_lock); + + /* Send command for getting status */ + ret = kraken3_write_expanded(priv, z53_get_status_cmd, Z53_GET_STATUS_CMD_LENGTH); + if (ret < 0) + goto unlock_and_return; + + /* Wait for completion from kraken3_raw_event() */ + ret = wait_for_completion_interruptible_timeout(&priv->status_report_processed, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + ret = -ETIMEDOUT; + +unlock_and_return: + mutex_unlock(&priv->z53_status_request_lock); + if (ret < 0) + return ret; + + return 0; +} + +static int kraken3_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, + long *val) +{ + struct kraken3_data *priv = dev_get_drvdata(dev); + int ret; + + if (time_after(jiffies, priv->updated + msecs_to_jiffies(STATUS_VALIDITY))) { + if (priv->kind == X53) + ret = kraken3_read_x53(priv); + else + ret = kraken3_read_z53(priv); + + if (ret < 0) + return ret; + + if (priv->is_device_faulty) + return -ENODATA; + } + + switch (type) { + case hwmon_temp: + *val = priv->temp_input[channel]; + break; + case hwmon_fan: + *val = priv->fan_input[channel]; + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + *val = priv->channel_info[channel].mode; + break; + case hwmon_pwm_input: + *val = priv->channel_info[channel].reported_duty; + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static int kraken3_read_string(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, const char **str) +{ + switch (type) { + case hwmon_temp: + *str = kraken3_temp_label[channel]; + break; + case hwmon_fan: + *str = kraken3_fan_label[channel]; + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +/* Writes custom curve to device */ +static int kraken3_write_curve(struct kraken3_data *priv, u8 *curve_array, int channel) +{ + u8 fixed_duty_cmd[SET_CURVE_DUTY_CMD_LENGTH]; + int ret; + + /* Copy command header */ + memcpy(fixed_duty_cmd, set_pump_duty_cmd_header, SET_CURVE_DUTY_CMD_HEADER_LENGTH); + + /* Set the correct ID for writing pump/fan duty (0x01 or 0x02, respectively) */ + fixed_duty_cmd[SET_DUTY_ID_OFFSET] = channel + 1; + + /* Copy curve to command */ + memcpy(fixed_duty_cmd + SET_CURVE_DUTY_CMD_HEADER_LENGTH, curve_array, CUSTOM_CURVE_POINTS); + + ret = kraken3_write_expanded(priv, fixed_duty_cmd, SET_CURVE_DUTY_CMD_LENGTH); + return ret; +} + +static int kraken3_write_fixed_duty(struct kraken3_data *priv, long val, int channel) +{ + u8 fixed_curve_points[CUSTOM_CURVE_POINTS]; + int ret, percent_val, i; + + percent_val = kraken3_pwm_to_percent(val, channel); + if (percent_val < 0) + return percent_val; + + /* + * The devices can only control the duty through a curve. + * Since we're setting a fixed duty here, fill the whole curve + * (ranging from 20C to 59C) with the same duty, except for + * the last point, the critical temperature, where it's maxed + * out for safety. + */ + + /* Fill the custom curve with the fixed value we're setting */ + for (i = 0; i < CUSTOM_CURVE_POINTS - 1; i++) + fixed_curve_points[i] = percent_val; + + /* Force duty to 100% at critical temp */ + fixed_curve_points[CUSTOM_CURVE_POINTS - 1] = 100; + + /* Write the fixed duty curve to the device */ + ret = kraken3_write_curve(priv, fixed_curve_points, channel); + return ret; +} + +static int kraken3_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, + long val) +{ + struct kraken3_data *priv = dev_get_drvdata(dev); + int ret; + + switch (type) { + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + /* Remember the last set fixed duty for channel */ + priv->channel_info[channel].fixed_duty = val; + + if (priv->channel_info[channel].mode == manual) { + ret = kraken3_write_fixed_duty(priv, val, channel); + if (ret < 0) + return ret; + + /* + * Lock onto this value and report it until next interrupt status + * report is received, so userspace tools can continue to work. + */ + priv->channel_info[channel].reported_duty = val; + } + break; + case hwmon_pwm_enable: + if (val < 0 || val > 2) + return -EINVAL; + + switch (val) { + case 0: + /* Set channel to 100%, direct duty value */ + ret = kraken3_write_fixed_duty(priv, 255, channel); + if (ret < 0) + return ret; + + /* We don't control anything anymore */ + priv->channel_info[channel].mode = off; + break; + case 1: + /* Apply the last known direct duty value */ + ret = + kraken3_write_fixed_duty(priv, + priv->channel_info[channel].fixed_duty, + channel); + if (ret < 0) + return ret; + + priv->channel_info[channel].mode = manual; + break; + case 2: + /* Apply the curve and note as enabled */ + ret = + kraken3_write_curve(priv, + priv->channel_info[channel].pwm_points, + channel); + if (ret < 0) + return ret; + + priv->channel_info[channel].mode = curve; + break; + default: + break; + } + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static ssize_t kraken3_fan_curve_pwm_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct sensor_device_attribute_2 *dev_attr = to_sensor_dev_attr_2(attr); + struct kraken3_data *priv = dev_get_drvdata(dev); + long val; + int ret; + + if (kstrtol(buf, 10, &val) < 0) + return -EINVAL; + + val = kraken3_pwm_to_percent(val, dev_attr->nr); + if (val < 0) + return val; + + priv->channel_info[dev_attr->nr].pwm_points[dev_attr->index] = val; + + if (priv->channel_info[dev_attr->nr].mode == curve) { + /* Apply the curve */ + ret = + kraken3_write_curve(priv, + priv->channel_info[dev_attr->nr].pwm_points, dev_attr->nr); + if (ret < 0) + return ret; + } + + return count; +} + +static umode_t kraken3_curve_props_are_visible(struct kobject *kobj, struct attribute *attr, + int index) +{ + struct device *dev = kobj_to_dev(kobj); + struct kraken3_data *priv = dev_get_drvdata(dev); + + /* Only Z53 has the fan curve */ + if (index >= CUSTOM_CURVE_POINTS && priv->kind != Z53) + return 0; + + return attr->mode; +} + +/* Custom pump curve from 20C to 59C (critical temp) */ +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point1_pwm, kraken3_fan_curve_pwm, 0, 0); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point2_pwm, kraken3_fan_curve_pwm, 0, 1); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point3_pwm, kraken3_fan_curve_pwm, 0, 2); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point4_pwm, kraken3_fan_curve_pwm, 0, 3); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point5_pwm, kraken3_fan_curve_pwm, 0, 4); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point6_pwm, kraken3_fan_curve_pwm, 0, 5); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point7_pwm, kraken3_fan_curve_pwm, 0, 6); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point8_pwm, kraken3_fan_curve_pwm, 0, 7); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point9_pwm, kraken3_fan_curve_pwm, 0, 8); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point10_pwm, kraken3_fan_curve_pwm, 0, 9); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point11_pwm, kraken3_fan_curve_pwm, 0, 10); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point12_pwm, kraken3_fan_curve_pwm, 0, 11); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point13_pwm, kraken3_fan_curve_pwm, 0, 12); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point14_pwm, kraken3_fan_curve_pwm, 0, 13); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point15_pwm, kraken3_fan_curve_pwm, 0, 14); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point16_pwm, kraken3_fan_curve_pwm, 0, 15); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point17_pwm, kraken3_fan_curve_pwm, 0, 16); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point18_pwm, kraken3_fan_curve_pwm, 0, 17); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point19_pwm, kraken3_fan_curve_pwm, 0, 18); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point20_pwm, kraken3_fan_curve_pwm, 0, 19); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point21_pwm, kraken3_fan_curve_pwm, 0, 20); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point22_pwm, kraken3_fan_curve_pwm, 0, 21); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point23_pwm, kraken3_fan_curve_pwm, 0, 22); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point24_pwm, kraken3_fan_curve_pwm, 0, 23); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point25_pwm, kraken3_fan_curve_pwm, 0, 24); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point26_pwm, kraken3_fan_curve_pwm, 0, 25); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point27_pwm, kraken3_fan_curve_pwm, 0, 26); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point28_pwm, kraken3_fan_curve_pwm, 0, 27); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point29_pwm, kraken3_fan_curve_pwm, 0, 28); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point30_pwm, kraken3_fan_curve_pwm, 0, 29); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point31_pwm, kraken3_fan_curve_pwm, 0, 30); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point32_pwm, kraken3_fan_curve_pwm, 0, 31); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point33_pwm, kraken3_fan_curve_pwm, 0, 32); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point34_pwm, kraken3_fan_curve_pwm, 0, 33); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point35_pwm, kraken3_fan_curve_pwm, 0, 34); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point36_pwm, kraken3_fan_curve_pwm, 0, 35); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point37_pwm, kraken3_fan_curve_pwm, 0, 36); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point38_pwm, kraken3_fan_curve_pwm, 0, 37); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point39_pwm, kraken3_fan_curve_pwm, 0, 38); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point40_pwm, kraken3_fan_curve_pwm, 0, 39); + +/* Custom fan curve from 20C to 59C (critical temp) */ +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point1_pwm, kraken3_fan_curve_pwm, 1, 0); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point2_pwm, kraken3_fan_curve_pwm, 1, 1); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point3_pwm, kraken3_fan_curve_pwm, 1, 2); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point4_pwm, kraken3_fan_curve_pwm, 1, 3); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point5_pwm, kraken3_fan_curve_pwm, 1, 4); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point6_pwm, kraken3_fan_curve_pwm, 1, 5); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point7_pwm, kraken3_fan_curve_pwm, 1, 6); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point8_pwm, kraken3_fan_curve_pwm, 1, 7); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point9_pwm, kraken3_fan_curve_pwm, 1, 8); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point10_pwm, kraken3_fan_curve_pwm, 1, 9); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point11_pwm, kraken3_fan_curve_pwm, 1, 10); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point12_pwm, kraken3_fan_curve_pwm, 1, 11); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point13_pwm, kraken3_fan_curve_pwm, 1, 12); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point14_pwm, kraken3_fan_curve_pwm, 1, 13); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point15_pwm, kraken3_fan_curve_pwm, 1, 14); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point16_pwm, kraken3_fan_curve_pwm, 1, 15); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point17_pwm, kraken3_fan_curve_pwm, 1, 16); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point18_pwm, kraken3_fan_curve_pwm, 1, 17); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point19_pwm, kraken3_fan_curve_pwm, 1, 18); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point20_pwm, kraken3_fan_curve_pwm, 1, 19); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point21_pwm, kraken3_fan_curve_pwm, 1, 20); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point22_pwm, kraken3_fan_curve_pwm, 1, 21); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point23_pwm, kraken3_fan_curve_pwm, 1, 22); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point24_pwm, kraken3_fan_curve_pwm, 1, 23); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point25_pwm, kraken3_fan_curve_pwm, 1, 24); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point26_pwm, kraken3_fan_curve_pwm, 1, 25); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point27_pwm, kraken3_fan_curve_pwm, 1, 26); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point28_pwm, kraken3_fan_curve_pwm, 1, 27); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point29_pwm, kraken3_fan_curve_pwm, 1, 28); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point30_pwm, kraken3_fan_curve_pwm, 1, 29); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point31_pwm, kraken3_fan_curve_pwm, 1, 30); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point32_pwm, kraken3_fan_curve_pwm, 1, 31); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point33_pwm, kraken3_fan_curve_pwm, 1, 32); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point34_pwm, kraken3_fan_curve_pwm, 1, 33); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point35_pwm, kraken3_fan_curve_pwm, 1, 34); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point36_pwm, kraken3_fan_curve_pwm, 1, 35); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point37_pwm, kraken3_fan_curve_pwm, 1, 36); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point38_pwm, kraken3_fan_curve_pwm, 1, 37); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point39_pwm, kraken3_fan_curve_pwm, 1, 38); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point40_pwm, kraken3_fan_curve_pwm, 1, 39); + +static struct attribute *kraken3_curve_attrs[] = { + /* Pump control curve */ + &sensor_dev_attr_temp1_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point6_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point7_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point8_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point9_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point10_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point11_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point12_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point13_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point14_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point15_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point16_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point17_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point18_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point19_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point20_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point21_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point22_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point23_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point24_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point25_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point26_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point27_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point28_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point29_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point30_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point31_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point32_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point33_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point34_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point35_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point36_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point37_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point38_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point39_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point40_pwm.dev_attr.attr, + /* Fan control curve (Z53 only) */ + &sensor_dev_attr_temp2_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point6_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point7_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point8_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point9_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point10_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point11_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point12_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point13_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point14_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point15_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point16_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point17_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point18_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point19_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point20_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point21_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point22_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point23_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point24_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point25_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point26_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point27_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point28_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point29_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point30_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point31_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point32_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point33_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point34_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point35_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point36_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point37_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point38_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point39_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point40_pwm.dev_attr.attr, + NULL +}; + +static const struct attribute_group kraken3_curves_group = { + .attrs = kraken3_curve_attrs, + .is_visible = kraken3_curve_props_are_visible +}; + +static const struct attribute_group *kraken3_groups[] = { + &kraken3_curves_group, + NULL +}; + +static const struct hwmon_ops kraken3_hwmon_ops = { + .is_visible = kraken3_is_visible, + .read = kraken3_read, + .read_string = kraken3_read_string, + .write = kraken3_write +}; + +static const struct hwmon_channel_info *kraken3_info[] = { + HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | HWMON_T_LABEL), + HWMON_CHANNEL_INFO(fan, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL), + HWMON_CHANNEL_INFO(pwm, + HWMON_PWM_INPUT | HWMON_PWM_ENABLE, + HWMON_PWM_INPUT | HWMON_PWM_ENABLE), + NULL +}; + +static const struct hwmon_chip_info kraken3_chip_info = { + .ops = &kraken3_hwmon_ops, + .info = kraken3_info, +}; + +static int kraken3_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + int i; + + if (size < MIN_REPORT_LENGTH) + return 0; + + if (report->id == FIRMWARE_REPORT_ID) { + /* Read firmware version */ + for (i = 0; i < 3; i++) + priv->firmware_version[i] = data[FIRMWARE_VERSION_OFFSET + i]; + + if (!completion_done(&priv->fw_version_processed)) + complete_all(&priv->fw_version_processed); + + return 0; + } + + if (report->id != STATUS_REPORT_ID) + return 0; + + if (data[TEMP_SENSOR_START_OFFSET] == 0xff && data[TEMP_SENSOR_END_OFFSET] == 0xff) { + hid_err_once(hdev, + "firmware or device is possibly damaged (is SATA power connected?), not parsing reports\n"); + + /* + * Mark first X-series device report as received, + * as well as all for Z-series, if faulty. + */ + spin_lock(&priv->status_completion_lock); + if (priv->kind != X53 || !completion_done(&priv->status_report_processed)) { + priv->is_device_faulty = true; + complete_all(&priv->status_report_processed); + } + spin_unlock(&priv->status_completion_lock); + + return 0; + } + + /* Received normal data */ + priv->is_device_faulty = false; + + /* Temperature and fan sensor readings */ + priv->temp_input[0] = + data[TEMP_SENSOR_START_OFFSET] * 1000 + data[TEMP_SENSOR_END_OFFSET] * 100; + + priv->fan_input[0] = get_unaligned_le16(data + PUMP_SPEED_OFFSET); + priv->channel_info[0].reported_duty = kraken3_percent_to_pwm(data[PUMP_DUTY_OFFSET]); + + spin_lock(&priv->status_completion_lock); + if (priv->kind == X53 && !completion_done(&priv->status_report_processed)) { + /* Mark first X-series device report as received */ + complete_all(&priv->status_report_processed); + } else if (priv->kind == Z53) { + /* Additional readings for Z53 */ + priv->fan_input[1] = get_unaligned_le16(data + Z53_FAN_SPEED_OFFSET); + priv->channel_info[1].reported_duty = + kraken3_percent_to_pwm(data[Z53_FAN_DUTY_OFFSET]); + + if (!completion_done(&priv->status_report_processed)) + complete_all(&priv->status_report_processed); + } + spin_unlock(&priv->status_completion_lock); + + priv->updated = jiffies; + + return 0; +} + +static int kraken3_init_device(struct hid_device *hdev) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + int ret; + + /* Set the polling interval */ + ret = kraken3_write_expanded(priv, set_interval_cmd, SET_INTERVAL_CMD_LENGTH); + if (ret < 0) + return ret; + + /* Finalize the init process */ + ret = kraken3_write_expanded(priv, finish_init_cmd, FINISH_INIT_CMD_LENGTH); + if (ret < 0) + return ret; + + return 0; +} + +static int kraken3_get_fw_ver(struct hid_device *hdev) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + int ret; + + ret = kraken3_write_expanded(priv, get_fw_version_cmd, GET_FW_VERSION_CMD_LENGTH); + if (ret < 0) + return ret; + + ret = wait_for_completion_interruptible_timeout(&priv->fw_version_processed, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + + return 0; +} + +static int __maybe_unused kraken3_reset_resume(struct hid_device *hdev) +{ + int ret; + + ret = kraken3_init_device(hdev); + if (ret) + hid_err(hdev, "req init (reset_resume) failed with %d\n", ret); + + return ret; +} + +static int firmware_version_show(struct seq_file *seqf, void *unused) +{ + struct kraken3_data *priv = seqf->private; + + seq_printf(seqf, "%u.%u.%u\n", priv->firmware_version[0], priv->firmware_version[1], + priv->firmware_version[2]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(firmware_version); + +static void kraken3_debugfs_init(struct kraken3_data *priv) +{ + char name[64]; + + if (!priv->firmware_version[0]) + return; /* Nothing to display in debugfs */ + + scnprintf(name, sizeof(name), "%s_%s-%s", DRIVER_NAME, kraken3_device_names[priv->kind], + dev_name(&priv->hdev->dev)); + + priv->debugfs = debugfs_create_dir(name, NULL); + debugfs_create_file("firmware_version", 0444, priv->debugfs, priv, &firmware_version_fops); +} + +static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + struct kraken3_data *priv; + int ret; + + priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->hdev = hdev; + hid_set_drvdata(hdev, priv); + + /* + * Initialize ->updated to STATUS_VALIDITY seconds in the past, making + * the initial empty data invalid for kraken3_read without the need for + * a special case there. + */ + priv->updated = jiffies - msecs_to_jiffies(STATUS_VALIDITY); + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "hid parse failed with %d\n", ret); + return ret; + } + + /* Enable hidraw so existing user-space tools can continue to work */ + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) { + hid_err(hdev, "hid hw start failed with %d\n", ret); + return ret; + } + + ret = hid_hw_open(hdev); + if (ret) { + hid_err(hdev, "hid hw open failed with %d\n", ret); + goto fail_and_stop; + } + + switch (hdev->product) { + case USB_PRODUCT_ID_X53: + case USB_PRODUCT_ID_X53_SECOND: + priv->kind = X53; + break; + case USB_PRODUCT_ID_Z53: + priv->kind = Z53; + break; + default: + break; + } + + priv->buffer = devm_kzalloc(&hdev->dev, MAX_REPORT_LENGTH, GFP_KERNEL); + if (!priv->buffer) { + ret = -ENOMEM; + goto fail_and_close; + } + + mutex_init(&priv->buffer_lock); + mutex_init(&priv->z53_status_request_lock); + init_completion(&priv->fw_version_processed); + init_completion(&priv->status_report_processed); + spin_lock_init(&priv->status_completion_lock); + + hid_device_io_start(hdev); + ret = kraken3_init_device(hdev); + if (ret < 0) { + hid_err(hdev, "device init failed with %d\n", ret); + goto fail_and_close; + } + + ret = kraken3_get_fw_ver(hdev); + if (ret < 0) + hid_warn(hdev, "fw version request failed with %d\n", ret); + + priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, + kraken3_device_names[priv->kind], priv, + &kraken3_chip_info, kraken3_groups); + if (IS_ERR(priv->hwmon_dev)) { + ret = PTR_ERR(priv->hwmon_dev); + hid_err(hdev, "hwmon registration failed with %d\n", ret); + goto fail_and_close; + } + + kraken3_debugfs_init(priv); + + return 0; + +fail_and_close: + hid_hw_close(hdev); +fail_and_stop: + hid_hw_stop(hdev); + return ret; +} + +static void kraken3_remove(struct hid_device *hdev) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + + debugfs_remove_recursive(priv->debugfs); + hwmon_device_unregister(priv->hwmon_dev); + + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static const struct hid_device_id kraken3_table[] = { + /* NZXT Kraken X53/X63/X73 have two possible product IDs */ + { HID_USB_DEVICE(USB_VENDOR_ID_NZXT, USB_PRODUCT_ID_X53) }, + { HID_USB_DEVICE(USB_VENDOR_ID_NZXT, USB_PRODUCT_ID_X53_SECOND) }, + { HID_USB_DEVICE(USB_VENDOR_ID_NZXT, USB_PRODUCT_ID_Z53) }, + { } +}; + +MODULE_DEVICE_TABLE(hid, kraken3_table); + +static struct hid_driver kraken3_driver = { + .name = DRIVER_NAME, + .id_table = kraken3_table, + .probe = kraken3_probe, + .remove = kraken3_remove, + .raw_event = kraken3_raw_event, +#ifdef CONFIG_PM + .reset_resume = kraken3_reset_resume, +#endif +}; + +static int __init kraken3_init(void) +{ + return hid_register_driver(&kraken3_driver); +} + +static void __exit kraken3_exit(void) +{ + hid_unregister_driver(&kraken3_driver); +} + +/* When compiled into the kernel, initialize after the HID bus */ +late_initcall(kraken3_init); +module_exit(kraken3_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Jonas Malaco "); +MODULE_AUTHOR("Aleksa Savic "); +MODULE_DESCRIPTION("Hwmon driver for NZXT Kraken X53/X63/X73, Z53/Z63/Z73 coolers"); From ed3e03790c5c9f29f032dde9bb784e198984a759 Mon Sep 17 00:00:00 2001 From: Aleksa Savic Date: Mon, 8 Jan 2024 10:44:50 +0100 Subject: [PATCH 0753/1038] hwmon: Add driver for ASUS ROG RYUJIN II 360 AIO cooler This driver exposes hardware sensors of the ASUS ROG RYUJIN II 360 all-in-one CPU liquid cooler, which communicates through a proprietary USB HID protocol. Report offsets were initially discovered in [1] by Florian Freudiger. Available sensors are pump, internal and external (controller) fan speed in RPM, their duties in PWM, as well as coolant temperature. Attaching external fans to the controller is optional and allows them to be controlled from the device. If not connected, the fan-related sensors will report zeroes. The controller is a separate hardware unit that comes bundled with the AIO and connects to it to allow fan control. The addressable LCD screen is not supported in this driver and should be controlled through userspace tools. [1]: https://github.com/liquidctl/liquidctl/pull/653 Tested-by: Florian Freudiger Signed-off-by: Aleksa Savic Link: https://lore.kernel.org/r/20240108094453.22986-1-savicaleksa83@gmail.com [groeck: Add HID dependency] Signed-off-by: Guenter Roeck --- Documentation/hwmon/asus_rog_ryujin.rst | 47 ++ Documentation/hwmon/index.rst | 1 + MAINTAINERS | 6 + drivers/hwmon/Kconfig | 10 + drivers/hwmon/Makefile | 1 + drivers/hwmon/asus_rog_ryujin.c | 609 ++++++++++++++++++++++++ 6 files changed, 674 insertions(+) create mode 100644 Documentation/hwmon/asus_rog_ryujin.rst create mode 100644 drivers/hwmon/asus_rog_ryujin.c diff --git a/Documentation/hwmon/asus_rog_ryujin.rst b/Documentation/hwmon/asus_rog_ryujin.rst new file mode 100644 index 000000000000..9f77da070022 --- /dev/null +++ b/Documentation/hwmon/asus_rog_ryujin.rst @@ -0,0 +1,47 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver asus_rog_ryujin +============================= + +Supported devices: + +* ASUS ROG RYUJIN II 360 + +Author: Aleksa Savic + +Description +----------- + +This driver enables hardware monitoring support for the listed ASUS ROG RYUJIN +all-in-one CPU liquid coolers. Available sensors are pump, internal and external +(controller) fan speed in RPM, their duties in PWM, as well as coolant temperature. + +Attaching external fans to the controller is optional and allows them to be +controlled from the device. If not connected, the fan-related sensors will +report zeroes. The controller is a separate hardware unit that comes bundled +with the AIO and connects to it to allow fan control. + +The addressable LCD screen is not supported in this driver and should +be controlled through userspace tools. + +Usage notes +----------- + +As these are USB HIDs, the driver can be loaded automatically by the kernel and +supports hot swapping. + +Sysfs entries +------------- + +=========== ============================================= +fan1_input Pump speed (in rpm) +fan2_input Internal fan speed (in rpm) +fan3_input External (controller) fan 1 speed (in rpm) +fan4_input External (controller) fan 2 speed (in rpm) +fan5_input External (controller) fan 3 speed (in rpm) +fan6_input External (controller) fan 4 speed (in rpm) +temp1_input Coolant temperature (in millidegrees Celsius) +pwm1 Pump duty +pwm2 Internal fan duty +pwm3 External (controller) fan duty +=========== ============================================= diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index 6f8a4a7524e8..c19f53d9b3ab 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -46,6 +46,7 @@ Hardware Monitoring Kernel Drivers asc7621 aspeed-pwm-tacho asus_ec_sensors + asus_rog_ryujin asus_wmi_sensors bcm54140 bel-pfe diff --git a/MAINTAINERS b/MAINTAINERS index b9906f88c1b7..8fc0ca8e881c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3181,6 +3181,12 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git F: drivers/platform/x86/asus-tf103c-dock.c +ASUS ROG RYUJIN AIO HARDWARE MONITOR DRIVER +M: Aleksa Savic +L: linux-hwmon@vger.kernel.org +S: Maintained +F: drivers/hwmon/asus_rog_ryujin.c + ASUS WIRELESS RADIO CONTROL DRIVER M: João Paulo Rechi Vita L: platform-driver-x86@vger.kernel.org diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 3904bb297d61..e4b24ad93961 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -301,6 +301,16 @@ config SENSORS_ASC7621 This driver can also be built as a module. If so, the module will be called asc7621. +config SENSORS_ASUS_ROG_RYUJIN + tristate "ASUS ROG RYUJIN II 360 hardware monitoring driver" + depends on HID + help + If you say yes here you get support for the fans and sensors of + the ASUS ROG RYUJIN II 360 AIO CPU liquid cooler. + + This driver can also be built as a module. If so, the module + will be called asus_rog_ryujin. + config SENSORS_AXI_FAN_CONTROL tristate "Analog Devices FAN Control HDL Core driver" help diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 76e6dfef9f24..e3faee7be51a 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -55,6 +55,7 @@ obj-$(CONFIG_SENSORS_ARM_SCPI) += scpi-hwmon.o obj-$(CONFIG_SENSORS_AS370) += as370-hwmon.o obj-$(CONFIG_SENSORS_ASC7621) += asc7621.o obj-$(CONFIG_SENSORS_ASPEED) += aspeed-pwm-tacho.o +obj-$(CONFIG_SENSORS_ASUS_ROG_RYUJIN) += asus_rog_ryujin.o obj-$(CONFIG_SENSORS_ATXP1) += atxp1.o obj-$(CONFIG_SENSORS_AXI_FAN_CONTROL) += axi-fan-control.o obj-$(CONFIG_SENSORS_BT1_PVT) += bt1-pvt.o diff --git a/drivers/hwmon/asus_rog_ryujin.c b/drivers/hwmon/asus_rog_ryujin.c new file mode 100644 index 000000000000..f8b20346a995 --- /dev/null +++ b/drivers/hwmon/asus_rog_ryujin.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * hwmon driver for Asus ROG Ryujin II 360 AIO cooler. + * + * Copyright 2024 Aleksa Savic + */ + +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "asus_rog_ryujin" + +#define USB_VENDOR_ID_ASUS_ROG 0x0b05 +#define USB_PRODUCT_ID_RYUJIN_AIO 0x1988 /* ASUS ROG RYUJIN II 360 */ + +#define STATUS_VALIDITY 1500 /* ms */ +#define MAX_REPORT_LENGTH 65 + +/* Cooler status report offsets */ +#define RYUJIN_TEMP_SENSOR_1 3 +#define RYUJIN_TEMP_SENSOR_2 4 +#define RYUJIN_PUMP_SPEED 5 +#define RYUJIN_INTERNAL_FAN_SPEED 7 + +/* Cooler duty report offsets */ +#define RYUJIN_PUMP_DUTY 4 +#define RYUJIN_INTERNAL_FAN_DUTY 5 + +/* Controller status (speeds) report offsets */ +#define RYUJIN_CONTROLLER_SPEED_1 5 +#define RYUJIN_CONTROLLER_SPEED_2 7 +#define RYUJIN_CONTROLLER_SPEED_3 9 +#define RYUJIN_CONTROLLER_SPEED_4 3 + +/* Controller duty report offsets */ +#define RYUJIN_CONTROLLER_DUTY 4 + +/* Control commands and their inner offsets */ +#define RYUJIN_CMD_PREFIX 0xEC + +static const u8 get_cooler_status_cmd[] = { RYUJIN_CMD_PREFIX, 0x99 }; +static const u8 get_cooler_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0x9A }; +static const u8 get_controller_speed_cmd[] = { RYUJIN_CMD_PREFIX, 0xA0 }; +static const u8 get_controller_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0xA1 }; + +#define RYUJIN_SET_COOLER_PUMP_DUTY_OFFSET 3 +#define RYUJIN_SET_COOLER_FAN_DUTY_OFFSET 4 +static const u8 set_cooler_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0x1A, 0x00, 0x00, 0x00 }; + +#define RYUJIN_SET_CONTROLLER_FAN_DUTY_OFFSET 4 +static const u8 set_controller_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0x21, 0x00, 0x00, 0x00 }; + +/* Command lengths */ +#define GET_CMD_LENGTH 2 /* Same length for all get commands */ +#define SET_CMD_LENGTH 5 /* Same length for all set commands */ + +/* Command response headers */ +#define RYUJIN_GET_COOLER_STATUS_CMD_RESPONSE 0x19 +#define RYUJIN_GET_COOLER_DUTY_CMD_RESPONSE 0x1A +#define RYUJIN_GET_CONTROLLER_SPEED_CMD_RESPONSE 0x20 +#define RYUJIN_GET_CONTROLLER_DUTY_CMD_RESPONSE 0x21 + +static const char *const rog_ryujin_temp_label[] = { + "Coolant temp" +}; + +static const char *const rog_ryujin_speed_label[] = { + "Pump speed", + "Internal fan speed", + "Controller fan 1 speed", + "Controller fan 2 speed", + "Controller fan 3 speed", + "Controller fan 4 speed", +}; + +struct rog_ryujin_data { + struct hid_device *hdev; + struct device *hwmon_dev; + /* For locking access to buffer */ + struct mutex buffer_lock; + /* For queueing multiple readers */ + struct mutex status_report_request_mutex; + /* For reinitializing the completions below */ + spinlock_t status_report_request_lock; + struct completion cooler_status_received; + struct completion controller_status_received; + struct completion cooler_duty_received; + struct completion controller_duty_received; + struct completion cooler_duty_set; + struct completion controller_duty_set; + + /* Sensor data */ + s32 temp_input[1]; + u16 speed_input[6]; /* Pump, internal fan and four controller fan speeds in RPM */ + u8 duty_input[3]; /* Pump, internal fan and controller fan duty in PWM */ + + u8 *buffer; + unsigned long updated; /* jiffies */ +}; + +static int rog_ryujin_percent_to_pwm(u16 val) +{ + return DIV_ROUND_CLOSEST(val * 255, 100); +} + +static int rog_ryujin_pwm_to_percent(long val) +{ + return DIV_ROUND_CLOSEST(val * 100, 255); +} + +static umode_t rog_ryujin_is_visible(const void *data, + enum hwmon_sensor_types type, u32 attr, int channel) +{ + switch (type) { + case hwmon_temp: + switch (attr) { + case hwmon_temp_label: + case hwmon_temp_input: + return 0444; + default: + break; + } + break; + case hwmon_fan: + switch (attr) { + case hwmon_fan_label: + case hwmon_fan_input: + return 0444; + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + return 0644; + default: + break; + } + break; + default: + break; + } + + return 0; +} + +/* Writes the command to the device with the rest of the report filled with zeroes */ +static int rog_ryujin_write_expanded(struct rog_ryujin_data *priv, const u8 *cmd, int cmd_length) +{ + int ret; + + mutex_lock(&priv->buffer_lock); + + memcpy_and_pad(priv->buffer, MAX_REPORT_LENGTH, cmd, cmd_length, 0x00); + ret = hid_hw_output_report(priv->hdev, priv->buffer, MAX_REPORT_LENGTH); + + mutex_unlock(&priv->buffer_lock); + return ret; +} + +/* Assumes priv->status_report_request_mutex is locked */ +static int rog_ryujin_execute_cmd(struct rog_ryujin_data *priv, const u8 *cmd, int cmd_length, + struct completion *status_completion) +{ + int ret; + + /* + * Disable raw event parsing for a moment to safely reinitialize the + * completion. Reinit is done because hidraw could have triggered + * the raw event parsing and marked the passed in completion as done. + */ + spin_lock_bh(&priv->status_report_request_lock); + reinit_completion(status_completion); + spin_unlock_bh(&priv->status_report_request_lock); + + /* Send command for getting data */ + ret = rog_ryujin_write_expanded(priv, cmd, cmd_length); + if (ret < 0) + return ret; + + ret = wait_for_completion_interruptible_timeout(status_completion, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + + return 0; +} + +static int rog_ryujin_get_status(struct rog_ryujin_data *priv) +{ + int ret = mutex_lock_interruptible(&priv->status_report_request_mutex); + + if (ret < 0) + return ret; + + if (!time_after(jiffies, priv->updated + msecs_to_jiffies(STATUS_VALIDITY))) { + /* Data is up to date */ + goto unlock_and_return; + } + + /* Retrieve cooler status */ + ret = + rog_ryujin_execute_cmd(priv, get_cooler_status_cmd, GET_CMD_LENGTH, + &priv->cooler_status_received); + if (ret < 0) + goto unlock_and_return; + + /* Retrieve controller status (speeds) */ + ret = + rog_ryujin_execute_cmd(priv, get_controller_speed_cmd, GET_CMD_LENGTH, + &priv->controller_status_received); + if (ret < 0) + goto unlock_and_return; + + /* Retrieve cooler duty */ + ret = + rog_ryujin_execute_cmd(priv, get_cooler_duty_cmd, GET_CMD_LENGTH, + &priv->cooler_duty_received); + if (ret < 0) + goto unlock_and_return; + + /* Retrieve controller duty */ + ret = + rog_ryujin_execute_cmd(priv, get_controller_duty_cmd, GET_CMD_LENGTH, + &priv->controller_duty_received); + if (ret < 0) + goto unlock_and_return; + + priv->updated = jiffies; + +unlock_and_return: + mutex_unlock(&priv->status_report_request_mutex); + if (ret < 0) + return ret; + + return 0; +} + +static int rog_ryujin_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct rog_ryujin_data *priv = dev_get_drvdata(dev); + int ret = rog_ryujin_get_status(priv); + + if (ret < 0) + return ret; + + switch (type) { + case hwmon_temp: + *val = priv->temp_input[channel]; + break; + case hwmon_fan: + *val = priv->speed_input[channel]; + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + *val = priv->duty_input[channel]; + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; /* unreachable */ + } + + return 0; +} + +static int rog_ryujin_read_string(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + switch (type) { + case hwmon_temp: + *str = rog_ryujin_temp_label[channel]; + break; + case hwmon_fan: + *str = rog_ryujin_speed_label[channel]; + break; + default: + return -EOPNOTSUPP; /* unreachable */ + } + + return 0; +} + +static int rog_ryujin_write_fixed_duty(struct rog_ryujin_data *priv, int channel, int val) +{ + u8 set_cmd[SET_CMD_LENGTH]; + int ret; + + if (channel < 2) { + /* + * Retrieve cooler duty since both pump and internal fan are set + * together, then write back with one of them modified. + */ + ret = mutex_lock_interruptible(&priv->status_report_request_mutex); + if (ret < 0) + return ret; + ret = + rog_ryujin_execute_cmd(priv, get_cooler_duty_cmd, GET_CMD_LENGTH, + &priv->cooler_duty_received); + if (ret < 0) + goto unlock_and_return; + + memcpy(set_cmd, set_cooler_duty_cmd, SET_CMD_LENGTH); + + /* Cooler duties are set as 0-100% */ + val = rog_ryujin_pwm_to_percent(val); + + if (channel == 0) { + /* Cooler pump duty */ + set_cmd[RYUJIN_SET_COOLER_PUMP_DUTY_OFFSET] = val; + set_cmd[RYUJIN_SET_COOLER_FAN_DUTY_OFFSET] = + rog_ryujin_pwm_to_percent(priv->duty_input[1]); + } else if (channel == 1) { + /* Cooler internal fan duty */ + set_cmd[RYUJIN_SET_COOLER_PUMP_DUTY_OFFSET] = + rog_ryujin_pwm_to_percent(priv->duty_input[0]); + set_cmd[RYUJIN_SET_COOLER_FAN_DUTY_OFFSET] = val; + } + + ret = rog_ryujin_execute_cmd(priv, set_cmd, SET_CMD_LENGTH, &priv->cooler_duty_set); +unlock_and_return: + mutex_unlock(&priv->status_report_request_mutex); + if (ret < 0) + return ret; + } else { + /* + * Controller fan duty (channel == 2). No need to retrieve current + * duty, so just send the command. + */ + memcpy(set_cmd, set_controller_duty_cmd, SET_CMD_LENGTH); + set_cmd[RYUJIN_SET_CONTROLLER_FAN_DUTY_OFFSET] = val; + + ret = + rog_ryujin_execute_cmd(priv, set_cmd, SET_CMD_LENGTH, + &priv->controller_duty_set); + if (ret < 0) + return ret; + } + + /* Lock onto this value until next refresh cycle */ + priv->duty_input[channel] = val; + + return 0; +} + +static int rog_ryujin_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, + long val) +{ + struct rog_ryujin_data *priv = dev_get_drvdata(dev); + int ret; + + switch (type) { + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + if (val < 0 || val > 255) + return -EINVAL; + + ret = rog_ryujin_write_fixed_duty(priv, channel, val); + if (ret < 0) + return ret; + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static const struct hwmon_ops rog_ryujin_hwmon_ops = { + .is_visible = rog_ryujin_is_visible, + .read = rog_ryujin_read, + .read_string = rog_ryujin_read_string, + .write = rog_ryujin_write +}; + +static const struct hwmon_channel_info *rog_ryujin_info[] = { + HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | HWMON_T_LABEL), + HWMON_CHANNEL_INFO(fan, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL), + HWMON_CHANNEL_INFO(pwm, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT), + NULL +}; + +static const struct hwmon_chip_info rog_ryujin_chip_info = { + .ops = &rog_ryujin_hwmon_ops, + .info = rog_ryujin_info, +}; + +static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, + int size) +{ + struct rog_ryujin_data *priv = hid_get_drvdata(hdev); + + if (data[0] != RYUJIN_CMD_PREFIX) + return 0; + + if (data[1] == RYUJIN_GET_COOLER_STATUS_CMD_RESPONSE) { + /* Received coolant temp and speeds of pump and internal fan */ + priv->temp_input[0] = + data[RYUJIN_TEMP_SENSOR_1] * 1000 + data[RYUJIN_TEMP_SENSOR_2] * 100; + priv->speed_input[0] = get_unaligned_le16(data + RYUJIN_PUMP_SPEED); + priv->speed_input[1] = get_unaligned_le16(data + RYUJIN_INTERNAL_FAN_SPEED); + + if (!completion_done(&priv->cooler_status_received)) + complete_all(&priv->cooler_status_received); + } else if (data[1] == RYUJIN_GET_CONTROLLER_SPEED_CMD_RESPONSE) { + /* Received speeds of four fans attached to the controller */ + priv->speed_input[2] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_1); + priv->speed_input[3] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_2); + priv->speed_input[4] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_3); + priv->speed_input[5] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_4); + + if (!completion_done(&priv->controller_status_received)) + complete_all(&priv->controller_status_received); + } else if (data[1] == RYUJIN_GET_COOLER_DUTY_CMD_RESPONSE) { + /* Received report for pump and internal fan duties (in %) */ + if (data[RYUJIN_PUMP_DUTY] == 0 && data[RYUJIN_INTERNAL_FAN_DUTY] == 0) { + /* + * We received a report with zeroes for duty in both places. + * The device returns this as a confirmation that setting values + * is successful. If we initiated a write, mark it as complete. + */ + if (!completion_done(&priv->cooler_duty_set)) + complete_all(&priv->cooler_duty_set); + else if (!completion_done(&priv->cooler_duty_received)) + /* + * We didn't initiate a write, but received both zeroes. + * This means that either both duties are actually zero, + * or that we received a success report caused by userspace. + * We're expecting a report, so parse it. + */ + goto read_cooler_duty; + return 0; + } +read_cooler_duty: + priv->duty_input[0] = rog_ryujin_percent_to_pwm(data[RYUJIN_PUMP_DUTY]); + priv->duty_input[1] = rog_ryujin_percent_to_pwm(data[RYUJIN_INTERNAL_FAN_DUTY]); + + if (!completion_done(&priv->cooler_duty_received)) + complete_all(&priv->cooler_duty_received); + } else if (data[1] == RYUJIN_GET_CONTROLLER_DUTY_CMD_RESPONSE) { + /* Received report for controller duty for fans (in PWM) */ + if (data[RYUJIN_CONTROLLER_DUTY] == 0) { + /* + * We received a report with a zero for duty. The device returns this as + * a confirmation that setting the controller duty value was successful. + * If we initiated a write, mark it as complete. + */ + if (!completion_done(&priv->controller_duty_set)) + complete_all(&priv->controller_duty_set); + else if (!completion_done(&priv->controller_duty_received)) + /* + * We didn't initiate a write, but received a zero for duty. + * This means that either the duty is actually zero, or that + * we received a success report caused by userspace. + * We're expecting a report, so parse it. + */ + goto read_controller_duty; + return 0; + } +read_controller_duty: + priv->duty_input[2] = data[RYUJIN_CONTROLLER_DUTY]; + + if (!completion_done(&priv->controller_duty_received)) + complete_all(&priv->controller_duty_received); + } + + return 0; +} + +static int rog_ryujin_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + struct rog_ryujin_data *priv; + int ret; + + priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->hdev = hdev; + hid_set_drvdata(hdev, priv); + + /* + * Initialize priv->updated to STATUS_VALIDITY seconds in the past, making + * the initial empty data invalid for rog_ryujin_read() without the need for + * a special case there. + */ + priv->updated = jiffies - msecs_to_jiffies(STATUS_VALIDITY); + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "hid parse failed with %d\n", ret); + return ret; + } + + /* Enable hidraw so existing user-space tools can continue to work */ + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) { + hid_err(hdev, "hid hw start failed with %d\n", ret); + return ret; + } + + ret = hid_hw_open(hdev); + if (ret) { + hid_err(hdev, "hid hw open failed with %d\n", ret); + goto fail_and_stop; + } + + priv->buffer = devm_kzalloc(&hdev->dev, MAX_REPORT_LENGTH, GFP_KERNEL); + if (!priv->buffer) { + ret = -ENOMEM; + goto fail_and_close; + } + + mutex_init(&priv->status_report_request_mutex); + mutex_init(&priv->buffer_lock); + spin_lock_init(&priv->status_report_request_lock); + init_completion(&priv->cooler_status_received); + init_completion(&priv->controller_status_received); + init_completion(&priv->cooler_duty_received); + init_completion(&priv->controller_duty_received); + init_completion(&priv->cooler_duty_set); + init_completion(&priv->controller_duty_set); + + priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "rog_ryujin", + priv, &rog_ryujin_chip_info, NULL); + if (IS_ERR(priv->hwmon_dev)) { + ret = PTR_ERR(priv->hwmon_dev); + hid_err(hdev, "hwmon registration failed with %d\n", ret); + goto fail_and_close; + } + + return 0; + +fail_and_close: + hid_hw_close(hdev); +fail_and_stop: + hid_hw_stop(hdev); + return ret; +} + +static void rog_ryujin_remove(struct hid_device *hdev) +{ + struct rog_ryujin_data *priv = hid_get_drvdata(hdev); + + hwmon_device_unregister(priv->hwmon_dev); + + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static const struct hid_device_id rog_ryujin_table[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_ASUS_ROG, USB_PRODUCT_ID_RYUJIN_AIO) }, + { } +}; + +MODULE_DEVICE_TABLE(hid, rog_ryujin_table); + +static struct hid_driver rog_ryujin_driver = { + .name = "rog_ryujin", + .id_table = rog_ryujin_table, + .probe = rog_ryujin_probe, + .remove = rog_ryujin_remove, + .raw_event = rog_ryujin_raw_event, +}; + +static int __init rog_ryujin_init(void) +{ + return hid_register_driver(&rog_ryujin_driver); +} + +static void __exit rog_ryujin_exit(void) +{ + hid_unregister_driver(&rog_ryujin_driver); +} + +/* When compiled into the kernel, initialize after the HID bus */ +late_initcall(rog_ryujin_init); +module_exit(rog_ryujin_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Aleksa Savic "); +MODULE_DESCRIPTION("Hwmon driver for Asus ROG Ryujin II 360 AIO cooler"); From c8c2074020a878ca6c6106a2cf224b536247e11a Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:37 +0800 Subject: [PATCH 0754/1038] hwmon: (coretemp) Introduce enum for attr index Introduce enum coretemp_attr_index to better describe the index of each sensor attribute. No functional change. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-5-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index b8fc8d1ef20d..32f99cf6308b 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -43,10 +43,18 @@ MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius"); #define BASE_SYSFS_ATTR_NO 2 /* Sysfs Base attr no for coretemp */ #define NUM_REAL_CORES 512 /* Number of Real cores per cpu */ #define CORETEMP_NAME_LENGTH 28 /* String Length of attrs */ -#define MAX_CORE_ATTRS 4 /* Maximum no of basic attrs */ -#define TOTAL_ATTRS (MAX_CORE_ATTRS + 1) #define MAX_CORE_DATA (NUM_REAL_CORES + BASE_SYSFS_ATTR_NO) +enum coretemp_attr_index { + ATTR_LABEL, + ATTR_CRIT_ALARM, + ATTR_TEMP, + ATTR_TJMAX, + ATTR_TTARGET, + MAX_CORE_ATTRS = ATTR_TJMAX + 1, /* Maximum no of basic attrs */ + TOTAL_ATTRS = ATTR_TTARGET + 1 /* Maximum no of possible attrs */ +}; + #ifdef CONFIG_SMP #define for_each_sibling(i, cpu) \ for_each_cpu(i, topology_sibling_cpumask(cpu)) From 25f8e01baa05dfeb1b477112b1fec94f768f27b4 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:38 +0800 Subject: [PATCH 0755/1038] hwmon: (coretemp) Remove unnecessary dependency of array index When sensor_device_attribute pointer is available, use container_of() to get the temp_data address. This removes the unnecessary dependency of cached index in pdata->core_data[]. No functional change. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-6-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 32f99cf6308b..9a7bfc046c72 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -342,7 +342,7 @@ static ssize_t show_label(struct device *dev, { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_LABEL]); if (tdata->is_pkg_data) return sprintf(buf, "Package id %u\n", pdata->pkg_id); @@ -355,8 +355,7 @@ static ssize_t show_crit_alarm(struct device *dev, { u32 eax, edx; struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_CRIT_ALARM]); mutex_lock(&tdata->update_lock); rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx); @@ -369,8 +368,7 @@ static ssize_t show_tjmax(struct device *dev, struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_TJMAX]); int tjmax; mutex_lock(&tdata->update_lock); @@ -384,8 +382,7 @@ static ssize_t show_ttarget(struct device *dev, struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_TTARGET]); int ttarget; mutex_lock(&tdata->update_lock); @@ -402,8 +399,7 @@ static ssize_t show_temp(struct device *dev, { u32 eax, edx; struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_TEMP]); int tjmax; mutex_lock(&tdata->update_lock); @@ -426,8 +422,7 @@ static ssize_t show_temp(struct device *dev, return sprintf(buf, "%d\n", tdata->temp); } -static int create_core_attrs(struct temp_data *tdata, struct device *dev, - int index) +static int create_core_attrs(struct temp_data *tdata, struct device *dev) { int i; static ssize_t (*const rd_ptr[TOTAL_ATTRS]) (struct device *dev, @@ -452,7 +447,6 @@ static int create_core_attrs(struct temp_data *tdata, struct device *dev, tdata->sd_attrs[i].dev_attr.attr.name = tdata->attr_name[i]; tdata->sd_attrs[i].dev_attr.attr.mode = 0444; tdata->sd_attrs[i].dev_attr.show = rd_ptr[i]; - tdata->sd_attrs[i].index = index; tdata->attrs[i] = &tdata->sd_attrs[i].dev_attr.attr; } tdata->attr_group.attrs = tdata->attrs; @@ -557,7 +551,7 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, pdata->core_data[index] = tdata; /* Create sysfs interfaces */ - err = create_core_attrs(tdata, pdata->hwmon_dev, index); + err = create_core_attrs(tdata, pdata->hwmon_dev); if (err) goto exit_free; From 18d8f5583388f38094b3b17ad3149d2a93a23646 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:39 +0800 Subject: [PATCH 0756/1038] hwmon: (coretemp) Replace sensor_device_attribute with device_attribute Replace sensor_device_attribute with device_attribute because sensor_device_attribute->index is no longer used. No functional change. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-7-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 9a7bfc046c72..cdd1e069d5c1 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -85,7 +85,7 @@ struct temp_data { u32 status_reg; int attr_size; bool is_pkg_data; - struct sensor_device_attribute sd_attrs[TOTAL_ATTRS]; + struct device_attribute sd_attrs[TOTAL_ATTRS]; char attr_name[TOTAL_ATTRS][CORETEMP_NAME_LENGTH]; struct attribute *attrs[TOTAL_ATTRS + 1]; struct attribute_group attr_group; @@ -340,9 +340,8 @@ static struct platform_device **zone_devices; static ssize_t show_label(struct device *dev, struct device_attribute *devattr, char *buf) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_LABEL]); + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_LABEL]); if (tdata->is_pkg_data) return sprintf(buf, "Package id %u\n", pdata->pkg_id); @@ -354,8 +353,8 @@ static ssize_t show_crit_alarm(struct device *dev, struct device_attribute *devattr, char *buf) { u32 eax, edx; - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_CRIT_ALARM]); + struct temp_data *tdata = container_of(devattr, struct temp_data, + sd_attrs[ATTR_CRIT_ALARM]); mutex_lock(&tdata->update_lock); rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx); @@ -367,8 +366,7 @@ static ssize_t show_crit_alarm(struct device *dev, static ssize_t show_tjmax(struct device *dev, struct device_attribute *devattr, char *buf) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_TJMAX]); + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TJMAX]); int tjmax; mutex_lock(&tdata->update_lock); @@ -381,8 +379,7 @@ static ssize_t show_tjmax(struct device *dev, static ssize_t show_ttarget(struct device *dev, struct device_attribute *devattr, char *buf) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_TTARGET]); + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TTARGET]); int ttarget; mutex_lock(&tdata->update_lock); @@ -398,8 +395,7 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *devattr, char *buf) { u32 eax, edx; - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct temp_data *tdata = container_of(attr, struct temp_data, sd_attrs[ATTR_TEMP]); + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TEMP]); int tjmax; mutex_lock(&tdata->update_lock); @@ -443,11 +439,11 @@ static int create_core_attrs(struct temp_data *tdata, struct device *dev) snprintf(tdata->attr_name[i], CORETEMP_NAME_LENGTH, "temp%d_%s", attr_no, suffixes[i]); - sysfs_attr_init(&tdata->sd_attrs[i].dev_attr.attr); - tdata->sd_attrs[i].dev_attr.attr.name = tdata->attr_name[i]; - tdata->sd_attrs[i].dev_attr.attr.mode = 0444; - tdata->sd_attrs[i].dev_attr.show = rd_ptr[i]; - tdata->attrs[i] = &tdata->sd_attrs[i].dev_attr.attr; + sysfs_attr_init(&tdata->sd_attrs[i].attr); + tdata->sd_attrs[i].attr.name = tdata->attr_name[i]; + tdata->sd_attrs[i].attr.mode = 0444; + tdata->sd_attrs[i].show = rd_ptr[i]; + tdata->attrs[i] = &tdata->sd_attrs[i].attr; } tdata->attr_group.attrs = tdata->attrs; return sysfs_create_group(&dev->kobj, &tdata->attr_group); From 87eb801925a0a391dce0d7519eb1d45a9d7e0953 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:40 +0800 Subject: [PATCH 0757/1038] hwmon: (coretemp) Remove redundant pdata->cpu_map[] pdata->cpu_map[] saves the mapping between cpu core id and the index in pdata->core_data[]. This is used to find the temp_data structure using cpu_core_id, by traversing the pdata->cpu_map[] array. But the same goal can be achieved by traversing the pdata->core_temp[] array directly. Remove redundant pdata->cpu_map[]. No functional change. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-8-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index cdd1e069d5c1..29ee8e0c0fe9 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -96,7 +96,6 @@ struct temp_data { struct platform_data { struct device *hwmon_dev; u16 pkg_id; - u16 cpu_map[NUM_REAL_CORES]; struct ida ida; struct cpumask cpumask; struct temp_data *core_data[MAX_CORE_DATA]; @@ -517,7 +516,6 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, if (index < 0) return index; - pdata->cpu_map[index] = topology_core_id(cpu); index += BASE_SYSFS_ATTR_NO; } @@ -696,7 +694,7 @@ static int coretemp_cpu_offline(unsigned int cpu) struct platform_device *pdev = coretemp_get_pdev(cpu); struct platform_data *pd; struct temp_data *tdata; - int i, indx = -1, target; + int i, target; /* No need to tear down any interfaces for suspend */ if (cpuhp_tasks_frozen) @@ -707,18 +705,16 @@ static int coretemp_cpu_offline(unsigned int cpu) if (!pd->hwmon_dev) return 0; - for (i = 0; i < NUM_REAL_CORES; i++) { - if (pd->cpu_map[i] == topology_core_id(cpu)) { - indx = i + BASE_SYSFS_ATTR_NO; + for (i = BASE_SYSFS_ATTR_NO; i < MAX_CORE_DATA; i++) { + if (pd->core_data[i] && pd->core_data[i]->cpu_core_id == topology_core_id(cpu)) break; - } } /* Too many cores and this core is not populated, just return */ - if (indx < 0) + if (i == MAX_CORE_DATA) return 0; - tdata = pd->core_data[indx]; + tdata = pd->core_data[i]; cpumask_clear_cpu(cpu, &pd->cpumask); @@ -729,7 +725,7 @@ static int coretemp_cpu_offline(unsigned int cpu) */ target = cpumask_any_and(&pd->cpumask, topology_sibling_cpumask(cpu)); if (target >= nr_cpu_ids) { - coretemp_remove_core(pd, indx); + coretemp_remove_core(pd, i); } else if (tdata && tdata->cpu == cpu) { mutex_lock(&tdata->update_lock); tdata->cpu = target; From b0b01414a26105b68cbce308df076ec66c72bddd Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:41 +0800 Subject: [PATCH 0758/1038] hwmon: (coretemp) Abstract core_temp helpers coretemp driver has an obscure and fragile logic for handling package and core temperature data. Place the logic in newly introduced helpers for further optimizations. No functional change. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-9-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 118 +++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 54 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 29ee8e0c0fe9..a19799a302a2 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -81,6 +81,7 @@ struct temp_data { int tjmax; unsigned long last_updated; unsigned int cpu; + unsigned int index; u32 cpu_core_id; u32 status_reg; int attr_size; @@ -474,14 +475,36 @@ static struct platform_device *coretemp_get_pdev(unsigned int cpu) return NULL; } -static struct temp_data *init_temp_data(unsigned int cpu, int pkg_flag) +static struct temp_data * +init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) { struct temp_data *tdata; + int index; tdata = kzalloc(sizeof(struct temp_data), GFP_KERNEL); if (!tdata) return NULL; + /* + * Get the index of tdata in pdata->core_data[] + * tdata for package: pdata->core_data[1] + * tdata for core: pdata->core_data[2] .. pdata->core_data[NUM_REAL_CORES + 1] + */ + if (pkg_flag) { + index = PKG_SYSFS_ATTR_NO; + } else { + index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); + if (index < 0) { + kfree(tdata); + return NULL; + } + index += BASE_SYSFS_ATTR_NO; + } + /* Index in pdata->core_data[] */ + tdata->index = index; + + pdata->core_data[index] = tdata; + tdata->status_reg = pkg_flag ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; tdata->is_pkg_data = pkg_flag; @@ -492,6 +515,30 @@ static struct temp_data *init_temp_data(unsigned int cpu, int pkg_flag) return tdata; } +static void destroy_temp_data(struct platform_data *pdata, struct temp_data *tdata) +{ + pdata->core_data[tdata->index] = NULL; + if (!tdata->is_pkg_data) + ida_free(&pdata->ida, tdata->index - BASE_SYSFS_ATTR_NO); + kfree(tdata); +} + +static struct temp_data *get_temp_data(struct platform_data *pdata, int cpu) +{ + int i; + + /* cpu < 0 means get pkg temp_data */ + if (cpu < 0) + return pdata->core_data[PKG_SYSFS_ATTR_NO]; + + for (i = BASE_SYSFS_ATTR_NO; i < MAX_CORE_DATA; i++) { + if (pdata->core_data[i] && + pdata->core_data[i]->cpu_core_id == topology_core_id(cpu)) + return pdata->core_data[i]; + } + return NULL; +} + static int create_core_data(struct platform_device *pdev, unsigned int cpu, int pkg_flag) { @@ -499,36 +546,19 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, struct platform_data *pdata = platform_get_drvdata(pdev); struct cpuinfo_x86 *c = &cpu_data(cpu); u32 eax, edx; - int err, index; + int err; if (!housekeeping_cpu(cpu, HK_TYPE_MISC)) return 0; - /* - * Get the index of tdata in pdata->core_data[] - * tdata for package: pdata->core_data[1] - * tdata for core: pdata->core_data[2] .. pdata->core_data[NUM_REAL_CORES + 1] - */ - if (pkg_flag) { - index = PKG_SYSFS_ATTR_NO; - } else { - index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); - if (index < 0) - return index; - - index += BASE_SYSFS_ATTR_NO; - } - - tdata = init_temp_data(cpu, pkg_flag); - if (!tdata) { - err = -ENOMEM; - goto ida_free; - } + tdata = init_temp_data(pdata, cpu, pkg_flag); + if (!tdata) + return -ENOMEM; /* Test if we can access the status register */ err = rdmsr_safe_on_cpu(cpu, tdata->status_reg, &eax, &edx); if (err) - goto exit_free; + goto err; /* Make sure tdata->tjmax is a valid indicator for dynamic/static tjmax */ get_tjmax(tdata, &pdev->dev); @@ -542,20 +572,15 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, if (get_ttarget(tdata, &pdev->dev) >= 0) tdata->attr_size++; - pdata->core_data[index] = tdata; - /* Create sysfs interfaces */ err = create_core_attrs(tdata, pdata->hwmon_dev); if (err) - goto exit_free; + goto err; return 0; -exit_free: - pdata->core_data[index] = NULL; - kfree(tdata); -ida_free: - if (!pkg_flag) - ida_free(&pdata->ida, index - BASE_SYSFS_ATTR_NO); + +err: + destroy_temp_data(pdata, tdata); return err; } @@ -566,10 +591,8 @@ coretemp_add_core(struct platform_device *pdev, unsigned int cpu, int pkg_flag) dev_err(&pdev->dev, "Adding Core %u failed\n", cpu); } -static void coretemp_remove_core(struct platform_data *pdata, int indx) +static void coretemp_remove_core(struct platform_data *pdata, struct temp_data *tdata) { - struct temp_data *tdata = pdata->core_data[indx]; - /* if we errored on add then this is already gone */ if (!tdata) return; @@ -577,11 +600,7 @@ static void coretemp_remove_core(struct platform_data *pdata, int indx) /* Remove the sysfs attributes */ sysfs_remove_group(&pdata->hwmon_dev->kobj, &tdata->attr_group); - kfree(pdata->core_data[indx]); - pdata->core_data[indx] = NULL; - - if (indx >= BASE_SYSFS_ATTR_NO) - ida_free(&pdata->ida, indx - BASE_SYSFS_ATTR_NO); + destroy_temp_data(pdata, tdata); } static int coretemp_device_add(int zoneid) @@ -694,7 +713,7 @@ static int coretemp_cpu_offline(unsigned int cpu) struct platform_device *pdev = coretemp_get_pdev(cpu); struct platform_data *pd; struct temp_data *tdata; - int i, target; + int target; /* No need to tear down any interfaces for suspend */ if (cpuhp_tasks_frozen) @@ -705,16 +724,7 @@ static int coretemp_cpu_offline(unsigned int cpu) if (!pd->hwmon_dev) return 0; - for (i = BASE_SYSFS_ATTR_NO; i < MAX_CORE_DATA; i++) { - if (pd->core_data[i] && pd->core_data[i]->cpu_core_id == topology_core_id(cpu)) - break; - } - - /* Too many cores and this core is not populated, just return */ - if (i == MAX_CORE_DATA) - return 0; - - tdata = pd->core_data[i]; + tdata = get_temp_data(pd, cpu); cpumask_clear_cpu(cpu, &pd->cpumask); @@ -725,7 +735,7 @@ static int coretemp_cpu_offline(unsigned int cpu) */ target = cpumask_any_and(&pd->cpumask, topology_sibling_cpumask(cpu)); if (target >= nr_cpu_ids) { - coretemp_remove_core(pd, i); + coretemp_remove_core(pd, tdata); } else if (tdata && tdata->cpu == cpu) { mutex_lock(&tdata->update_lock); tdata->cpu = target; @@ -735,10 +745,10 @@ static int coretemp_cpu_offline(unsigned int cpu) /* * If all cores in this pkg are offline, remove the interface. */ - tdata = pd->core_data[PKG_SYSFS_ATTR_NO]; + tdata = get_temp_data(pd, -1); if (cpumask_empty(&pd->cpumask)) { if (tdata) - coretemp_remove_core(pd, PKG_SYSFS_ATTR_NO); + coretemp_remove_core(pd, tdata); hwmon_device_unregister(pd->hwmon_dev); pd->hwmon_dev = NULL; return 0; From 326241f71f3d8de3c62236b559ce3f2f201aec4d Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:42 +0800 Subject: [PATCH 0759/1038] hwmon: (coretemp) Split package temp_data and core temp_data Saving package temp_data and core temp_data in one array with different offsets is fragile. Split them and clean up crabbed maths and macros. This also fixes a problem that pdata->core_data[0] was never used. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-10-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index a19799a302a2..1a3b5ae0baca 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -39,11 +39,8 @@ static int force_tjmax; module_param_named(tjmax, force_tjmax, int, 0444); MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius"); -#define PKG_SYSFS_ATTR_NO 1 /* Sysfs attribute for package temp */ -#define BASE_SYSFS_ATTR_NO 2 /* Sysfs Base attr no for coretemp */ #define NUM_REAL_CORES 512 /* Number of Real cores per cpu */ #define CORETEMP_NAME_LENGTH 28 /* String Length of attrs */ -#define MAX_CORE_DATA (NUM_REAL_CORES + BASE_SYSFS_ATTR_NO) enum coretemp_attr_index { ATTR_LABEL, @@ -99,7 +96,8 @@ struct platform_data { u16 pkg_id; struct ida ida; struct cpumask cpumask; - struct temp_data *core_data[MAX_CORE_DATA]; + struct temp_data *pkg_data; + struct temp_data *core_data[NUM_REAL_CORES]; struct device_attribute name_attr; }; @@ -479,31 +477,21 @@ static struct temp_data * init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) { struct temp_data *tdata; - int index; tdata = kzalloc(sizeof(struct temp_data), GFP_KERNEL); if (!tdata) return NULL; - /* - * Get the index of tdata in pdata->core_data[] - * tdata for package: pdata->core_data[1] - * tdata for core: pdata->core_data[2] .. pdata->core_data[NUM_REAL_CORES + 1] - */ if (pkg_flag) { - index = PKG_SYSFS_ATTR_NO; + pdata->pkg_data = tdata; } else { - index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); - if (index < 0) { + tdata->index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); + if (tdata->index < 0) { kfree(tdata); return NULL; } - index += BASE_SYSFS_ATTR_NO; + pdata->core_data[tdata->index] = tdata; } - /* Index in pdata->core_data[] */ - tdata->index = index; - - pdata->core_data[index] = tdata; tdata->status_reg = pkg_flag ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; @@ -517,9 +505,12 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) static void destroy_temp_data(struct platform_data *pdata, struct temp_data *tdata) { - pdata->core_data[tdata->index] = NULL; - if (!tdata->is_pkg_data) - ida_free(&pdata->ida, tdata->index - BASE_SYSFS_ATTR_NO); + if (tdata->is_pkg_data) { + pdata->pkg_data = NULL; + } else { + pdata->core_data[tdata->index] = NULL; + ida_free(&pdata->ida, tdata->index); + } kfree(tdata); } @@ -529,9 +520,9 @@ static struct temp_data *get_temp_data(struct platform_data *pdata, int cpu) /* cpu < 0 means get pkg temp_data */ if (cpu < 0) - return pdata->core_data[PKG_SYSFS_ATTR_NO]; + return pdata->pkg_data; - for (i = BASE_SYSFS_ATTR_NO; i < MAX_CORE_DATA; i++) { + for (i = 0; i < NUM_REAL_CORES; i++) { if (pdata->core_data[i] && pdata->core_data[i]->cpu_core_id == topology_core_id(cpu)) return pdata->core_data[i]; From 18b24a5f9ca3f180f0be72b98f170f043a5d8961 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:43 +0800 Subject: [PATCH 0760/1038] hwmon: (coretemp) Remove redundant temp_data->is_pkg_data temp_data->index saves the index in pdata->core_data[]. It is not used by package temp_data. Use temp_data->index as the indicator of package temp_data and remove redundant temp_data->is_pkg_data. No functional change. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-11-rui.zhang@intel.com Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 1a3b5ae0baca..e548f2145449 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -70,19 +70,16 @@ enum coretemp_attr_index { * @status_reg: One of IA32_THERM_STATUS or IA32_PACKAGE_THERM_STATUS, * from where the temperature values should be read. * @attr_size: Total number of pre-core attrs displayed in the sysfs. - * @is_pkg_data: If this is 1, the temp_data holds pkgtemp data. - * Otherwise, temp_data holds coretemp data. */ struct temp_data { int temp; int tjmax; unsigned long last_updated; unsigned int cpu; - unsigned int index; + int index; u32 cpu_core_id; u32 status_reg; int attr_size; - bool is_pkg_data; struct device_attribute sd_attrs[TOTAL_ATTRS]; char attr_name[TOTAL_ATTRS][CORETEMP_NAME_LENGTH]; struct attribute *attrs[TOTAL_ATTRS + 1]; @@ -149,6 +146,11 @@ static const struct tjmax_model tjmax_model_table[] = { */ }; +static bool is_pkg_temp_data(struct temp_data *tdata) +{ + return tdata->index < 0; +} + static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) { /* The 100C is default for both mobile and non mobile CPUs */ @@ -341,7 +343,7 @@ static ssize_t show_label(struct device *dev, struct platform_data *pdata = dev_get_drvdata(dev); struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_LABEL]); - if (tdata->is_pkg_data) + if (is_pkg_temp_data(tdata)) return sprintf(buf, "Package id %u\n", pdata->pkg_id); return sprintf(buf, "Core %u\n", tdata->cpu_core_id); @@ -433,7 +435,7 @@ static int create_core_attrs(struct temp_data *tdata, struct device *dev) * The attr number is always core id + 2 * The Pkgtemp will always show up as temp1_*, if available */ - int attr_no = tdata->is_pkg_data ? 1 : tdata->cpu_core_id + 2; + int attr_no = is_pkg_temp_data(tdata) ? 1 : tdata->cpu_core_id + 2; snprintf(tdata->attr_name[i], CORETEMP_NAME_LENGTH, "temp%d_%s", attr_no, suffixes[i]); @@ -484,6 +486,8 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) if (pkg_flag) { pdata->pkg_data = tdata; + /* Use tdata->index as indicator of package temp data */ + tdata->index = -1; } else { tdata->index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); if (tdata->index < 0) { @@ -495,7 +499,6 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) tdata->status_reg = pkg_flag ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; - tdata->is_pkg_data = pkg_flag; tdata->cpu = cpu; tdata->cpu_core_id = topology_core_id(cpu); tdata->attr_size = MAX_CORE_ATTRS; @@ -505,7 +508,7 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) static void destroy_temp_data(struct platform_data *pdata, struct temp_data *tdata) { - if (tdata->is_pkg_data) { + if (is_pkg_temp_data(tdata)) { pdata->pkg_data = NULL; } else { pdata->core_data[tdata->index] = NULL; From 1a793caf6f6991716cb07583ed7c27de84ef0cba Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 2 Feb 2024 17:21:44 +0800 Subject: [PATCH 0761/1038] hwmon: (coretemp) Use dynamic allocated memory for core temp_data The total memory needed for saving per core temperature data depends on the number of cores in a package. Using static allocated memory wastes memories on systems with low per package core count. Improve the code to use dynamic allocated memory so that it can be improved further when per package core count information becomes available. No functional change intended. Signed-off-by: Zhang Rui Link: https://lore.kernel.org/r/20240202092144.71180-12-rui.zhang@intel.com [groeck: Fixed continuation line alignment] Signed-off-by: Guenter Roeck --- drivers/hwmon/coretemp.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index e548f2145449..30402de2c889 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -91,10 +91,11 @@ struct temp_data { struct platform_data { struct device *hwmon_dev; u16 pkg_id; + int nr_cores; struct ida ida; struct cpumask cpumask; struct temp_data *pkg_data; - struct temp_data *core_data[NUM_REAL_CORES]; + struct temp_data **core_data; struct device_attribute name_attr; }; @@ -480,6 +481,20 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) { struct temp_data *tdata; + if (!pdata->core_data) { + /* + * TODO: + * The information of actual possible cores in a package is broken for now. + * Will replace hardcoded NUM_REAL_CORES with actual per package core count + * when this information becomes available. + */ + pdata->nr_cores = NUM_REAL_CORES; + pdata->core_data = kcalloc(pdata->nr_cores, sizeof(struct temp_data *), + GFP_KERNEL); + if (!pdata->core_data) + return NULL; + } + tdata = kzalloc(sizeof(struct temp_data), GFP_KERNEL); if (!tdata) return NULL; @@ -489,7 +504,7 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) /* Use tdata->index as indicator of package temp data */ tdata->index = -1; } else { - tdata->index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); + tdata->index = ida_alloc_max(&pdata->ida, pdata->nr_cores - 1, GFP_KERNEL); if (tdata->index < 0) { kfree(tdata); return NULL; @@ -510,6 +525,9 @@ static void destroy_temp_data(struct platform_data *pdata, struct temp_data *tda { if (is_pkg_temp_data(tdata)) { pdata->pkg_data = NULL; + kfree(pdata->core_data); + pdata->core_data = NULL; + pdata->nr_cores = 0; } else { pdata->core_data[tdata->index] = NULL; ida_free(&pdata->ida, tdata->index); @@ -525,7 +543,7 @@ static struct temp_data *get_temp_data(struct platform_data *pdata, int cpu) if (cpu < 0) return pdata->pkg_data; - for (i = 0; i < NUM_REAL_CORES; i++) { + for (i = 0; i < pdata->nr_cores; i++) { if (pdata->core_data[i] && pdata->core_data[i]->cpu_core_id == topology_core_id(cpu)) return pdata->core_data[i]; From 1b2ca93cd0592b1fcbc6f8b64e02552bc15f4bb4 Mon Sep 17 00:00:00 2001 From: Cosmo Chou Date: Tue, 6 Feb 2024 20:54:20 +0800 Subject: [PATCH 0762/1038] hwmon: Add driver for Astera Labs PT5161L retimer This driver implements support for temperature monitoring of Astera Labs PT5161L series PCIe retimer chips. This driver implementation originates from the CSDK available at Link: https://github.com/facebook/openbmc/tree/helium/common/recipes-lib/retimer-v2.14 The communication protocol utilized is based on the I2C/SMBus standard. Signed-off-by: Cosmo Chou Link: https://lore.kernel.org/r/20240206125420.3884300-2-chou.cosmo@gmail.com Signed-off-by: Guenter Roeck --- Documentation/hwmon/index.rst | 1 + Documentation/hwmon/pt5161l.rst | 42 ++ MAINTAINERS | 7 + drivers/hwmon/Kconfig | 10 + drivers/hwmon/Makefile | 1 + drivers/hwmon/pt5161l.c | 667 ++++++++++++++++++++++++++++++++ 6 files changed, 728 insertions(+) create mode 100644 Documentation/hwmon/pt5161l.rst create mode 100644 drivers/hwmon/pt5161l.c diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index c19f53d9b3ab..c4af2a894c42 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -190,6 +190,7 @@ Hardware Monitoring Kernel Drivers pmbus powerz powr1220 + pt5161l pxe1610 pwm-fan q54sj108a2 diff --git a/Documentation/hwmon/pt5161l.rst b/Documentation/hwmon/pt5161l.rst new file mode 100644 index 000000000000..1b97336991ea --- /dev/null +++ b/Documentation/hwmon/pt5161l.rst @@ -0,0 +1,42 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver pt5161l +===================== + +Supported chips: + + * Astera Labs PT5161L + + Prefix: 'pt5161l' + + Addresses scanned: I2C 0x20 - 0x27 + + Datasheet: Not publicly available. + +Authors: Cosmo Chou + +Description +----------- + +This driver implements support for temperature monitoring of Astera Labs +PT5161L series PCIe retimer chips. + +This driver implementation originates from the CSDK available at +https://github.com/facebook/openbmc/tree/helium/common/recipes-lib/retimer-v2.14 +The communication protocol utilized is based on the I2C/SMBus standard. + +Sysfs entries +---------------- + +================ ============================================== +temp1_input Measured temperature (in millidegrees Celsius) +================ ============================================== + +Debugfs entries +---------------- + +================ =============================== +fw_load_status Firmware load status +fw_ver Firmware version of the retimer +heartbeat_status Heartbeat status +================ =============================== diff --git a/MAINTAINERS b/MAINTAINERS index 8fc0ca8e881c..afe08a63f7a9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17698,6 +17698,13 @@ F: fs/pstore/ F: include/linux/pstore* K: \b(pstore|ramoops) +PT5161L HARDWARE MONITOR DRIVER +M: Cosmo Chou +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/pt5161l.rst +F: drivers/hwmon/pt5161l.c + PTP HARDWARE CLOCK SUPPORT M: Richard Cochran L: netdev@vger.kernel.org diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index e4b24ad93961..56260821d658 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1755,6 +1755,16 @@ source "drivers/hwmon/peci/Kconfig" source "drivers/hwmon/pmbus/Kconfig" +config SENSORS_PT5161L + tristate "Astera Labs PT5161L PCIe retimer hardware monitoring" + depends on I2C + help + If you say yes here you get support for temperature monitoring + on the Astera Labs PT5161L PCIe retimer. + + This driver can also be built as a module. If so, the module + will be called pt5161l. + config SENSORS_PWM_FAN tristate "PWM fan" depends on (PWM && OF) || COMPILE_TEST diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index e3faee7be51a..f45c31aff009 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -184,6 +184,7 @@ obj-$(CONFIG_SENSORS_PC87427) += pc87427.o obj-$(CONFIG_SENSORS_PCF8591) += pcf8591.o obj-$(CONFIG_SENSORS_POWERZ) += powerz.o obj-$(CONFIG_SENSORS_POWR1220) += powr1220.o +obj-$(CONFIG_SENSORS_PT5161L) += pt5161l.o obj-$(CONFIG_SENSORS_PWM_FAN) += pwm-fan.o obj-$(CONFIG_SENSORS_RASPBERRYPI_HWMON) += raspberrypi-hwmon.o obj-$(CONFIG_SENSORS_SBTSI) += sbtsi_temp.o diff --git a/drivers/hwmon/pt5161l.c b/drivers/hwmon/pt5161l.c new file mode 100644 index 000000000000..60361e39c474 --- /dev/null +++ b/drivers/hwmon/pt5161l.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Aries current average temp ADC code CSR */ +#define ARIES_CURRENT_AVG_TEMP_ADC_CSR 0x42c + +/* Device Load check register */ +#define ARIES_CODE_LOAD_REG 0x605 +/* Value indicating FW was loaded properly, [3:1] = 3'b111 */ +#define ARIES_LOAD_CODE 0xe + +/* Main Micro Heartbeat register */ +#define ARIES_MM_HEARTBEAT_ADDR 0x923 + +/* Reg offset to specify Address for MM assisted accesses */ +#define ARIES_MM_ASSIST_REG_ADDR_OFFSET 0xd99 +/* Reg offset to specify Command for MM assisted accesses */ +#define ARIES_MM_ASSIST_CMD_OFFSET 0xd9d +/* Reg offset to MM SPARE 0 used specify Address[7:0] */ +#define ARIES_MM_ASSIST_SPARE_0_OFFSET 0xd9f +/* Reg offset to MM SPARE 3 used specify Data Byte 0 */ +#define ARIES_MM_ASSIST_SPARE_3_OFFSET 0xda2 +/* Wide register reads */ +#define ARIES_MM_RD_WIDE_REG_2B 0x1d +#define ARIES_MM_RD_WIDE_REG_3B 0x1e +#define ARIES_MM_RD_WIDE_REG_4B 0x1f +#define ARIES_MM_RD_WIDE_REG_5B 0x20 + +/* Time delay between checking MM status of EEPROM write (microseconds) */ +#define ARIES_MM_STATUS_TIME 5000 + +/* AL Main SRAM DMEM offset (A0) */ +#define AL_MAIN_SRAM_DMEM_OFFSET (64 * 1024) +/* SRAM read command */ +#define AL_TG_RD_LOC_IND_SRAM 0x16 + +/* Offset for main micro FW info */ +#define ARIES_MAIN_MICRO_FW_INFO (96 * 1024 - 128) +/* FW Info (Major) offset location in struct */ +#define ARIES_MM_FW_VERSION_MAJOR 0 +/* FW Info (Minor) offset location in struct */ +#define ARIES_MM_FW_VERSION_MINOR 1 +/* FW Info (Build no.) offset location in struct */ +#define ARIES_MM_FW_VERSION_BUILD 2 + +#define ARIES_TEMP_CAL_CODE_DEFAULT 84 + +/* Struct defining FW version loaded on an Aries device */ +struct pt5161l_fw_ver { + u8 major; + u8 minor; + u16 build; +}; + +/* Each client has this additional data */ +struct pt5161l_data { + struct i2c_client *client; + struct dentry *debugfs; + struct pt5161l_fw_ver fw_ver; + struct mutex lock; /* for atomic I2C transactions */ + bool init_done; + bool code_load_okay; /* indicate if code load reg value is expected */ + bool mm_heartbeat_okay; /* indicate if Main Micro heartbeat is good */ + bool mm_wide_reg_access; /* MM assisted wide register access */ +}; + +static struct dentry *pt5161l_debugfs_dir; + +/* + * Write multiple data bytes to Aries over I2C + */ +static int pt5161l_write_block_data(struct pt5161l_data *data, u32 address, + u8 len, u8 *val) +{ + struct i2c_client *client = data->client; + int ret; + u8 remain_len = len; + u8 xfer_len, curr_len; + u8 buf[16]; + u8 cmd = 0x0F; /* [7]:pec_en, [4:2]:func, [1]:start, [0]:end */ + u8 config = 0x40; /* [6]:cfg_type, [4:1]:burst_len, [0]:address bit16 */ + + while (remain_len > 0) { + if (remain_len > 4) { + curr_len = 4; + remain_len -= 4; + } else { + curr_len = remain_len; + remain_len = 0; + } + + buf[0] = config | (curr_len - 1) << 1 | ((address >> 16) & 0x1); + buf[1] = (address >> 8) & 0xff; + buf[2] = address & 0xff; + memcpy(&buf[3], val, curr_len); + + xfer_len = 3 + curr_len; + ret = i2c_smbus_write_block_data(client, cmd, xfer_len, buf); + if (ret) + return ret; + + val += curr_len; + address += curr_len; + } + + return 0; +} + +/* + * Read multiple data bytes from Aries over I2C + */ +static int pt5161l_read_block_data(struct pt5161l_data *data, u32 address, + u8 len, u8 *val) +{ + struct i2c_client *client = data->client; + int ret, tries; + u8 remain_len = len; + u8 curr_len; + u8 wbuf[16], rbuf[24]; + u8 cmd = 0x08; /* [7]:pec_en, [4:2]:func, [1]:start, [0]:end */ + u8 config = 0x00; /* [6]:cfg_type, [4:1]:burst_len, [0]:address bit16 */ + + while (remain_len > 0) { + if (remain_len > 16) { + curr_len = 16; + remain_len -= 16; + } else { + curr_len = remain_len; + remain_len = 0; + } + + wbuf[0] = config | (curr_len - 1) << 1 | + ((address >> 16) & 0x1); + wbuf[1] = (address >> 8) & 0xff; + wbuf[2] = address & 0xff; + + for (tries = 0; tries < 3; tries++) { + ret = i2c_smbus_write_block_data(client, (cmd | 0x2), 3, + wbuf); + if (ret) + return ret; + + ret = i2c_smbus_read_block_data(client, (cmd | 0x1), + rbuf); + if (ret == curr_len) + break; + } + if (tries >= 3) + return ret; + + memcpy(val, rbuf, curr_len); + val += curr_len; + address += curr_len; + } + + return 0; +} + +static int pt5161l_read_wide_reg(struct pt5161l_data *data, u32 address, + u8 width, u8 *val) +{ + int ret, tries; + u8 buf[8]; + u8 status; + + /* + * Safely access wide registers using mailbox method to prevent + * risking conflict with Aries firmware; otherwise fallback to + * legacy, less secure method. + */ + if (data->mm_wide_reg_access) { + buf[0] = address & 0xff; + buf[1] = (address >> 8) & 0xff; + buf[2] = (address >> 16) & 0x1; + ret = pt5161l_write_block_data(data, + ARIES_MM_ASSIST_SPARE_0_OFFSET, + 3, buf); + if (ret) + return ret; + + /* Set command based on width */ + switch (width) { + case 2: + buf[0] = ARIES_MM_RD_WIDE_REG_2B; + break; + case 3: + buf[0] = ARIES_MM_RD_WIDE_REG_3B; + break; + case 4: + buf[0] = ARIES_MM_RD_WIDE_REG_4B; + break; + case 5: + buf[0] = ARIES_MM_RD_WIDE_REG_5B; + break; + default: + return -EINVAL; + } + ret = pt5161l_write_block_data(data, ARIES_MM_ASSIST_CMD_OFFSET, + 1, buf); + if (ret) + return ret; + + status = 0xff; + for (tries = 0; tries < 100; tries++) { + ret = pt5161l_read_block_data(data, + ARIES_MM_ASSIST_CMD_OFFSET, + 1, &status); + if (ret) + return ret; + + if (status == 0) + break; + + usleep_range(ARIES_MM_STATUS_TIME, + ARIES_MM_STATUS_TIME + 1000); + } + if (status != 0) + return -ETIMEDOUT; + + ret = pt5161l_read_block_data(data, + ARIES_MM_ASSIST_SPARE_3_OFFSET, + width, val); + if (ret) + return ret; + } else { + return pt5161l_read_block_data(data, address, width, val); + } + + return 0; +} + +/* + * Read multiple (up to eight) data bytes from micro SRAM over I2C + */ +static int +pt5161l_read_block_data_main_micro_indirect(struct pt5161l_data *data, + u32 address, u8 len, u8 *val) +{ + int ret, tries; + u8 buf[8]; + u8 i, status; + u32 uind_offs = ARIES_MM_ASSIST_REG_ADDR_OFFSET; + u32 eeprom_base, eeprom_addr; + + /* No multi-byte indirect support here. Hence read a byte at a time */ + eeprom_base = address - AL_MAIN_SRAM_DMEM_OFFSET; + for (i = 0; i < len; i++) { + eeprom_addr = eeprom_base + i; + buf[0] = eeprom_addr & 0xff; + buf[1] = (eeprom_addr >> 8) & 0xff; + buf[2] = (eeprom_addr >> 16) & 0xff; + ret = pt5161l_write_block_data(data, uind_offs, 3, buf); + if (ret) + return ret; + + buf[0] = AL_TG_RD_LOC_IND_SRAM; + ret = pt5161l_write_block_data(data, uind_offs + 4, 1, buf); + if (ret) + return ret; + + status = 0xff; + for (tries = 0; tries < 255; tries++) { + ret = pt5161l_read_block_data(data, uind_offs + 4, 1, + &status); + if (ret) + return ret; + + if (status == 0) + break; + } + if (status != 0) + return -ETIMEDOUT; + + ret = pt5161l_read_block_data(data, uind_offs + 3, 1, buf); + if (ret) + return ret; + + val[i] = buf[0]; + } + + return 0; +} + +/* + * Check firmware load status + */ +static int pt5161l_fw_load_check(struct pt5161l_data *data) +{ + int ret; + u8 buf[8]; + + ret = pt5161l_read_block_data(data, ARIES_CODE_LOAD_REG, 1, buf); + if (ret) + return ret; + + if (buf[0] < ARIES_LOAD_CODE) { + dev_dbg(&data->client->dev, + "Code Load reg unexpected. Not all modules are loaded %x\n", + buf[0]); + data->code_load_okay = false; + } else { + data->code_load_okay = true; + } + + return 0; +} + +/* + * Check main micro heartbeat + */ +static int pt5161l_heartbeat_check(struct pt5161l_data *data) +{ + int ret, tries; + u8 buf[8]; + u8 heartbeat; + bool hb_changed = false; + + ret = pt5161l_read_block_data(data, ARIES_MM_HEARTBEAT_ADDR, 1, buf); + if (ret) + return ret; + + heartbeat = buf[0]; + for (tries = 0; tries < 100; tries++) { + ret = pt5161l_read_block_data(data, ARIES_MM_HEARTBEAT_ADDR, 1, + buf); + if (ret) + return ret; + + if (buf[0] != heartbeat) { + hb_changed = true; + break; + } + } + data->mm_heartbeat_okay = hb_changed; + + return 0; +} + +/* + * Check the status of firmware + */ +static int pt5161l_fwsts_check(struct pt5161l_data *data) +{ + int ret; + u8 buf[8]; + u8 major = 0, minor = 0; + u16 build = 0; + + ret = pt5161l_fw_load_check(data); + if (ret) + return ret; + + ret = pt5161l_heartbeat_check(data); + if (ret) + return ret; + + if (data->code_load_okay && data->mm_heartbeat_okay) { + ret = pt5161l_read_block_data_main_micro_indirect(data, ARIES_MAIN_MICRO_FW_INFO + + ARIES_MM_FW_VERSION_MAJOR, + 1, &major); + if (ret) + return ret; + + ret = pt5161l_read_block_data_main_micro_indirect(data, ARIES_MAIN_MICRO_FW_INFO + + ARIES_MM_FW_VERSION_MINOR, + 1, &minor); + if (ret) + return ret; + + ret = pt5161l_read_block_data_main_micro_indirect(data, ARIES_MAIN_MICRO_FW_INFO + + ARIES_MM_FW_VERSION_BUILD, + 2, buf); + if (ret) + return ret; + build = buf[1] << 8 | buf[0]; + } + data->fw_ver.major = major; + data->fw_ver.minor = minor; + data->fw_ver.build = build; + + return 0; +} + +static int pt5161l_fw_is_at_least(struct pt5161l_data *data, u8 major, u8 minor, + u16 build) +{ + u32 ver = major << 24 | minor << 16 | build; + u32 curr_ver = data->fw_ver.major << 24 | data->fw_ver.minor << 16 | + data->fw_ver.build; + + if (curr_ver >= ver) + return true; + + return false; +} + +static int pt5161l_init_dev(struct pt5161l_data *data) +{ + int ret; + + mutex_lock(&data->lock); + ret = pt5161l_fwsts_check(data); + mutex_unlock(&data->lock); + if (ret) + return ret; + + /* Firmware 2.2.0 enables safe access to wide registers */ + if (pt5161l_fw_is_at_least(data, 2, 2, 0)) + data->mm_wide_reg_access = true; + + data->init_done = true; + + return 0; +} + +static int pt5161l_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct pt5161l_data *data = dev_get_drvdata(dev); + int ret; + u8 buf[8]; + long adc_code; + + switch (attr) { + case hwmon_temp_input: + if (!data->init_done) { + ret = pt5161l_init_dev(data); + if (ret) + return ret; + } + + mutex_lock(&data->lock); + ret = pt5161l_read_wide_reg(data, + ARIES_CURRENT_AVG_TEMP_ADC_CSR, 4, + buf); + mutex_unlock(&data->lock); + if (ret) { + dev_dbg(dev, "Read adc_code failed %d\n", ret); + return ret; + } + + adc_code = buf[3] << 24 | buf[2] << 16 | buf[1] << 8 | buf[0]; + if (adc_code == 0 || adc_code >= 0x3ff) { + dev_dbg(dev, "Invalid adc_code %lx\n", adc_code); + return -EIO; + } + + *val = 110000 + + ((adc_code - (ARIES_TEMP_CAL_CODE_DEFAULT + 250)) * + -320); + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static umode_t pt5161l_is_visible(const void *data, + enum hwmon_sensor_types type, u32 attr, + int channel) +{ + switch (attr) { + case hwmon_temp_input: + return 0444; + default: + break; + } + + return 0; +} + +static const struct hwmon_channel_info *pt5161l_info[] = { + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + NULL +}; + +static const struct hwmon_ops pt5161l_hwmon_ops = { + .is_visible = pt5161l_is_visible, + .read = pt5161l_read, +}; + +static const struct hwmon_chip_info pt5161l_chip_info = { + .ops = &pt5161l_hwmon_ops, + .info = pt5161l_info, +}; + +static ssize_t pt5161l_debugfs_read_fw_ver(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct pt5161l_data *data = file->private_data; + int ret; + char ver[32]; + + mutex_lock(&data->lock); + ret = pt5161l_fwsts_check(data); + mutex_unlock(&data->lock); + if (ret) + return ret; + + ret = snprintf(ver, sizeof(ver), "%u.%u.%u\n", data->fw_ver.major, + data->fw_ver.minor, data->fw_ver.build); + + return simple_read_from_buffer(buf, count, ppos, ver, ret); +} + +static const struct file_operations pt5161l_debugfs_ops_fw_ver = { + .read = pt5161l_debugfs_read_fw_ver, + .open = simple_open, +}; + +static ssize_t pt5161l_debugfs_read_fw_load_sts(struct file *file, + char __user *buf, size_t count, + loff_t *ppos) +{ + struct pt5161l_data *data = file->private_data; + int ret; + bool status = false; + char health[16]; + + mutex_lock(&data->lock); + ret = pt5161l_fw_load_check(data); + mutex_unlock(&data->lock); + if (ret == 0) + status = data->code_load_okay; + + ret = snprintf(health, sizeof(health), "%s\n", + status ? "normal" : "abnormal"); + + return simple_read_from_buffer(buf, count, ppos, health, ret); +} + +static const struct file_operations pt5161l_debugfs_ops_fw_load_sts = { + .read = pt5161l_debugfs_read_fw_load_sts, + .open = simple_open, +}; + +static ssize_t pt5161l_debugfs_read_hb_sts(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct pt5161l_data *data = file->private_data; + int ret; + bool status = false; + char health[16]; + + mutex_lock(&data->lock); + ret = pt5161l_heartbeat_check(data); + mutex_unlock(&data->lock); + if (ret == 0) + status = data->mm_heartbeat_okay; + + ret = snprintf(health, sizeof(health), "%s\n", + status ? "normal" : "abnormal"); + + return simple_read_from_buffer(buf, count, ppos, health, ret); +} + +static const struct file_operations pt5161l_debugfs_ops_hb_sts = { + .read = pt5161l_debugfs_read_hb_sts, + .open = simple_open, +}; + +static int pt5161l_init_debugfs(struct pt5161l_data *data) +{ + data->debugfs = debugfs_create_dir(dev_name(&data->client->dev), + pt5161l_debugfs_dir); + + debugfs_create_file("fw_ver", 0444, data->debugfs, data, + &pt5161l_debugfs_ops_fw_ver); + + debugfs_create_file("fw_load_status", 0444, data->debugfs, data, + &pt5161l_debugfs_ops_fw_load_sts); + + debugfs_create_file("heartbeat_status", 0444, data->debugfs, data, + &pt5161l_debugfs_ops_hb_sts); + + return 0; +} + +static int pt5161l_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct device *hwmon_dev; + struct pt5161l_data *data; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->client = client; + mutex_init(&data->lock); + pt5161l_init_dev(data); + dev_set_drvdata(dev, data); + + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, + data, + &pt5161l_chip_info, + NULL); + + pt5161l_init_debugfs(data); + + return PTR_ERR_OR_ZERO(hwmon_dev); +} + +static void pt5161l_remove(struct i2c_client *client) +{ + struct pt5161l_data *data = i2c_get_clientdata(client); + + debugfs_remove_recursive(data->debugfs); +} + +static const struct of_device_id __maybe_unused pt5161l_of_match[] = { + { .compatible = "asteralabs,pt5161l" }, + {}, +}; +MODULE_DEVICE_TABLE(of, pt5161l_of_match); + +static const struct acpi_device_id __maybe_unused pt5161l_acpi_match[] = { + { "PT5161L", 0 }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, pt5161l_acpi_match); + +static const struct i2c_device_id pt5161l_id[] = { + { "pt5161l", 0 }, + {} +}; +MODULE_DEVICE_TABLE(i2c, pt5161l_id); + +static struct i2c_driver pt5161l_driver = { + .class = I2C_CLASS_HWMON, + .driver = { + .name = "pt5161l", + .of_match_table = of_match_ptr(pt5161l_of_match), + .acpi_match_table = ACPI_PTR(pt5161l_acpi_match), + }, + .probe = pt5161l_probe, + .remove = pt5161l_remove, + .id_table = pt5161l_id, +}; + +static int __init pt5161l_init(void) +{ + pt5161l_debugfs_dir = debugfs_create_dir("pt5161l", NULL); + return i2c_add_driver(&pt5161l_driver); +} + +static void __exit pt5161l_exit(void) +{ + i2c_del_driver(&pt5161l_driver); + debugfs_remove_recursive(pt5161l_debugfs_dir); +} + +module_init(pt5161l_init); +module_exit(pt5161l_exit); + +MODULE_AUTHOR("Cosmo Chou "); +MODULE_DESCRIPTION("Hwmon driver for Astera Labs Aries PCIe retimer"); +MODULE_LICENSE("GPL"); From f16fb6d23b68699eed97fe1edee0d8eecde14a67 Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Wed, 7 Feb 2024 22:17:08 +0100 Subject: [PATCH 0763/1038] hwmon: (chipcap2) fix uninitialized variable in cc2_get_reg_val() The reg_val variable in cc2_get_reg_val() might be used without a known value if cc2_read_reg() fails. That leads to a useless data conversion because the returned error means the read operation failed and the data is not relevant. That makes its initial value irrelevant as well, so skip the data conversion instead. If no error happens, a value is assigned to reg_val and the data conversion is required. Reported-by: Dan Carpenter Closes: https://lore.kernel.org/linux-hwmon/294e4634-89d4-415e-a723-b208d8770d7c@gmail.com/T/#t Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240207-chipcap2_init_vars-v1-1-08cafe43e20e@gmail.com Signed-off-by: Guenter Roeck --- drivers/hwmon/chipcap2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c index a62c507b1042..3b604fc5d8ae 100644 --- a/drivers/hwmon/chipcap2.c +++ b/drivers/hwmon/chipcap2.c @@ -324,7 +324,9 @@ static int cc2_get_reg_val(struct cc2_data *data, u8 reg, long *val) int ret; ret = cc2_read_reg(data, reg, ®_val); - *val = cc2_rh_convert(reg_val); + if (!ret) + *val = cc2_rh_convert(reg_val); + cc2_disable(data); return ret; From efd49b8eef659e0aa962ec1dbf18740e34735219 Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Wed, 7 Feb 2024 22:17:09 +0100 Subject: [PATCH 0764/1038] hwmon: (chipcap2) fix return path in cc2_request_alarm_irqs() The return path can be improved by returning upon first failure. The current implementation would try to register the second interrupt even if the first one failed, which is unnecessary. Moreover, if no irqs are available, the return value should be zero (the driver supports the use case with no interrupts). Currently the initial value is unassigned and that may lead to returning an unknown value if stack variables are not automatically set to zero and no irqs were provided. Reported-by: Dan Carpenter Closes: https://lore.kernel.org/linux-hwmon/294e4634-89d4-415e-a723-b208d8770d7c@gmail.com/T/#t Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240207-chipcap2_init_vars-v1-2-08cafe43e20e@gmail.com Signed-off-by: Guenter Roeck --- drivers/hwmon/chipcap2.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c index 3b604fc5d8ae..6ccceae21f70 100644 --- a/drivers/hwmon/chipcap2.c +++ b/drivers/hwmon/chipcap2.c @@ -670,7 +670,7 @@ static int cc2_request_ready_irq(struct cc2_data *data, struct device *dev) static int cc2_request_alarm_irqs(struct cc2_data *data, struct device *dev) { - int ret; + int ret = 0; data->irq_low = fwnode_irq_get_byname(dev_fwnode(dev), "low"); if (data->irq_low > 0) { @@ -679,8 +679,10 @@ static int cc2_request_alarm_irqs(struct cc2_data *data, struct device *dev) IRQF_ONESHOT | IRQF_TRIGGER_RISING, dev_name(dev), data); - if (!ret) - data->rh_alarm.low_alarm_visible = true; + if (ret) + return ret; + + data->rh_alarm.low_alarm_visible = true; } data->irq_high = fwnode_irq_get_byname(dev_fwnode(dev), "high"); @@ -690,8 +692,10 @@ static int cc2_request_alarm_irqs(struct cc2_data *data, struct device *dev) IRQF_ONESHOT | IRQF_TRIGGER_RISING, dev_name(dev), data); - if (!ret) - data->rh_alarm.high_alarm_visible = true; + if (ret) + return ret; + + data->rh_alarm.high_alarm_visible = true; } return ret; From 692cf83bc3c10c8af4dce0d9d2596c72b1037f6c Mon Sep 17 00:00:00 2001 From: Sebastian Kranz Date: Fri, 9 Feb 2024 10:01:23 +0100 Subject: [PATCH 0765/1038] hwmon: (oxp-sensors) Add support for Ayaneo Air Plus 7320u. Add support for handheld AYANEO AIR Plus with the same EC registers to add proper fan control. Functionality was tested successfully. Signed-off-by: Sebastian Kranz Link: https://lore.kernel.org/r/20240209090157.3232-1-tklightforce@googlemail.com [groeck: Fixed up commit message] Signed-off-by: Guenter Roeck --- Documentation/hwmon/oxp-sensors.rst | 1 + drivers/hwmon/oxp-sensors.c | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/Documentation/hwmon/oxp-sensors.rst b/Documentation/hwmon/oxp-sensors.rst index 3adeb7406243..55b1ef61625e 100644 --- a/Documentation/hwmon/oxp-sensors.rst +++ b/Documentation/hwmon/oxp-sensors.rst @@ -33,6 +33,7 @@ Currently the driver supports the following handhelds: - AOK ZOE A1 PRO - Aya Neo 2 - Aya Neo AIR + - Aya Neo AIR Plus (Mendocino) - Aya Neo AIR Pro - Aya Neo Geek - OneXPlayer AMD diff --git a/drivers/hwmon/oxp-sensors.c b/drivers/hwmon/oxp-sensors.c index ea9602063eab..8d3b0f86cc57 100644 --- a/drivers/hwmon/oxp-sensors.c +++ b/drivers/hwmon/oxp-sensors.c @@ -43,6 +43,7 @@ enum oxp_board { aok_zoe_a1 = 1, aya_neo_2, aya_neo_air, + aya_neo_air_plus_mendo, aya_neo_air_pro, aya_neo_geek, oxp_mini_amd, @@ -98,6 +99,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)aya_neo_air, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "AB05-Mendocino"), + }, + .driver_data = (void *)aya_neo_air_plus_mendo, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), @@ -332,6 +340,7 @@ static int oxp_platform_read(struct device *dev, enum hwmon_sensor_types type, switch (board) { case aya_neo_2: case aya_neo_air: + case aya_neo_air_plus_mendo: case aya_neo_air_pro: case aya_neo_geek: case oxp_mini_amd: @@ -374,6 +383,7 @@ static int oxp_platform_write(struct device *dev, enum hwmon_sensor_types type, switch (board) { case aya_neo_2: case aya_neo_air: + case aya_neo_air_plus_mendo: case aya_neo_air_pro: case aya_neo_geek: case oxp_mini_amd: From 1b5239f70fcd2d7fe86f0f5473006c2896db07a8 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Wed, 14 Feb 2024 15:36:43 +0100 Subject: [PATCH 0766/1038] hwmon: (axi-fan-control) Use device firmware agnostic API Don't directly use OF and use device property APIs. In addition, this makes the probe() code neater and also allow us to move the of_device_id table to it's natural place. While at it, make sure to explicitly include mod_devicetable.h for the of_device_id table. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240214-axi-fan-control-no-of-v1-1-43ca656fe2e3@analog.com Signed-off-by: Guenter Roeck --- drivers/hwmon/axi-fan-control.c | 39 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index 19b9bf3d75ef..8dfe3b6c5a17 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -13,8 +13,9 @@ #include #include #include -#include +#include #include +#include /* register map */ #define ADI_REG_RSTN 0x0080 @@ -368,12 +369,12 @@ static irqreturn_t axi_fan_control_irq_handler(int irq, void *data) } static int axi_fan_control_init(struct axi_fan_control_data *ctl, - const struct device_node *np) + const struct device *dev) { int ret; /* get fan pulses per revolution */ - ret = of_property_read_u32(np, "pulses-per-revolution", &ctl->ppr); + ret = device_property_read_u32(dev, "pulses-per-revolution", &ctl->ppr); if (ret) return ret; @@ -443,25 +444,16 @@ static struct attribute *axi_fan_control_attrs[] = { }; ATTRIBUTE_GROUPS(axi_fan_control); -static const u32 version_1_0_0 = ADI_AXI_PCORE_VER(1, 0, 'a'); - -static const struct of_device_id axi_fan_control_of_match[] = { - { .compatible = "adi,axi-fan-control-1.00.a", - .data = (void *)&version_1_0_0}, - {}, -}; -MODULE_DEVICE_TABLE(of, axi_fan_control_of_match); - static int axi_fan_control_probe(struct platform_device *pdev) { struct axi_fan_control_data *ctl; struct clk *clk; - const struct of_device_id *id; + const unsigned int *id; const char *name = "axi_fan_control"; u32 version; int ret; - id = of_match_node(axi_fan_control_of_match, pdev->dev.of_node); + id = device_get_match_data(&pdev->dev); if (!id) return -EINVAL; @@ -485,18 +477,18 @@ static int axi_fan_control_probe(struct platform_device *pdev) version = axi_ioread(ADI_AXI_REG_VERSION, ctl); if (ADI_AXI_PCORE_VER_MAJOR(version) != - ADI_AXI_PCORE_VER_MAJOR((*(u32 *)id->data))) { + ADI_AXI_PCORE_VER_MAJOR((*id))) { dev_err(&pdev->dev, "Major version mismatch. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n", - ADI_AXI_PCORE_VER_MAJOR((*(u32 *)id->data)), - ADI_AXI_PCORE_VER_MINOR((*(u32 *)id->data)), - ADI_AXI_PCORE_VER_PATCH((*(u32 *)id->data)), + ADI_AXI_PCORE_VER_MAJOR(*id), + ADI_AXI_PCORE_VER_MINOR(*id), + ADI_AXI_PCORE_VER_PATCH(*id), ADI_AXI_PCORE_VER_MAJOR(version), ADI_AXI_PCORE_VER_MINOR(version), ADI_AXI_PCORE_VER_PATCH(version)); return -ENODEV; } - ret = axi_fan_control_init(ctl, pdev->dev.of_node); + ret = axi_fan_control_init(ctl, &pdev->dev); if (ret) { dev_err(&pdev->dev, "Failed to initialize device\n"); return ret; @@ -527,6 +519,15 @@ static int axi_fan_control_probe(struct platform_device *pdev) return 0; } +static const u32 version_1_0_0 = ADI_AXI_PCORE_VER(1, 0, 'a'); + +static const struct of_device_id axi_fan_control_of_match[] = { + { .compatible = "adi,axi-fan-control-1.00.a", + .data = (void *)&version_1_0_0}, + {}, +}; +MODULE_DEVICE_TABLE(of, axi_fan_control_of_match); + static struct platform_driver axi_fan_control_driver = { .driver = { .name = "axi_fan_control_driver", From 0b5f91d47d72d706360777fe03e40eab2f51a94a Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Wed, 14 Feb 2024 15:36:44 +0100 Subject: [PATCH 0767/1038] hwmon: (axi-fan-control) Make use of sysfs_emit() Use sysfs_emit() instead of directly call sprintf(). Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240214-axi-fan-control-no-of-v1-2-43ca656fe2e3@analog.com Signed-off-by: Guenter Roeck --- drivers/hwmon/axi-fan-control.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index 8dfe3b6c5a17..efd42a4f5951 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -84,7 +84,7 @@ static ssize_t axi_fan_control_show(struct device *dev, struct device_attribute temp = DIV_ROUND_CLOSEST_ULL(temp * 509314ULL, 65535) - 280230; - return sprintf(buf, "%u\n", temp); + return sysfs_emit(buf, "%u\n", temp); } static ssize_t axi_fan_control_store(struct device *dev, struct device_attribute *da, From ec823656c1e0e5f49e92ed86cee9fb26585da18e Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Wed, 14 Feb 2024 15:36:45 +0100 Subject: [PATCH 0768/1038] hwmon: (axi-fan-control) Make use of dev_err_probe() Use dev_err_probe() to slightly simplify printing errors during probe. No functional changes intended. Signed-off-by: Nuno Sa Link: https://lore.kernel.org/r/20240214-axi-fan-control-no-of-v1-3-43ca656fe2e3@analog.com Signed-off-by: Guenter Roeck --- drivers/hwmon/axi-fan-control.c | 40 +++++++++++++++------------------ 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index efd42a4f5951..35c862eb158b 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -466,10 +466,9 @@ static int axi_fan_control_probe(struct platform_device *pdev) return PTR_ERR(ctl->base); clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "clk_get failed with %ld\n", PTR_ERR(clk)); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk), + "clk_get failed\n"); ctl->clk_rate = clk_get_rate(clk); if (!ctl->clk_rate) @@ -477,22 +476,20 @@ static int axi_fan_control_probe(struct platform_device *pdev) version = axi_ioread(ADI_AXI_REG_VERSION, ctl); if (ADI_AXI_PCORE_VER_MAJOR(version) != - ADI_AXI_PCORE_VER_MAJOR((*id))) { - dev_err(&pdev->dev, "Major version mismatch. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n", - ADI_AXI_PCORE_VER_MAJOR(*id), - ADI_AXI_PCORE_VER_MINOR(*id), - ADI_AXI_PCORE_VER_PATCH(*id), - ADI_AXI_PCORE_VER_MAJOR(version), - ADI_AXI_PCORE_VER_MINOR(version), - ADI_AXI_PCORE_VER_PATCH(version)); - return -ENODEV; - } + ADI_AXI_PCORE_VER_MAJOR((*id))) + return dev_err_probe(&pdev->dev, -ENODEV, + "Major version mismatch. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n", + ADI_AXI_PCORE_VER_MAJOR(*id), + ADI_AXI_PCORE_VER_MINOR(*id), + ADI_AXI_PCORE_VER_PATCH(*id), + ADI_AXI_PCORE_VER_MAJOR(version), + ADI_AXI_PCORE_VER_MINOR(version), + ADI_AXI_PCORE_VER_PATCH(version)); ret = axi_fan_control_init(ctl, &pdev->dev); - if (ret) { - dev_err(&pdev->dev, "Failed to initialize device\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to initialize device\n"); ctl->hdev = devm_hwmon_device_register_with_info(&pdev->dev, name, @@ -511,10 +508,9 @@ static int axi_fan_control_probe(struct platform_device *pdev) axi_fan_control_irq_handler, IRQF_ONESHOT | IRQF_TRIGGER_HIGH, pdev->driver_override, ctl); - if (ret) { - dev_err(&pdev->dev, "failed to request an irq, %d", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to request an irq\n"); return 0; } From d88cef1d17f62f9a7bcefb8f37d222c8fcc89f6c Mon Sep 17 00:00:00 2001 From: Okan Akyuz Date: Fri, 16 Feb 2024 10:52:12 +0300 Subject: [PATCH 0769/1038] hwmon: (max6620) Update broken Datasheet URL in driver documentation The URL for the MAX6620 datasheet has changed. Update it to reflect the current location. Signed-off-by: Okan Akyuz Link: https://lore.kernel.org/r/20240216075212.69118-1-okanakyuz@okanakyuz.com [groeck: Updated subject and patch description] Signed-off-by: Guenter Roeck --- Documentation/hwmon/max6620.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/hwmon/max6620.rst b/Documentation/hwmon/max6620.rst index 84c1c44d3de4..d70173bf0242 100644 --- a/Documentation/hwmon/max6620.rst +++ b/Documentation/hwmon/max6620.rst @@ -11,7 +11,7 @@ Supported chips: Addresses scanned: none - Datasheet: http://pdfserv.maxim-ic.com/en/ds/MAX6620.pdf + Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/max6620.pdf Authors: - L\. Grunenberg From 4adee4e1a354bd318205afd3f8defc99299fb47a Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Sat, 17 Feb 2024 08:16:51 -0800 Subject: [PATCH 0770/1038] MAINTAINERS: Drop redundant hwmon entries I am listed as maintainer of several individual hardware monitoring drivers and for the hardware monitoring subsystem itself. That is redundant and just bloats the MAINTAINERS file. Drop all the redundant entries. Signed-off-by: Guenter Roeck --- MAINTAINERS | 108 ---------------------------------------------------- 1 file changed, 108 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index afe08a63f7a9..9c14c97d1056 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10493,22 +10493,6 @@ L: linux-fbdev@vger.kernel.org S: Orphan F: drivers/video/fbdev/imsttfb.c -INA209 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml -F: Documentation/hwmon/ina209.rst -F: drivers/hwmon/ina209.c - -INA2XX HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/ina2xx.rst -F: drivers/hwmon/ina2xx.c -F: include/linux/platform_data/ina2xx.h - INDEX OF FURTHER KERNEL DOCUMENTATION M: Carlos Bilbao S: Maintained @@ -11484,14 +11468,6 @@ S: Maintained F: arch/x86/include/asm/jailhouse_para.h F: arch/x86/kernel/jailhouse.c -JC42.4 TEMPERATURE SENSOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/jedec,jc42.yaml -F: Documentation/hwmon/jc42.rst -F: drivers/hwmon/jc42.c - JFS FILESYSTEM M: Dave Kleikamp L: jfs-discussion@lists.sourceforge.net @@ -12557,13 +12533,6 @@ F: Documentation/hwmon/lm90.rst F: drivers/hwmon/lm90.c F: include/dt-bindings/thermal/lm90.h -LM95234 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/lm95234.rst -F: drivers/hwmon/lm95234.c - LME2510 MEDIA DRIVER M: Malcolm Priestley L: linux-media@vger.kernel.org @@ -12767,13 +12736,6 @@ W: https://ez.analog.com/linux-software-drivers F: Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml F: drivers/iio/temperature/ltc2983.c -LTC4261 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/ltc4261.rst -F: drivers/hwmon/ltc4261.c - LTC4282 HARDWARE MONITOR DRIVER M: Nuno Sa L: linux-hwmon@vger.kernel.org @@ -13129,13 +13091,6 @@ S: Maintained F: Documentation/hwmon/max15301.rst F: drivers/hwmon/pmbus/max15301.c -MAX16065 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/max16065.rst -F: drivers/hwmon/max16065.c - MAX2175 SDR TUNER DRIVER M: Ramesh Shanmugasundaram L: linux-media@vger.kernel.org @@ -13160,15 +13115,6 @@ S: Orphan F: Documentation/hwmon/max6650.rst F: drivers/hwmon/max6650.c -MAX6697 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/max6697.txt -F: Documentation/hwmon/max6697.rst -F: drivers/hwmon/max6697.c -F: include/linux/platform_data/max6697.h - MAX9286 QUAD GMSL DESERIALIZER DRIVER M: Jacopo Mondi M: Kieran Bingham @@ -15075,15 +15021,6 @@ M: Samuel Mendoza-Jonas S: Maintained F: net/ncsi/ -NCT6775 HARDWARE MONITOR DRIVER - CORE & PLATFORM DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/nct6775.rst -F: drivers/hwmon/nct6775-core.c -F: drivers/hwmon/nct6775-platform.c -F: drivers/hwmon/nct6775.h - NCT6775 HARDWARE MONITOR DRIVER - I2C DRIVER M: Zev Weiss L: linux-hwmon@vger.kernel.org @@ -17450,35 +17387,6 @@ S: Maintained F: Documentation/hwmon/pm6764tr.rst F: drivers/hwmon/pmbus/pm6764tr.c -PMBUS HARDWARE MONITORING DRIVERS -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -W: http://hwmon.wiki.kernel.org/ -W: http://www.roeck-us.net/linux/drivers/ -T: git git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git -F: Documentation/devicetree/bindings/hwmon/ltc2978.txt -F: Documentation/devicetree/bindings/hwmon/max31785.txt -F: Documentation/hwmon/adm1275.rst -F: Documentation/hwmon/ibm-cffps.rst -F: Documentation/hwmon/ir35221.rst -F: Documentation/hwmon/lm25066.rst -F: Documentation/hwmon/ltc2978.rst -F: Documentation/hwmon/ltc3815.rst -F: Documentation/hwmon/max16064.rst -F: Documentation/hwmon/max20751.rst -F: Documentation/hwmon/max31785.rst -F: Documentation/hwmon/max34440.rst -F: Documentation/hwmon/max8688.rst -F: Documentation/hwmon/pmbus-core.rst -F: Documentation/hwmon/pmbus.rst -F: Documentation/hwmon/tps40422.rst -F: Documentation/hwmon/ucd9000.rst -F: Documentation/hwmon/ucd9200.rst -F: Documentation/hwmon/zl6100.rst -F: drivers/hwmon/pmbus/ -F: include/linux/pmbus.h - PMC SIERRA MaxRAID DRIVER L: linux-scsi@vger.kernel.org S: Orphan @@ -22182,22 +22090,6 @@ F: drivers/mmc/host/renesas_sdhi* F: drivers/mmc/host/tmio_mmc* F: include/linux/mfd/tmio.h -TMP401 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/ti,tmp401.yaml -F: Documentation/hwmon/tmp401.rst -F: drivers/hwmon/tmp401.c - -TMP464 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/ti,tmp464.yaml -F: Documentation/hwmon/tmp464.rst -F: drivers/hwmon/tmp464.c - TMP513 HARDWARE MONITOR DRIVER M: Eric Tremblay L: linux-hwmon@vger.kernel.org From 22d409ead1f4f388e368812731f2f8b8892ed615 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Wed, 21 Feb 2024 07:51:58 -0800 Subject: [PATCH 0771/1038] dt-bindings: hwmon: nuvoton,nct6775: Add compatible value for NCT6799 While NCT6799 is mostly compatible to NCT6798, it needs a separate compatible entry because it is not completely compatible and does require chip specific code in the driver. Signed-off-by: Guenter Roeck Acked-by: Conor Dooley Acked-by: Zev Weiss Link: https://lore.kernel.org/r/20240221155158.2234898-1-linux@roeck-us.net Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml b/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml index 358b262431fc..e3db642878d4 100644 --- a/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml +++ b/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml @@ -25,6 +25,7 @@ properties: - nuvoton,nct6796 - nuvoton,nct6797 - nuvoton,nct6798 + - nuvoton,nct6799 reg: maxItems: 1 From f1ed8af54bba07c81af97a4c999686d2ccafbfb3 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Fri, 23 Feb 2024 16:21:05 +0000 Subject: [PATCH 0772/1038] dt-bindings: hwmon/pmbus: ti,lm25066: document regulators All devices documented in the lm25066 binding are intended for use with a regulator, be that for purely monitoring purposes (lm25056) or, for the other devices, as the controller of that regulator. The binding does not currently allow regulator child nodes, so add one. Each of these devices interacts with only a single regulator and documentation refers to it as "Vout", hence the choice of child node name. Acked-by: Zev Weiss Signed-off-by: Conor Dooley Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240223-tingling-mutt-dd55dd87ff5e@spud Signed-off-by: Guenter Roeck --- .../devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml index da8292bc32f5..4373e9c86e54 100644 --- a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml +++ b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml @@ -34,6 +34,18 @@ properties: Shunt (sense) resistor value in micro-Ohms default: 1000 + regulators: + type: object + + properties: + vout: + $ref: /schemas/regulator/regulator.yaml# + type: object + + unevaluatedProperties: false + + additionalProperties: false + required: - compatible - reg From bad582f9879812bcf74adb520e005051eb021cfb Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Fri, 23 Feb 2024 16:21:06 +0000 Subject: [PATCH 0773/1038] regulator: dt-bindings: promote infineon buck converters to their own binding These devices are regulators may need to make use of the common regulator properties, but these are not permitted while only documented in trivial-devices.yaml Signed-off-by: Conor Dooley Reviewed-by: Krzysztof Kozlowski Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20240223-blabber-obnoxious-353e519541a6@spud [groeck: Dropped empty line at end of patch] Signed-off-by: Guenter Roeck --- .../bindings/regulator/infineon,ir38060.yaml | 45 +++++++++++++++++++ .../devicetree/bindings/trivial-devices.yaml | 8 ---- 2 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml diff --git a/Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml b/Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml new file mode 100644 index 000000000000..e6ffbc2a2298 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/infineon,ir38060.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Infineon Buck Regulators with PMBUS interfaces + +maintainers: + - Not Me. + +allOf: + - $ref: regulator.yaml# + +properties: + compatible: + enum: + - infineon,ir38060 + - infineon,ir38064 + - infineon,ir38164 + - infineon,ir38263 + + reg: + maxItems: 1 + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + regulator@34 { + compatible = "infineon,ir38060"; + reg = <0x34>; + + regulator-min-microvolt = <437500>; + regulator-max-microvolt = <1387500>; + }; + }; diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 842eb65e4c03..2210964faaf6 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -143,14 +143,6 @@ properties: - ibm,cffps2 # Infineon IR36021 digital POL buck controller - infineon,ir36021 - # Infineon IR38060 Voltage Regulator - - infineon,ir38060 - # Infineon IR38064 Voltage Regulator - - infineon,ir38064 - # Infineon IR38164 Voltage Regulator - - infineon,ir38164 - # Infineon IR38263 Voltage Regulator - - infineon,ir38263 # Infineon IRPS5401 Voltage Regulator (PMIC) - infineon,irps5401 # Infineon TLV493D-A1B6 I2C 3D Magnetic Sensor From 8be143b978717ce953c1c42cc57c36cd543334a4 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Fri, 23 Feb 2024 16:21:07 +0000 Subject: [PATCH 0774/1038] hwmon: (pmbus/tda38640) Use PMBUS_REGULATOR_ONE to declare regulator If a chip only provides a single regulator, it should be named 'vout' and not 'vout0'. Declare regulator using PMBUS_REGULATOR_ONE() to make that happen. Cc: Conor Dooley Cc: Naresh Solanki Cc: Patrick Rudolph Signed-off-by: Guenter Roeck Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20240223-catnap-companion-c42fdd8ad110@spud Signed-off-by: Guenter Roeck --- drivers/hwmon/pmbus/tda38640.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/pmbus/tda38640.c b/drivers/hwmon/pmbus/tda38640.c index 09cd114b1736..c31889a036f0 100644 --- a/drivers/hwmon/pmbus/tda38640.c +++ b/drivers/hwmon/pmbus/tda38640.c @@ -15,7 +15,7 @@ #include "pmbus.h" static const struct regulator_desc __maybe_unused tda38640_reg_desc[] = { - PMBUS_REGULATOR("vout", 0), + PMBUS_REGULATOR_ONE("vout"), }; struct tda38640_data { From 8f832d23ead3108eb74afffb5def676f8cebf35c Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Fri, 23 Feb 2024 16:21:08 +0000 Subject: [PATCH 0775/1038] hwmon: (pmbus/lm25066) Use PMBUS_REGULATOR_ONE to declare regulator If a chip only provides a single regulator, it should be named 'vout' and not 'vout0'. Declare regulator using PMBUS_REGULATOR_ONE() to make that happen. Cc: Conor Dooley Cc: Naresh Solanki Cc: Zev Weiss Signed-off-by: Guenter Roeck Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20240223-player-buckskin-01405c5889c4@spud Signed-off-by: Guenter Roeck --- drivers/hwmon/pmbus/lm25066.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c index 3a20df5a43ec..cfffa4cdc0df 100644 --- a/drivers/hwmon/pmbus/lm25066.c +++ b/drivers/hwmon/pmbus/lm25066.c @@ -437,7 +437,7 @@ static int lm25066_write_word_data(struct i2c_client *client, int page, int reg, #if IS_ENABLED(CONFIG_SENSORS_LM25066_REGULATOR) static const struct regulator_desc lm25066_reg_desc[] = { - PMBUS_REGULATOR("vout", 0), + PMBUS_REGULATOR_ONE("vout"), }; #endif From cb7222997e9c64326f4e58d0aad02007225e4975 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Fri, 23 Feb 2024 16:21:09 +0000 Subject: [PATCH 0776/1038] hwmon: (pmbus/ir38064) Use PMBUS_REGULATOR_ONE to declare regulator If a chip only provides a single regulator, it should be named 'vout' and not 'vout0'. Declare regulator using PMBUS_REGULATOR_ONE() to make that happen. Cc: Conor Dooley Cc: Naresh Solanki Cc: Patrick Rudolph Signed-off-by: Guenter Roeck Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20240223-harmless-covenant-9cd3d4f1cfd2@spud Signed-off-by: Guenter Roeck --- drivers/hwmon/pmbus/ir38064.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwmon/pmbus/ir38064.c b/drivers/hwmon/pmbus/ir38064.c index 04185be3fdb6..69e18cb468f6 100644 --- a/drivers/hwmon/pmbus/ir38064.c +++ b/drivers/hwmon/pmbus/ir38064.c @@ -22,7 +22,7 @@ #if IS_ENABLED(CONFIG_SENSORS_IR38064_REGULATOR) static const struct regulator_desc ir38064_reg_desc[] = { - PMBUS_REGULATOR("vout", 0), + PMBUS_REGULATOR_ONE("vout"), }; #endif /* CONFIG_SENSORS_IR38064_REGULATOR */ From d590900b62f0a502931b331f2e7a1748580f403f Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 24 Feb 2024 11:14:50 +0100 Subject: [PATCH 0777/1038] dt-bindings: hwmon: add common properties Add a common hwmon schema for two properties: 1. "label", because Linux hwmon core code parses it. 2. "shunt-resistor-micro-ohms", because several devices already use it. Acked-by: Guenter Roeck Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240224-dt-bindings-hwmon-common-v2-1-b446eecf5480@linaro.org Signed-off-by: Guenter Roeck --- .../bindings/hwmon/hwmon-common.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Documentation/devicetree/bindings/hwmon/hwmon-common.yaml diff --git a/Documentation/devicetree/bindings/hwmon/hwmon-common.yaml b/Documentation/devicetree/bindings/hwmon/hwmon-common.yaml new file mode 100644 index 000000000000..dc86b5c72cf2 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/hwmon-common.yaml @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/hwmon-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hardware Monitoring Devices Common Properties + +maintainers: + - Guenter Roeck + +properties: + label: + description: A descriptive name for this device. + + shunt-resistor-micro-ohms: + description: The value of current sense resistor. + +additionalProperties: true From 67db0ea3ece240b85896a4e4054af1e277815960 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 24 Feb 2024 11:14:51 +0100 Subject: [PATCH 0778/1038] dt-bindings: hwmon: ti,ina2xx: use common hwmon schema Reference common hwmon schema which allows to drop the "label" property. Acked-by: Rob Herring Acked-by: Guenter Roeck Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240224-dt-bindings-hwmon-common-v2-2-b446eecf5480@linaro.org Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml index a099bb71415e..df86c2c92037 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml @@ -36,9 +36,6 @@ properties: "#io-channel-cells": const: 1 - label: - description: A descriptive name for this device. - shunt-resistor: description: Shunt resistor value in micro-Ohm. @@ -73,7 +70,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | From 09458a0e7c8b780f762897e92e9020c9c016b9e9 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 24 Feb 2024 11:14:52 +0100 Subject: [PATCH 0779/1038] dt-bindings: hwmon: adi,adm1275: use common hwmon schema Reference common hwmon schema which allows to drop the "shunt-resistor-micro-ohms" property. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240224-dt-bindings-hwmon-common-v2-3-b446eecf5480@linaro.org Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml index ab87f51c5aef..b68061294964 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml @@ -33,10 +33,6 @@ properties: reg: maxItems: 1 - shunt-resistor-micro-ohms: - description: - Shunt resistor value in micro-Ohm. - adi,volt-curr-sample-average: description: | Number of samples to be used to report voltage and current values. @@ -50,6 +46,7 @@ properties: enum: [1, 2, 4, 8, 16, 32, 64, 128] allOf: + - $ref: hwmon-common.yaml# - if: properties: compatible: @@ -107,7 +104,7 @@ required: - compatible - reg -additionalProperties: false +unevaluatedProperties: false examples: - | From 2988ce56b5610a2802d0974371de53c004ea4888 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 24 Feb 2024 11:14:53 +0100 Subject: [PATCH 0780/1038] dt-bindings: hwmon: lltc,ltc4286: use common hwmon schema Reference common hwmon schema which allows to drop the "shunt-resistor-micro-ohms" property. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240224-dt-bindings-hwmon-common-v2-4-b446eecf5480@linaro.org Signed-off-by: Guenter Roeck --- .../devicetree/bindings/hwmon/lltc,ltc4286.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml b/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml index 98ca163d3486..853df9fef6c8 100644 --- a/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml +++ b/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml @@ -25,15 +25,14 @@ properties: The default is 102.4 volts. type: boolean - shunt-resistor-micro-ohms: - description: - Resistor value micro-ohms. - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | From a095d8c0230b16eb6ea824925d670215624a0590 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 24 Feb 2024 11:14:54 +0100 Subject: [PATCH 0781/1038] dt-bindings: hwmon: reference common hwmon schema Reference common hwmon schema in the bindings which have and customize the "shunt-resistor-micro-ohms" property. While this does not make the binding smaller, it brings common definition of that property and also allows generic "label" (parsed by Linux hwmon core code). Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240224-dt-bindings-hwmon-common-v2-5-b446eecf5480@linaro.org Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml | 5 ++++- Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml | 5 ++++- Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml | 5 ++++- .../devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml | 5 ++++- Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml | 5 ++++- Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml b/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml index 2e45364d0543..be7e9e91a3a8 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml @@ -46,7 +46,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml b/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml index 5cb66e97e816..6401b0a9aff4 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml @@ -31,7 +31,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml b/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml index e62aff670478..8f0095bb7f6e 100644 --- a/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml +++ b/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml @@ -25,7 +25,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml index 4373e9c86e54..a20f140dc79a 100644 --- a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml +++ b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml @@ -50,7 +50,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/hwmon/hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml b/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml index cdd1489e0c54..227858e76058 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml @@ -72,7 +72,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml index ebc8d466c1aa..f58248c29e22 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml @@ -35,7 +35,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | From 5b1d7a0f904b0785d1f6e12770648410093744ac Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sun, 25 Feb 2024 21:28:41 +0100 Subject: [PATCH 0782/1038] hwmon: (sis5595) drop unused DIV_TO_REG function 'DIV_TO_REG' function is not used: sis5595.c:159:18: error: unused function 'DIV_TO_REG' [-Werror,-Wunused-function] Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240225202841.60740-1-krzysztof.kozlowski@linaro.org Signed-off-by: Guenter Roeck --- drivers/hwmon/sis5595.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/hwmon/sis5595.c b/drivers/hwmon/sis5595.c index 641be1f7f9cd..e73b1522f3ce 100644 --- a/drivers/hwmon/sis5595.c +++ b/drivers/hwmon/sis5595.c @@ -153,13 +153,9 @@ static inline s8 TEMP_TO_REG(long val) } /* - * FAN DIV: 1, 2, 4, or 8 (defaults to 2) - * REG: 0, 1, 2, or 3 (respectively) (defaults to 1) + * FAN DIV: 1, 2, 4, or 8 + * REG: 0, 1, 2, or 3 (respectively) */ -static inline u8 DIV_TO_REG(int val) -{ - return val == 8 ? 3 : val == 4 ? 2 : val == 1 ? 0 : 1; -} #define DIV_FROM_REG(val) (1 << (val)) /* From cdeeaaba174886aa6c1ff4c0c5449c5066dbe82f Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 23 Feb 2024 19:27:17 +0100 Subject: [PATCH 0783/1038] mm, slab: deprecate SLAB_MEM_SPREAD flag The SLAB_MEM_SPREAD flag used to be implemented in SLAB, which was removed. SLUB instead relies on the page allocator's NUMA policies. Change the flag's value to 0 to free up the value it had, and mark it for full removal once all users are gone. Reported-by: Steven Rostedt Closes: https://lore.kernel.org/all/20240131172027.10f64405@gandalf.local.home/ Reviewed-and-tested-by: Xiongwei Song Reviewed-by: Chengming Zhou Reviewed-by: Roman Gushchin Acked-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 5 +++-- mm/slab.h | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index b5f5ee8308d0..b1675ff6b904 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -96,8 +96,6 @@ */ /* Defer freeing slabs to RCU */ #define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) -/* Spread some memory over cpuset */ -#define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) /* Trace allocations and frees */ #define SLAB_TRACE ((slab_flags_t __force)0x00200000U) @@ -164,6 +162,9 @@ #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ +/* Obsolete unused flag, to be removed */ +#define SLAB_MEM_SPREAD ((slab_flags_t __force)0U) + /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * diff --git a/mm/slab.h b/mm/slab.h index 54deeb0428c6..f4534eefb35d 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -469,7 +469,6 @@ static inline bool is_kmalloc_cache(struct kmem_cache *s) SLAB_STORE_USER | \ SLAB_TRACE | \ SLAB_CONSISTENCY_CHECKS | \ - SLAB_MEM_SPREAD | \ SLAB_NOLEAKTRACE | \ SLAB_RECLAIM_ACCOUNT | \ SLAB_TEMPORARY | \ From cc61eb851c9ae38546d7df6076fd883d3dbc322d Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 23 Feb 2024 19:27:18 +0100 Subject: [PATCH 0784/1038] mm, slab: use an enum to define SLAB_ cache creation flags The values of SLAB_ cache creation flags are defined by hand, which is tedious and error-prone. Use an enum to assign the bit number and a __SLAB_FLAG_BIT() macro to #define the final flags. This renumbers the flag values, which is OK as they are only used internally. Also define a __SLAB_FLAG_UNUSED macro to assign value to flags disabled by their respective config options in a unified and sparse-friendly way. Reviewed-and-tested-by: Xiongwei Song Reviewed-by: Chengming Zhou Reviewed-by: Roman Gushchin Acked-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 94 +++++++++++++++++++++++++++++++------------- mm/slub.c | 6 +-- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index b1675ff6b904..f6323763cd61 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -21,29 +21,69 @@ #include #include +enum _slab_flag_bits { + _SLAB_CONSISTENCY_CHECKS, + _SLAB_RED_ZONE, + _SLAB_POISON, + _SLAB_KMALLOC, + _SLAB_HWCACHE_ALIGN, + _SLAB_CACHE_DMA, + _SLAB_CACHE_DMA32, + _SLAB_STORE_USER, + _SLAB_PANIC, + _SLAB_TYPESAFE_BY_RCU, + _SLAB_TRACE, +#ifdef CONFIG_DEBUG_OBJECTS + _SLAB_DEBUG_OBJECTS, +#endif + _SLAB_NOLEAKTRACE, + _SLAB_NO_MERGE, +#ifdef CONFIG_FAILSLAB + _SLAB_FAILSLAB, +#endif +#ifdef CONFIG_MEMCG_KMEM + _SLAB_ACCOUNT, +#endif +#ifdef CONFIG_KASAN_GENERIC + _SLAB_KASAN, +#endif + _SLAB_NO_USER_FLAGS, +#ifdef CONFIG_KFENCE + _SLAB_SKIP_KFENCE, +#endif +#ifndef CONFIG_SLUB_TINY + _SLAB_RECLAIM_ACCOUNT, +#endif + _SLAB_OBJECT_POISON, + _SLAB_CMPXCHG_DOUBLE, + _SLAB_FLAGS_LAST_BIT +}; + +#define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr))) +#define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U)) /* * Flags to pass to kmem_cache_create(). * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op */ /* DEBUG: Perform (expensive) checks on alloc/free */ -#define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) +#define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS) /* DEBUG: Red zone objs in a cache */ -#define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) +#define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE) /* DEBUG: Poison objects */ -#define SLAB_POISON ((slab_flags_t __force)0x00000800U) +#define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON) /* Indicate a kmalloc slab */ -#define SLAB_KMALLOC ((slab_flags_t __force)0x00001000U) +#define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC) /* Align objs on cache lines */ -#define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) +#define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN) /* Use GFP_DMA memory */ -#define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) +#define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA) /* Use GFP_DMA32 memory */ -#define SLAB_CACHE_DMA32 ((slab_flags_t __force)0x00008000U) +#define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32) /* DEBUG: Store the last owner for bug hunting */ -#define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) +#define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER) /* Panic if kmem_cache_create() fails */ -#define SLAB_PANIC ((slab_flags_t __force)0x00040000U) +#define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC) /* * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! * @@ -95,19 +135,19 @@ * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. */ /* Defer freeing slabs to RCU */ -#define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) +#define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU) /* Trace allocations and frees */ -#define SLAB_TRACE ((slab_flags_t __force)0x00200000U) +#define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE) /* Flag to prevent checks on free */ #ifdef CONFIG_DEBUG_OBJECTS -# define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) +# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS) #else -# define SLAB_DEBUG_OBJECTS 0 +# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED #endif /* Avoid kmemleak tracing */ -#define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) +#define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE) /* * Prevent merging with compatible kmem caches. This flag should be used @@ -119,25 +159,25 @@ * - performance critical caches, should be very rare and consulted with slab * maintainers, and not used together with CONFIG_SLUB_TINY */ -#define SLAB_NO_MERGE ((slab_flags_t __force)0x01000000U) +#define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE) /* Fault injection mark */ #ifdef CONFIG_FAILSLAB -# define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) +# define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB) #else -# define SLAB_FAILSLAB 0 +# define SLAB_FAILSLAB __SLAB_FLAG_UNUSED #endif /* Account to memcg */ #ifdef CONFIG_MEMCG_KMEM -# define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) +# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) #else -# define SLAB_ACCOUNT 0 +# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED #endif #ifdef CONFIG_KASAN_GENERIC -#define SLAB_KASAN ((slab_flags_t __force)0x08000000U) +#define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN) #else -#define SLAB_KASAN 0 +#define SLAB_KASAN __SLAB_FLAG_UNUSED #endif /* @@ -145,25 +185,25 @@ * Intended for caches created for self-tests so they have only flags * specified in the code and other flags are ignored. */ -#define SLAB_NO_USER_FLAGS ((slab_flags_t __force)0x10000000U) +#define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS) #ifdef CONFIG_KFENCE -#define SLAB_SKIP_KFENCE ((slab_flags_t __force)0x20000000U) +#define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE) #else -#define SLAB_SKIP_KFENCE 0 +#define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED #endif /* The following flags affect the page allocator grouping pages by mobility */ /* Objects are reclaimable */ #ifndef CONFIG_SLUB_TINY -#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) +#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT) #else -#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0) +#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ /* Obsolete unused flag, to be removed */ -#define SLAB_MEM_SPREAD ((slab_flags_t __force)0U) +#define SLAB_MEM_SPREAD __SLAB_FLAG_UNUSED /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. diff --git a/mm/slub.c b/mm/slub.c index 2ef88bbf56a3..2934ef5f3cff 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -306,13 +306,13 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) /* Internal SLUB flags */ /* Poison object */ -#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U) +#define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON) /* Use cmpxchg_double */ #ifdef system_has_freelist_aba -#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U) +#define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE) #else -#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0U) +#define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED #endif /* From 96d8dbb6f65041b670a79e8ae76f67cc11dee203 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 23 Feb 2024 19:27:19 +0100 Subject: [PATCH 0785/1038] mm, slab, kasan: replace kasan_never_merge() with SLAB_NO_MERGE The SLAB_KASAN flag prevents merging of caches in some configurations, which is handled in a rather complicated way via kasan_never_merge(). Since we now have a generic SLAB_NO_MERGE flag, we can instead use it for KASAN caches in addition to SLAB_KASAN in those configurations, and simplify the SLAB_NEVER_MERGE handling. Tested-by: Xiongwei Song Reviewed-by: Chengming Zhou Reviewed-by: Andrey Konovalov Tested-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/kasan.h | 6 ------ mm/kasan/generic.c | 22 ++++++---------------- mm/slab_common.c | 2 +- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/include/linux/kasan.h b/include/linux/kasan.h index dbb06d789e74..70d6a8f6e25d 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -429,7 +429,6 @@ struct kasan_cache { }; size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object); -slab_flags_t kasan_never_merge(void); void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags); @@ -446,11 +445,6 @@ static inline size_t kasan_metadata_size(struct kmem_cache *cache, { return 0; } -/* And thus nothing prevents cache merging. */ -static inline slab_flags_t kasan_never_merge(void) -{ - return 0; -} /* And no cache-related metadata initialization is required. */ static inline void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index df6627f62402..27297dc4a55b 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -334,14 +334,6 @@ DEFINE_ASAN_SET_SHADOW(f3); DEFINE_ASAN_SET_SHADOW(f5); DEFINE_ASAN_SET_SHADOW(f8); -/* Only allow cache merging when no per-object metadata is present. */ -slab_flags_t kasan_never_merge(void) -{ - if (!kasan_requires_meta()) - return 0; - return SLAB_KASAN; -} - /* * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. * For larger allocations larger redzones are used. @@ -370,15 +362,13 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, return; /* - * SLAB_KASAN is used to mark caches that are sanitized by KASAN - * and that thus have per-object metadata. - * Currently this flag is used in two places: - * 1. In slab_ksize() to account for per-object metadata when - * calculating the size of the accessible memory within the object. - * 2. In slab_common.c via kasan_never_merge() to prevent merging of - * caches with per-object metadata. + * SLAB_KASAN is used to mark caches that are sanitized by KASAN and + * that thus have per-object metadata. Currently, this flag is used in + * slab_ksize() to account for per-object metadata when calculating the + * size of the accessible memory within the object. Additionally, we use + * SLAB_NO_MERGE to prevent merging of caches with per-object metadata. */ - *flags |= SLAB_KASAN; + *flags |= SLAB_KASAN | SLAB_NO_MERGE; ok_size = *size; diff --git a/mm/slab_common.c b/mm/slab_common.c index 238293b1dbe1..7cfa2f1ce655 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -50,7 +50,7 @@ static DECLARE_WORK(slab_caches_to_rcu_destroy_work, */ #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ - SLAB_FAILSLAB | SLAB_NO_MERGE | kasan_never_merge()) + SLAB_FAILSLAB | SLAB_NO_MERGE) #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ SLAB_CACHE_DMA32 | SLAB_ACCOUNT) From 144a0008b30bc133cbaa9da7cf56d7cf2bacc4f9 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Mon, 19 Feb 2024 09:08:33 +0530 Subject: [PATCH 0786/1038] pwm: dwc: drop redundant error check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pcim_iomap_table() fails only if pcim_iomap_regions() fails. No need to check for failure if the latter is already successful. Suggested-by: Andy Shevchenko Signed-off-by: Raag Jadav Tested-by: Jarkko Nikula Link: https://lore.kernel.org/r/20240219033835.11369-3-raag.jadav@intel.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index c0e586688e57..7dbb72c80ef5 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -51,11 +51,8 @@ static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) return ret; } + /* No need to check for failure, pcim_iomap_regions() does it for us. */ dwc->base = pcim_iomap_table(pci)[0]; - if (!dwc->base) { - dev_err(dev, "Base address missing\n"); - return -ENOMEM; - } ret = devm_pwmchip_add(dev, chip); if (ret) From ebf2c89eb95ec9598da19184632b765de72a7db4 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Mon, 19 Feb 2024 09:08:34 +0530 Subject: [PATCH 0787/1038] pwm: dwc: Add 16 channel support for Intel Elkhart Lake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intel Elkhart Lake PSE includes two instances of PWM as a single PCI function with 8 channels each. Add support for the remaining channels. Signed-off-by: Raag Jadav Tested-by: Jarkko Nikula Tested-by: Lakshmi Sowjanya D Link: https://lore.kernel.org/r/20240219033835.11369-4-raag.jadav@intel.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc.c | 38 +++++++++++++++++++++++++++++--------- drivers/pwm/pwm-dwc.h | 5 +++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index 7dbb72c80ef5..de95352081fa 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -25,17 +25,32 @@ #include "pwm-dwc.h" -static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) +/* Elkhart Lake */ +static const struct dwc_pwm_info ehl_pwm_info = { + .nr = 2, + .size = 0x1000, +}; + +static int dwc_pwm_init_one(struct device *dev, void __iomem *base, unsigned int offset) { - struct device *dev = &pci->dev; struct pwm_chip *chip; struct dwc_pwm *dwc; - int ret; chip = dwc_pwm_alloc(dev); if (IS_ERR(chip)) return PTR_ERR(chip); + dwc = to_dwc_pwm(chip); + dwc->base = base + offset; + + return devm_pwmchip_add(dev, chip); +} + +static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) +{ + const struct dwc_pwm_info *info; + struct device *dev = &pci->dev; + int i, ret; ret = pcim_enable_device(pci); if (ret) { @@ -51,12 +66,17 @@ static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) return ret; } - /* No need to check for failure, pcim_iomap_regions() does it for us. */ - dwc->base = pcim_iomap_table(pci)[0]; + info = (const struct dwc_pwm_info *)id->driver_data; - ret = devm_pwmchip_add(dev, chip); - if (ret) - return ret; + for (i = 0; i < info->nr; i++) { + /* + * No need to check for pcim_iomap_table() failure, + * pcim_iomap_regions() already does it for us. + */ + ret = dwc_pwm_init_one(dev, pcim_iomap_table(pci)[0], i * info->size); + if (ret) + return ret; + } pm_runtime_put(dev); pm_runtime_allow(dev); @@ -108,7 +128,7 @@ static int dwc_pwm_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(dwc_pwm_pm_ops, dwc_pwm_suspend, dwc_pwm_resume); static const struct pci_device_id dwc_pwm_id_table[] = { - { PCI_VDEVICE(INTEL, 0x4bb7) }, /* Elkhart Lake */ + { PCI_VDEVICE(INTEL, 0x4bb7), (kernel_ulong_t)&ehl_pwm_info }, { } /* Terminating Entry */ }; MODULE_DEVICE_TABLE(pci, dwc_pwm_id_table); diff --git a/drivers/pwm/pwm-dwc.h b/drivers/pwm/pwm-dwc.h index 5887371803fd..a8b074841ae8 100644 --- a/drivers/pwm/pwm-dwc.h +++ b/drivers/pwm/pwm-dwc.h @@ -33,6 +33,11 @@ MODULE_IMPORT_NS(dwc_pwm); #define DWC_TIM_CTRL_INT_MASK BIT(2) #define DWC_TIM_CTRL_PWM BIT(3) +struct dwc_pwm_info { + unsigned int nr; + unsigned int size; +}; + struct dwc_pwm_ctx { u32 cnt; u32 cnt2; From 9e3440d2d57b459cd12c5a4d15e95a1181568e4f Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Mon, 19 Feb 2024 09:08:35 +0530 Subject: [PATCH 0788/1038] pwm: dwc: simplify error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify error handling in ->probe() function using dev_err_probe() helper and while at it, drop error codes from the message to prevent duplication. Signed-off-by: Raag Jadav Tested-by: Jarkko Nikula Link: https://lore.kernel.org/r/20240219033835.11369-5-raag.jadav@intel.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-dwc.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index de95352081fa..676eaf8d7a53 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -53,18 +53,14 @@ static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) int i, ret; ret = pcim_enable_device(pci); - if (ret) { - dev_err(dev, "Failed to enable device (%pe)\n", ERR_PTR(ret)); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Failed to enable device\n"); pci_set_master(pci); ret = pcim_iomap_regions(pci, BIT(0), pci_name(pci)); - if (ret) { - dev_err(dev, "Failed to iomap PCI BAR (%pe)\n", ERR_PTR(ret)); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Failed to iomap PCI BAR\n"); info = (const struct dwc_pwm_info *)id->driver_data; From f837fe1bffe6976ed5e7198b1892ea248b75358b Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Mon, 19 Feb 2024 20:52:27 -0800 Subject: [PATCH 0789/1038] gpio: Add ChromeOS EC GPIO driver The ChromeOS embedded controller (EC) supports setting the state of GPIOs when the system is unlocked, and getting the state of GPIOs in all cases. The GPIOs are on the EC itself, so the EC acts similar to a GPIO expander. Add a driver to get and set the GPIOs on the EC through the host command interface. Signed-off-by: Stephen Boyd Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/Kconfig | 10 ++ drivers/gpio/Makefile | 1 + drivers/gpio/gpio-cros-ec.c | 209 ++++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 drivers/gpio/gpio-cros-ec.c diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 3081406ff57a..3fbb0bdb15c1 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -1241,6 +1241,16 @@ config GPIO_BD9571MWV This driver can also be built as a module. If so, the module will be called gpio-bd9571mwv. +config GPIO_CROS_EC + tristate "ChromeOS EC GPIO support" + depends on CROS_EC + help + GPIO driver for the ChromeOS Embedded Controller (EC). GPIOs + cannot be set unless the system is unlocked. + + This driver can also be built as a module. If so, the module + will be called gpio-cros-ec. + config GPIO_CRYSTAL_COVE tristate "GPIO support for Crystal Cove PMIC" depends on (X86 || COMPILE_TEST) && INTEL_SOC_PMIC diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 9e40af196aae..7ae4d81de1df 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -46,6 +46,7 @@ obj-$(CONFIG_GPIO_BT8XX) += gpio-bt8xx.o obj-$(CONFIG_GPIO_CADENCE) += gpio-cadence.o obj-$(CONFIG_GPIO_CLPS711X) += gpio-clps711x.o obj-$(CONFIG_GPIO_SNPS_CREG) += gpio-creg-snps.o +obj-$(CONFIG_GPIO_CROS_EC) += gpio-cros-ec.o obj-$(CONFIG_GPIO_CRYSTAL_COVE) += gpio-crystalcove.o obj-$(CONFIG_GPIO_CS5535) += gpio-cs5535.o obj-$(CONFIG_GPIO_DA9052) += gpio-da9052.o diff --git a/drivers/gpio/gpio-cros-ec.c b/drivers/gpio/gpio-cros-ec.c new file mode 100644 index 000000000000..842e1c060414 --- /dev/null +++ b/drivers/gpio/gpio-cros-ec.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2024 Google LLC + * + * This driver provides the ability to control GPIOs on the Chrome OS EC. + * There isn't any direction control, and setting values on GPIOs is only + * possible when the system is unlocked. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Prefix all names to avoid collisions with EC <-> AP nets */ +static const char cros_ec_gpio_prefix[] = "EC:"; + +/* Setting gpios is only supported when the system is unlocked */ +static void cros_ec_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) +{ + const char *name = gc->names[gpio] + strlen(cros_ec_gpio_prefix); + struct cros_ec_device *cros_ec = gpiochip_get_data(gc); + struct ec_params_gpio_set params = { + .val = val, + }; + int ret; + ssize_t copied; + + copied = strscpy(params.name, name, sizeof(params.name)); + if (copied < 0) + return; + + ret = cros_ec_cmd(cros_ec, 0, EC_CMD_GPIO_SET, ¶ms, + sizeof(params), NULL, 0); + if (ret < 0) + dev_err(gc->parent, "error setting gpio%d (%s) on EC: %d\n", gpio, name, ret); +} + +static int cros_ec_gpio_get(struct gpio_chip *gc, unsigned int gpio) +{ + const char *name = gc->names[gpio] + strlen(cros_ec_gpio_prefix); + struct cros_ec_device *cros_ec = gpiochip_get_data(gc); + struct ec_params_gpio_get params; + struct ec_response_gpio_get response; + int ret; + ssize_t copied; + + copied = strscpy(params.name, name, sizeof(params.name)); + if (copied < 0) + return -EINVAL; + + ret = cros_ec_cmd(cros_ec, 0, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) { + dev_err(gc->parent, "error getting gpio%d (%s) on EC: %d\n", gpio, name, ret); + return ret; + } + + return response.val; +} + +#define CROS_EC_GPIO_INPUT BIT(8) +#define CROS_EC_GPIO_OUTPUT BIT(9) + +static int cros_ec_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + const char *name = gc->names[gpio] + strlen(cros_ec_gpio_prefix); + struct cros_ec_device *cros_ec = gpiochip_get_data(gc); + struct ec_params_gpio_get_v1 params = { + .subcmd = EC_GPIO_GET_INFO, + .get_info.index = gpio, + }; + struct ec_response_gpio_get_v1 response; + int ret; + + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) { + dev_err(gc->parent, "error getting direction of gpio%d (%s) on EC: %d\n", gpio, name, ret); + return ret; + } + + if (response.get_info.flags & CROS_EC_GPIO_INPUT) + return GPIO_LINE_DIRECTION_IN; + + if (response.get_info.flags & CROS_EC_GPIO_OUTPUT) + return GPIO_LINE_DIRECTION_OUT; + + return -EINVAL; +} + +/* Query EC for all gpio line names */ +static int cros_ec_gpio_init_names(struct cros_ec_device *cros_ec, struct gpio_chip *gc) +{ + struct ec_params_gpio_get_v1 params = { + .subcmd = EC_GPIO_GET_INFO, + }; + struct ec_response_gpio_get_v1 response; + int ret, i; + /* EC may not NUL terminate */ + size_t name_len = strlen(cros_ec_gpio_prefix) + sizeof(response.get_info.name) + 1; + ssize_t copied; + const char **names; + char *str; + + names = devm_kcalloc(gc->parent, gc->ngpio, sizeof(*names), GFP_KERNEL); + if (!names) + return -ENOMEM; + gc->names = names; + + str = devm_kcalloc(gc->parent, gc->ngpio, name_len, GFP_KERNEL); + if (!str) + return -ENOMEM; + + /* Get gpio line names one at a time */ + for (i = 0; i < gc->ngpio; i++) { + params.get_info.index = i; + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) { + dev_err_probe(gc->parent, ret, "error getting gpio%d info\n", i); + return ret; + } + + names[i] = str; + copied = scnprintf(str, name_len, "%s%s", cros_ec_gpio_prefix, + response.get_info.name); + if (copied < 0) + return copied; + + str += copied + 1; + } + + return 0; +} + +/* Query EC for number of gpios */ +static int cros_ec_gpio_ngpios(struct cros_ec_device *cros_ec) +{ + struct ec_params_gpio_get_v1 params = { + .subcmd = EC_GPIO_GET_COUNT, + }; + struct ec_response_gpio_get_v1 response; + int ret; + + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) + return ret; + + return response.get_count.val; +} + +static int cros_ec_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *parent = dev->parent; + struct cros_ec_dev *ec_dev = dev_get_drvdata(parent); + struct cros_ec_device *cros_ec = ec_dev->ec_dev; + struct gpio_chip *gc; + int ngpios; + int ret; + + /* Use the fwnode from the protocol device, e.g. cros-ec-spi */ + device_set_node(dev, dev_fwnode(cros_ec->dev)); + + ngpios = cros_ec_gpio_ngpios(cros_ec); + if (ngpios < 0) { + dev_err_probe(dev, ngpios, "error getting gpio count\n"); + return ngpios; + } + + gc = devm_kzalloc(dev, sizeof(*gc), GFP_KERNEL); + if (!gc) + return -ENOMEM; + + gc->ngpio = ngpios; + gc->parent = dev; + ret = cros_ec_gpio_init_names(cros_ec, gc); + if (ret) + return ret; + + gc->can_sleep = true; + gc->label = dev_name(dev); + gc->base = -1; + gc->set = cros_ec_gpio_set; + gc->get = cros_ec_gpio_get; + gc->get_direction = cros_ec_gpio_get_direction; + + return devm_gpiochip_add_data(dev, gc, cros_ec); +} + +static struct platform_driver cros_ec_gpio_driver = { + .probe = cros_ec_gpio_probe, + .driver = { + .name = "cros-ec-gpio", + }, +}; +module_platform_driver(cros_ec_gpio_driver); + +MODULE_DESCRIPTION("ChromeOS EC GPIO Driver"); +MODULE_LICENSE("GPL"); From bb92804ba2b6636e28db05f589a9a8ef62a07917 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 19:49:25 +0000 Subject: [PATCH 0790/1038] regmap: kunit: Add a test for ranges in combination with windows In preparation for taking advantage of the SPI support for pre-coooked messages add a test case covering the use of windows on a raw regmap, unfortunately the parameterisation prevents direct reuse and we will want to add some raw specific coverage anyway. Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regmap-test-format-v1-1-41e4fdfb1c1f@kernel.org Signed-off-by: Mark Brown --- drivers/base/regmap/regmap-kunit.c | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/drivers/base/regmap/regmap-kunit.c b/drivers/base/regmap/regmap-kunit.c index 026bdcb45127..4ef401aca7ea 100644 --- a/drivers/base/regmap/regmap-kunit.c +++ b/drivers/base/regmap/regmap-kunit.c @@ -1318,6 +1318,71 @@ static void raw_sync(struct kunit *test) regmap_exit(map); } +static void raw_ranges(struct kunit *test) +{ + struct raw_test_types *t = (struct raw_test_types *)test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = raw_regmap_config; + config.volatile_reg = test_range_all_volatile; + config.ranges = &test_range; + config.num_ranges = 1; + config.max_register = test_range.range_max; + + map = gen_raw_regmap(&config, t, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Reset the page to a non-zero value to trigger a change */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.selector_reg, + test_range.range_max)); + + /* Check we set the page and use the window for writes */ + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.range_min, 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, + test_range.range_min + + test_range.window_len, + 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + /* Same for reads */ + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, test_range.range_min, &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, + test_range.range_min + + test_range.window_len, + &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + /* No physical access triggered in the virtual range */ + for (i = test_range.range_min; i < test_range.range_max; i++) { + KUNIT_EXPECT_FALSE(test, data->read[i]); + KUNIT_EXPECT_FALSE(test, data->written[i]); + } + + regmap_exit(map); +} + static struct kunit_case regmap_test_cases[] = { KUNIT_CASE_PARAM(basic_read_write, regcache_types_gen_params), KUNIT_CASE_PARAM(bulk_write, regcache_types_gen_params), @@ -1345,6 +1410,7 @@ static struct kunit_case regmap_test_cases[] = { KUNIT_CASE_PARAM(raw_write, raw_test_types_gen_params), KUNIT_CASE_PARAM(raw_noinc_write, raw_test_types_gen_params), KUNIT_CASE_PARAM(raw_sync, raw_test_cache_types_gen_params), + KUNIT_CASE_PARAM(raw_ranges, raw_test_cache_types_gen_params), {} }; From 4d52f575e258c6f93f4180c21afda8634b0d2af5 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:24 +0100 Subject: [PATCH 0791/1038] regulator: max8973: Finalize switch to GPIO descriptors The dvs gpio was still using a legacy number passed from the platform data. There are no in-tree users of the platform data so just switch it to a gpio descriptor and obtain it in probe(), the device tree users will work just as fine with this. Drop the entirely unused enable_gpio from the platform data as well. The device tree bindings mentions this but the driver does not look for it and makes no use of it: it should probably be implemented properly in a separate patch. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-1-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/max8973-regulator.c | 36 +++++++-------------- include/linux/regulator/max8973-regulator.h | 6 ---- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c index 8d5193207552..f8bb6828feef 100644 --- a/drivers/regulator/max8973-regulator.c +++ b/drivers/regulator/max8973-regulator.c @@ -20,9 +20,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -102,7 +100,7 @@ struct max8973_chip { struct regulator_desc desc; struct regmap *regmap; bool enable_external_control; - int dvs_gpio; + struct gpio_desc *dvs_gpiod; int lru_index[MAX8973_MAX_VOUT_REG]; int curr_vout_val[MAX8973_MAX_VOUT_REG]; int curr_vout_reg; @@ -184,7 +182,7 @@ static int max8973_dcdc_set_voltage_sel(struct regulator_dev *rdev, * If gpios are available to select the VOUT register then least * recently used register for new configuration. */ - if (gpio_is_valid(max->dvs_gpio)) + if (max->dvs_gpiod) found = find_voltage_set_register(max, vsel, &vout_reg, &gpio_val); @@ -201,8 +199,8 @@ static int max8973_dcdc_set_voltage_sel(struct regulator_dev *rdev, } /* Select proper VOUT register vio gpios */ - if (gpio_is_valid(max->dvs_gpio)) { - gpio_set_value_cansleep(max->dvs_gpio, gpio_val & 0x1); + if (max->dvs_gpiod) { + gpiod_set_value_cansleep(max->dvs_gpiod, gpio_val & 0x1); max->curr_gpio_val = gpio_val; } return 0; @@ -531,7 +529,6 @@ static struct max8973_regulator_platform_data *max8973_parse_dt( pdata->enable_ext_control = of_property_read_bool(np, "maxim,externally-enable"); - pdata->dvs_gpio = of_get_named_gpio(np, "maxim,dvs-gpio", 0); ret = of_property_read_u32(np, "maxim,dvs-default-state", &pval); if (!ret) @@ -612,13 +609,17 @@ static int max8973_probe(struct i2c_client *client) return -EIO; } - if (pdata->dvs_gpio == -EPROBE_DEFER) - return -EPROBE_DEFER; - max = devm_kzalloc(&client->dev, sizeof(*max), GFP_KERNEL); if (!max) return -ENOMEM; + max->dvs_gpiod = devm_gpiod_get_optional(&client->dev, "maxim,dvs", + (pdata->dvs_def_state) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW); + if (IS_ERR(max->dvs_gpiod)) + return dev_err_probe(&client->dev, PTR_ERR(max->dvs_gpiod), + "failed to obtain dvs gpio\n"); + gpiod_set_consumer_name(max->dvs_gpiod, "max8973-dvs"); + max->regmap = devm_regmap_init_i2c(client, &max8973_regmap_config); if (IS_ERR(max->regmap)) { ret = PTR_ERR(max->regmap); @@ -663,7 +664,6 @@ static int max8973_probe(struct i2c_client *client) max->desc.ramp_delay_table = max8973_buck_ramp_table; max->desc.n_ramp_values = ARRAY_SIZE(max8973_buck_ramp_table); - max->dvs_gpio = (pdata->dvs_gpio) ? pdata->dvs_gpio : -EINVAL; max->enable_external_control = pdata->enable_ext_control; max->curr_gpio_val = pdata->dvs_def_state; max->curr_vout_reg = MAX8973_VOUT + pdata->dvs_def_state; @@ -671,21 +671,9 @@ static int max8973_probe(struct i2c_client *client) max->lru_index[0] = max->curr_vout_reg; - if (gpio_is_valid(max->dvs_gpio)) { - int gpio_flags; + if (max->dvs_gpiod) { int i; - gpio_flags = (pdata->dvs_def_state) ? - GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW; - ret = devm_gpio_request_one(&client->dev, max->dvs_gpio, - gpio_flags, "max8973-dvs"); - if (ret) { - dev_err(&client->dev, - "gpio_request for gpio %d failed, err = %d\n", - max->dvs_gpio, ret); - return ret; - } - /* * Initialize the lru index with vout_reg id * The index 0 will be most recently used and diff --git a/include/linux/regulator/max8973-regulator.h b/include/linux/regulator/max8973-regulator.h index 8313e7ed6aec..a225e9eeb30d 100644 --- a/include/linux/regulator/max8973-regulator.h +++ b/include/linux/regulator/max8973-regulator.h @@ -48,10 +48,6 @@ * control signal from EN input pin. If it is false then * voltage output will be enabled/disabled through EN bit of * device register. - * @enable_gpio: Enable GPIO. If EN pin is controlled through GPIO from host - * then GPIO number can be provided. If no GPIO controlled then - * it should be -1. - * @dvs_gpio: GPIO for dvs. It should be -1 if this is tied with fixed logic. * @dvs_def_state: Default state of dvs. 1 if it is high else 0. */ struct max8973_regulator_platform_data { @@ -59,8 +55,6 @@ struct max8973_regulator_platform_data { unsigned long control_flags; unsigned long junction_temp_warning; bool enable_ext_control; - int enable_gpio; - int dvs_gpio; unsigned dvs_def_state:1; }; From e450a2b3a335332d4a51fe10c9fff8150c6e2364 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:25 +0100 Subject: [PATCH 0792/1038] regulator: da9055: Fully convert to GPIO descriptors The DA9055 regulator was touched before, requireing enable GPIOs to be passed from pdata. As we have a device for each regulator, obtain the three gpios ren ("regulator enable"), rsel ("regulator select") and the ena ("enable") GPIO associated with the regulator enable directly from the device and cut down on the amount of GPIO numbers passed as platform data. The ren and rsel are just requested as inputs: these are actually handled by hardware. The ena gpios are driven actively by the regulator core. There are no in-tree users, but the regulators are instantiated from the (undocumed) device tree nodes with "dlg,da9055-regulator" as compatible, and by simply adding regulator-enable-gpios, regulator-select-gpios and enable-gpios to this DT node, all will work as before. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-2-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/da9055-regulator.c | 48 ++++++++++++++-------------- include/linux/mfd/da9055/pdata.h | 13 -------- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/drivers/regulator/da9055-regulator.c b/drivers/regulator/da9055-regulator.c index 8fd9ac787588..352547c375bd 100644 --- a/drivers/regulator/da9055-regulator.c +++ b/drivers/regulator/da9055-regulator.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -413,31 +412,35 @@ static struct da9055_regulator_info da9055_regulator_info[] = { * GPIO can control regulator state and/or select the regulator register * set A/B for voltage ramping. */ -static int da9055_gpio_init(struct da9055_regulator *regulator, +static int da9055_gpio_init(struct device *dev, + struct da9055_regulator *regulator, struct regulator_config *config, struct da9055_pdata *pdata, int id) { struct da9055_regulator_info *info = regulator->info; + struct gpio_desc *ren; + struct gpio_desc *ena; + struct gpio_desc *rsel; int ret = 0; - if (!pdata) - return 0; + /* Look for "regulator-enable-gpios" GPIOs in the regulator node */ + ren = devm_gpiod_get_optional(dev, "regulator-enable", GPIOD_IN); + if (IS_ERR(ren)) + return PTR_ERR(ren); - if (pdata->gpio_ren && pdata->gpio_ren[id]) { - char name[18]; - int gpio_mux = pdata->gpio_ren[id]; + if (ren) { + /* This GPIO is not optional at this point */ + ena = devm_gpiod_get(dev, "enable", GPIOD_OUT_HIGH); + if (IS_ERR(ena)) + return PTR_ERR(ena); - config->ena_gpiod = pdata->ena_gpiods[id]; + config->ena_gpiod = ena; /* * GPI pin is muxed with regulator to control the * regulator state. */ - sprintf(name, "DA9055 GPI %d", gpio_mux); - ret = devm_gpio_request_one(config->dev, gpio_mux, GPIOF_DIR_IN, - name); - if (ret < 0) - goto err; + gpiod_set_consumer_name(ren, "DA9055 ren GPI"); /* * Let the regulator know that its state is controlled @@ -448,24 +451,22 @@ static int da9055_gpio_init(struct da9055_regulator *regulator, pdata->reg_ren[id] << DA9055_E_GPI_SHIFT); if (ret < 0) - goto err; + return ret; } - if (pdata->gpio_rsel && pdata->gpio_rsel[id]) { - char name[18]; - int gpio_mux = pdata->gpio_rsel[id]; + /* Look for "regulator-select-gpios" GPIOs in the regulator node */ + rsel = devm_gpiod_get_optional(dev, "regulator-select", GPIOD_IN); + if (IS_ERR(rsel)) + return PTR_ERR(rsel); + if (rsel) { regulator->reg_rselect = pdata->reg_rsel[id]; /* * GPI pin is muxed with regulator to select the * regulator register set A/B for voltage ramping. */ - sprintf(name, "DA9055 GPI %d", gpio_mux); - ret = devm_gpio_request_one(config->dev, gpio_mux, GPIOF_DIR_IN, - name); - if (ret < 0) - goto err; + gpiod_set_consumer_name(rsel, "DA9055 rsel GPI"); /* * Let the regulator know that its register set A/B @@ -477,7 +478,6 @@ static int da9055_gpio_init(struct da9055_regulator *regulator, << DA9055_V_GPI_SHIFT); } -err: return ret; } @@ -532,7 +532,7 @@ static int da9055_regulator_probe(struct platform_device *pdev) if (pdata) config.init_data = pdata->regulators[pdev->id]; - ret = da9055_gpio_init(regulator, &config, pdata, pdev->id); + ret = da9055_gpio_init(&pdev->dev, regulator, &config, pdata, pdev->id); if (ret < 0) return ret; diff --git a/include/linux/mfd/da9055/pdata.h b/include/linux/mfd/da9055/pdata.h index d3f126990ad0..137a2b067512 100644 --- a/include/linux/mfd/da9055/pdata.h +++ b/include/linux/mfd/da9055/pdata.h @@ -7,7 +7,6 @@ #define DA9055_MAX_REGULATORS 8 struct da9055; -struct gpio_desc; enum gpio_select { NO_GPIO = 0, @@ -23,16 +22,6 @@ struct da9055_pdata { struct regulator_init_data *regulators[DA9055_MAX_REGULATORS]; /* Enable RTC in RESET Mode */ bool reset_enable; - /* - * GPI muxed pin to control - * regulator state A/B, 0 if not available. - */ - int *gpio_ren; - /* - * GPI muxed pin to control - * regulator set, 0 if not available. - */ - int *gpio_rsel; /* * Regulator mode control bits value (GPI offset) that * controls the regulator state, 0 if not available. @@ -43,7 +32,5 @@ struct da9055_pdata { * controls the regulator set A/B, 0 if not available. */ enum gpio_select *reg_rsel; - /* GPIO descriptors to enable regulator, NULL if not available */ - struct gpio_desc **ena_gpiods; }; #endif /* __DA9055_PDATA_H */ From 95daa868f22b509ad641bf003d9d441d6a2fa505 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:26 +0100 Subject: [PATCH 0793/1038] regulator: lp8788-buck: Fully convert to GPIO descriptors This converts the LP8788 BUCK regulator driver to use GPIO descriptors. BUCK1 can use one DVS GPIO and BUCK2 can use two DVS GPIOS, and no more so just hardcode two GPIO descriptors into the per-DVS state containers. Obtain the descriptors from each regulators subdevice. As there are no in-tree users, board files need to populate descriptor tables for the buck regulator devices when they want to use this driver. BUCK1 need a GPIO descriptor at index 0 and BUCK2 needs two GPIO descriptors at indices 0 and 1. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-3-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/lp8788-buck.c | 64 ++++++++++++++------------------- include/linux/mfd/lp8788.h | 9 ++--- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/drivers/regulator/lp8788-buck.c b/drivers/regulator/lp8788-buck.c index e97ade09dede..712eaa6ff8ab 100644 --- a/drivers/regulator/lp8788-buck.c +++ b/drivers/regulator/lp8788-buck.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include /* register address */ #define LP8788_EN_BUCK 0x0C @@ -69,8 +69,8 @@ #define BUCK_FPWM_SHIFT(x) (x) enum lp8788_dvs_state { - DVS_LOW = GPIOF_OUT_INIT_LOW, - DVS_HIGH = GPIOF_OUT_INIT_HIGH, + DVS_LOW = 0, + DVS_HIGH = 1, }; enum lp8788_dvs_mode { @@ -89,6 +89,8 @@ struct lp8788_buck { struct lp8788 *lp; struct regulator_dev *regulator; void *dvs; + struct gpio_desc *gpio1; + struct gpio_desc *gpio2; /* Only used on BUCK2 */ }; /* BUCK 1 ~ 4 voltage ranges */ @@ -106,8 +108,7 @@ static void lp8788_buck1_set_dvs(struct lp8788_buck *buck) return; pinstate = dvs->vsel == DVS_SEL_V0 ? DVS_LOW : DVS_HIGH; - if (gpio_is_valid(dvs->gpio)) - gpio_set_value(dvs->gpio, pinstate); + gpiod_set_value(buck->gpio1, pinstate); } static void lp8788_buck2_set_dvs(struct lp8788_buck *buck) @@ -139,11 +140,8 @@ static void lp8788_buck2_set_dvs(struct lp8788_buck *buck) return; } - if (gpio_is_valid(dvs->gpio[0])) - gpio_set_value(dvs->gpio[0], pin1); - - if (gpio_is_valid(dvs->gpio[1])) - gpio_set_value(dvs->gpio[1], pin2); + gpiod_set_value(buck->gpio1, pin1); + gpiod_set_value(buck->gpio2, pin2); } static void lp8788_set_dvs(struct lp8788_buck *buck, enum lp8788_buck_id id) @@ -202,19 +200,13 @@ static u8 lp8788_select_buck_vout_addr(struct lp8788_buck *buck, enum lp8788_buck_id id) { enum lp8788_dvs_mode mode = lp8788_get_buck_dvs_ctrl_mode(buck, id); - struct lp8788_buck1_dvs *b1_dvs; - struct lp8788_buck2_dvs *b2_dvs; u8 val, idx, addr; int pin1, pin2; switch (id) { case BUCK1: if (mode == EXTPIN) { - b1_dvs = (struct lp8788_buck1_dvs *)buck->dvs; - if (!b1_dvs) - goto err; - - idx = gpio_get_value(b1_dvs->gpio) ? 1 : 0; + idx = gpiod_get_value(buck->gpio1); } else { lp8788_read_byte(buck->lp, LP8788_BUCK_DVS_SEL, &val); idx = (val & LP8788_BUCK1_DVS_M) >> LP8788_BUCK1_DVS_S; @@ -223,12 +215,8 @@ static u8 lp8788_select_buck_vout_addr(struct lp8788_buck *buck, break; case BUCK2: if (mode == EXTPIN) { - b2_dvs = (struct lp8788_buck2_dvs *)buck->dvs; - if (!b2_dvs) - goto err; - - pin1 = gpio_get_value(b2_dvs->gpio[0]); - pin2 = gpio_get_value(b2_dvs->gpio[1]); + pin1 = gpiod_get_value(buck->gpio1); + pin2 = gpiod_get_value(buck->gpio2); if (pin1 == PIN_LOW && pin2 == PIN_LOW) idx = 0; @@ -424,28 +412,28 @@ static int lp8788_dvs_gpio_request(struct platform_device *pdev, enum lp8788_buck_id id) { struct lp8788_platform_data *pdata = buck->lp->pdata; - char *b1_name = "LP8788_B1_DVS"; - char *b2_name[] = { "LP8788_B2_DVS1", "LP8788_B2_DVS2" }; - int i, gpio, ret; + struct device *dev = &pdev->dev; switch (id) { case BUCK1: - gpio = pdata->buck1_dvs->gpio; - ret = devm_gpio_request_one(&pdev->dev, gpio, DVS_LOW, - b1_name); - if (ret) - return ret; + buck->gpio1 = devm_gpiod_get(dev, "dvs", GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B1_DVS"); buck->dvs = pdata->buck1_dvs; break; case BUCK2: - for (i = 0; i < LP8788_NUM_BUCK2_DVS; i++) { - gpio = pdata->buck2_dvs->gpio[i]; - ret = devm_gpio_request_one(&pdev->dev, gpio, - DVS_LOW, b2_name[i]); - if (ret) - return ret; - } + buck->gpio1 = devm_gpiod_get_index(dev, "dvs", 0, GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS1"); + + buck->gpio2 = devm_gpiod_get_index(dev, "dvs", 1, GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS2"); + buck->dvs = pdata->buck2_dvs; break; default: diff --git a/include/linux/mfd/lp8788.h b/include/linux/mfd/lp8788.h index 3d5c480d58ea..51b47966a04d 100644 --- a/include/linux/mfd/lp8788.h +++ b/include/linux/mfd/lp8788.h @@ -10,7 +10,6 @@ #ifndef __MFD_LP8788_H__ #define __MFD_LP8788_H__ -#include #include #include #include @@ -159,21 +158,17 @@ struct lp8788; /* * lp8788_buck1_dvs - * @gpio : gpio pin number for dvs control * @vsel : dvs selector for buck v1 register */ struct lp8788_buck1_dvs { - int gpio; enum lp8788_dvs_sel vsel; }; /* * lp8788_buck2_dvs - * @gpio : two gpio pin numbers are used for dvs * @vsel : dvs selector for buck v2 register */ struct lp8788_buck2_dvs { - int gpio[LP8788_NUM_BUCK2_DVS]; enum lp8788_dvs_sel vsel; }; @@ -268,8 +263,8 @@ struct lp8788_vib_platform_data { * @buck_data : regulator initial data for buck * @dldo_data : regulator initial data for digital ldo * @aldo_data : regulator initial data for analog ldo - * @buck1_dvs : gpio configurations for buck1 dvs - * @buck2_dvs : gpio configurations for buck2 dvs + * @buck1_dvs : configurations for buck1 dvs + * @buck2_dvs : configurations for buck2 dvs * @chg_pdata : platform data for charger driver * @alarm_sel : rtc alarm selection (1 or 2) * @bl_pdata : configurable data for backlight driver From 84618d5e31cfd01fc3f53a8c2ebb68bc43d8b760 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:27 +0100 Subject: [PATCH 0794/1038] regulator: max8997: Convert to GPIO descriptors This rewrites the max8997 regulator driver to fetch the dvs regulators as descriptors. This will likely mostly come from the device tree since there are no in-tree users of the platform data, but supplying GPIO descriptor tables from board files is also possible if needed. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-4-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/max8997-regulator.c | 85 ++++++++------------------- include/linux/mfd/max8997.h | 1 - 2 files changed, 25 insertions(+), 61 deletions(-) diff --git a/drivers/regulator/max8997-regulator.c b/drivers/regulator/max8997-regulator.c index 0b38eaa73597..5f201ee9a5b8 100644 --- a/drivers/regulator/max8997-regulator.c +++ b/drivers/regulator/max8997-regulator.c @@ -9,8 +9,7 @@ #include #include -#include -#include +#include #include #include #include @@ -32,7 +31,7 @@ struct max8997_data { u8 buck1_vol[8]; u8 buck2_vol[8]; u8 buck5_vol[8]; - int buck125_gpios[3]; + struct gpio_desc *buck125_gpiods[3]; int buck125_gpioindex; bool ignore_gpiodvs_side_effect; @@ -52,9 +51,9 @@ static inline void max8997_set_gpio(struct max8997_data *max8997) int set2 = ((max8997->buck125_gpioindex) >> 1) & 0x1; int set1 = ((max8997->buck125_gpioindex) >> 2) & 0x1; - gpio_set_value(max8997->buck125_gpios[0], set1); - gpio_set_value(max8997->buck125_gpios[1], set2); - gpio_set_value(max8997->buck125_gpios[2], set3); + gpiod_set_value(max8997->buck125_gpiods[0], set1); + gpiod_set_value(max8997->buck125_gpiods[1], set2); + gpiod_set_value(max8997->buck125_gpiods[2], set3); } struct voltage_map_desc { @@ -873,31 +872,13 @@ static struct regulator_desc regulators[] = { }; #ifdef CONFIG_OF -static int max8997_pmic_dt_parse_dvs_gpio(struct platform_device *pdev, - struct max8997_platform_data *pdata, - struct device_node *pmic_np) -{ - int i, gpio; - - for (i = 0; i < 3; i++) { - gpio = of_get_named_gpio(pmic_np, - "max8997,pmic-buck125-dvs-gpios", i); - if (!gpio_is_valid(gpio)) { - dev_err(&pdev->dev, "invalid gpio[%d]: %d\n", i, gpio); - return -EINVAL; - } - pdata->buck125_gpios[i] = gpio; - } - return 0; -} - static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev, struct max8997_platform_data *pdata) { struct max8997_dev *iodev = dev_get_drvdata(pdev->dev.parent); struct device_node *pmic_np, *regulators_np, *reg_np; struct max8997_regulator_data *rdata; - unsigned int i, dvs_voltage_nr = 1, ret; + unsigned int i, dvs_voltage_nr = 1; pmic_np = iodev->dev->of_node; if (!pmic_np) { @@ -949,10 +930,6 @@ static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev, if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || pdata->buck5_gpiodvs) { - ret = max8997_pmic_dt_parse_dvs_gpio(pdev, pdata, pmic_np); - if (ret) - return -EINVAL; - if (of_property_read_u32(pmic_np, "max8997,pmic-buck125-default-dvs-idx", &pdata->buck125_default_idx)) { @@ -1039,7 +1016,6 @@ static int max8997_pmic_probe(struct platform_device *pdev) max8997->buck1_gpiodvs = pdata->buck1_gpiodvs; max8997->buck2_gpiodvs = pdata->buck2_gpiodvs; max8997->buck5_gpiodvs = pdata->buck5_gpiodvs; - memcpy(max8997->buck125_gpios, pdata->buck125_gpios, sizeof(int) * 3); max8997->ignore_gpiodvs_side_effect = pdata->ignore_gpiodvs_side_effect; nr_dvs = (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || @@ -1110,38 +1086,27 @@ static int max8997_pmic_probe(struct platform_device *pdev) */ if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || pdata->buck5_gpiodvs) { + const char *gpio_names[3] = {"MAX8997 SET1", "MAX8997 SET2", "MAX8997 SET3"}; - if (!gpio_is_valid(pdata->buck125_gpios[0]) || - !gpio_is_valid(pdata->buck125_gpios[1]) || - !gpio_is_valid(pdata->buck125_gpios[2])) { - dev_err(&pdev->dev, "GPIO NOT VALID\n"); - return -EINVAL; + for (i = 0; i < 3; i++) { + enum gpiod_flags flags; + + if (max8997->buck125_gpioindex & BIT(2 - i)) + flags = GPIOD_OUT_HIGH; + else + flags = GPIOD_OUT_LOW; + + max8997->buck125_gpiods[i] = devm_gpiod_get_index(iodev->dev, + "max8997,pmic-buck125-dvs", + i, + flags); + if (IS_ERR(max8997->buck125_gpiods[i])) { + ret = PTR_ERR(max8997->buck125_gpiods[i]); + return dev_err_probe(iodev->dev, ret, "cant get GPIO %d (%d)\n", + i, ret); + } + gpiod_set_consumer_name(max8997->buck125_gpiods[i], gpio_names[i]); } - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[0], - "MAX8997 SET1"); - if (ret) - return ret; - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[1], - "MAX8997 SET2"); - if (ret) - return ret; - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[2], - "MAX8997 SET3"); - if (ret) - return ret; - - gpio_direction_output(pdata->buck125_gpios[0], - (max8997->buck125_gpioindex >> 2) - & 0x1); /* SET1 */ - gpio_direction_output(pdata->buck125_gpios[1], - (max8997->buck125_gpioindex >> 1) - & 0x1); /* SET2 */ - gpio_direction_output(pdata->buck125_gpios[2], - (max8997->buck125_gpioindex >> 0) - & 0x1); /* SET3 */ } /* DVS-GPIO disabled */ diff --git a/include/linux/mfd/max8997.h b/include/linux/mfd/max8997.h index 6193905abbb5..5c2cc1103437 100644 --- a/include/linux/mfd/max8997.h +++ b/include/linux/mfd/max8997.h @@ -178,7 +178,6 @@ struct max8997_platform_data { * */ bool ignore_gpiodvs_side_effect; - int buck125_gpios[3]; /* GPIO of [0]SET1, [1]SET2, [2]SET3 */ int buck125_default_idx; /* Default value of SET1, 2, 3 */ unsigned int buck1_voltage[8]; /* buckx_voltage in uV */ bool buck1_gpiodvs; From f25828a1eae1ee1a9257e2818b237b8208bd383e Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 20 Feb 2024 09:36:28 +0100 Subject: [PATCH 0795/1038] regulator: max8998: Convert to GPIO descriptors This rewrites the max8998 regulator driver to fetch the dvs regulators as descriptors. This will likely mostly come from the device tree since there are no in-tree users of the platform data, but supplying GPIO descriptor tables from board files is also possible if needed. Signed-off-by: Linus Walleij Link: https://msgid.link/r/20240220-descriptors-regulators-v1-5-097f608694be@linaro.org Acked-by: Lee Jones Signed-off-by: Mark Brown --- drivers/regulator/max8998.c | 144 +++++++++++++----------------------- include/linux/mfd/max8998.h | 6 -- 2 files changed, 51 insertions(+), 99 deletions(-) diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c index fadb4717384a..254a77887f66 100644 --- a/drivers/regulator/max8998.c +++ b/drivers/regulator/max8998.c @@ -10,12 +10,12 @@ #include #include #include -#include +#include +#include #include #include #include #include -#include #include #include #include @@ -31,6 +31,9 @@ struct max8998_data { unsigned int buck1_idx; /* index to last changed voltage */ /* value in a set */ unsigned int buck2_idx; + struct gpio_desc *buck1_gpio1; + struct gpio_desc *buck1_gpio2; + struct gpio_desc *buck2_gpio; }; static const unsigned int charger_current_table[] = { @@ -227,15 +230,15 @@ static int max8998_set_voltage_ldo_sel(struct regulator_dev *rdev, return ret; } -static inline void buck1_gpio_set(int gpio1, int gpio2, int v) +static inline void buck1_gpio_set(struct gpio_desc *gpio1, struct gpio_desc *gpio2, int v) { - gpio_set_value(gpio1, v & 0x1); - gpio_set_value(gpio2, (v >> 1) & 0x1); + gpiod_set_value(gpio1, v & 0x1); + gpiod_set_value(gpio2, (v >> 1) & 0x1); } -static inline void buck2_gpio_set(int gpio, int v) +static inline void buck2_gpio_set(struct gpio_desc *gpio, int v) { - gpio_set_value(gpio, v & 0x1); + gpiod_set_value(gpio, v & 0x1); } static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, @@ -260,16 +263,15 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, selector, max8998->buck1_vol[0], max8998->buck1_vol[1], max8998->buck1_vol[2], max8998->buck1_vol[3]); - if (gpio_is_valid(pdata->buck1_set1) && - gpio_is_valid(pdata->buck1_set2)) { + if (max8998->buck1_gpio1 && max8998->buck1_gpio2) { /* check if requested voltage */ /* value is already defined */ for (j = 0; j < ARRAY_SIZE(max8998->buck1_vol); j++) { if (max8998->buck1_vol[j] == selector) { max8998->buck1_idx = j; - buck1_gpio_set(pdata->buck1_set1, - pdata->buck1_set2, j); + buck1_gpio_set(max8998->buck1_gpio1, + max8998->buck1_gpio2, j); goto buck1_exit; } } @@ -286,13 +288,13 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, &shift, &mask); ret = max8998_write_reg(i2c, reg, selector); - buck1_gpio_set(pdata->buck1_set1, - pdata->buck1_set2, max8998->buck1_idx); + buck1_gpio_set(max8998->buck1_gpio1, + max8998->buck1_gpio2, max8998->buck1_idx); buck1_last_val++; buck1_exit: dev_dbg(max8998->dev, "%s: SET1:%d, SET2:%d\n", - i2c->name, gpio_get_value(pdata->buck1_set1), - gpio_get_value(pdata->buck1_set2)); + i2c->name, gpiod_get_value(max8998->buck1_gpio1), + gpiod_get_value(max8998->buck1_gpio2)); break; } else { ret = max8998_write_reg(i2c, reg, selector); @@ -303,14 +305,13 @@ buck1_exit: dev_dbg(max8998->dev, "BUCK2, selector:%d buck2_vol1:%d, buck2_vol2:%d\n", selector, max8998->buck2_vol[0], max8998->buck2_vol[1]); - if (gpio_is_valid(pdata->buck2_set3)) { - + if (max8998->buck2_gpio) { /* check if requested voltage */ /* value is already defined */ for (j = 0; j < ARRAY_SIZE(max8998->buck2_vol); j++) { if (max8998->buck2_vol[j] == selector) { max8998->buck2_idx = j; - buck2_gpio_set(pdata->buck2_set3, j); + buck2_gpio_set(max8998->buck2_gpio, j); goto buck2_exit; } } @@ -322,10 +323,10 @@ buck1_exit: ®, &shift, &mask); ret = max8998_write_reg(i2c, reg, selector); max8998->buck2_vol[max8998->buck2_idx] = selector; - buck2_gpio_set(pdata->buck2_set3, max8998->buck2_idx); + buck2_gpio_set(max8998->buck2_gpio, max8998->buck2_idx); buck2_exit: dev_dbg(max8998->dev, "%s: SET3:%d\n", i2c->name, - gpio_get_value(pdata->buck2_set3)); + gpiod_get_value(max8998->buck2_gpio)); } else { ret = max8998_write_reg(i2c, reg, selector); } @@ -539,36 +540,6 @@ static const struct regulator_desc regulators[] = { charger_current_table, MAX8998_REG_CHGR1, 0x7), }; -static int max8998_pmic_dt_parse_dvs_gpio(struct max8998_dev *iodev, - struct max8998_platform_data *pdata, - struct device_node *pmic_np) -{ - int gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck1-dvs-gpios", 0); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck1 gpio[0]: %d\n", gpio); - return -EINVAL; - } - pdata->buck1_set1 = gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck1-dvs-gpios", 1); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck1 gpio[1]: %d\n", gpio); - return -EINVAL; - } - pdata->buck1_set2 = gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck2-dvs-gpio", 0); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck 2 gpio: %d\n", gpio); - return -EINVAL; - } - pdata->buck2_set3 = gpio; - - return 0; -} - static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, struct max8998_platform_data *pdata) { @@ -614,10 +585,6 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, of_node_put(reg_np); of_node_put(regulators_np); - ret = max8998_pmic_dt_parse_dvs_gpio(iodev, pdata, pmic_np); - if (ret) - return -EINVAL; - pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock"); ret = of_property_read_u32(pmic_np, @@ -665,6 +632,7 @@ static int max8998_pmic_probe(struct platform_device *pdev) struct regulator_dev *rdev; struct max8998_data *max8998; struct i2c_client *i2c; + enum gpiod_flags flags; int i, ret; unsigned int v; @@ -693,37 +661,38 @@ static int max8998_pmic_probe(struct platform_device *pdev) max8998->buck1_idx = pdata->buck1_default_idx; max8998->buck2_idx = pdata->buck2_default_idx; - /* NOTE: */ - /* For unused GPIO NOT marked as -1 (thereof equal to 0) WARN_ON */ - /* will be displayed */ - /* Check if MAX8998 voltage selection GPIOs are defined */ - if (gpio_is_valid(pdata->buck1_set1) && - gpio_is_valid(pdata->buck1_set2)) { - /* Check if SET1 is not equal to 0 */ - if (!pdata->buck1_set1) { - dev_err(&pdev->dev, - "MAX8998 SET1 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck1_set1); - return -EIO; - } - /* Check if SET2 is not equal to 0 */ - if (!pdata->buck1_set2) { - dev_err(&pdev->dev, - "MAX8998 SET2 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck1_set2); - return -EIO; - } + flags = (max8998->buck1_idx & BIT(0)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck1_gpio1 = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck1-dvs", + 0, + flags); + if (IS_ERR(max8998->buck1_gpio1)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck1_gpio1), + "could not get BUCK1 GPIO1\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK1_SET1"); - gpio_request(pdata->buck1_set1, "MAX8998 BUCK1_SET1"); - gpio_direction_output(pdata->buck1_set1, - max8998->buck1_idx & 0x1); + flags = (max8998->buck1_idx & BIT(1)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck1_gpio2 = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck1-dvs", + 1, + flags); + if (IS_ERR(max8998->buck1_gpio2)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck1_gpio2), + "could not get BUCK1 GPIO2\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK1_SET2"); + flags = (max8998->buck2_idx & BIT(0)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck2_gpio = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck2-dvs", + 0, + flags); + if (IS_ERR(max8998->buck2_gpio)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck2_gpio), + "could not get BUCK2 GPIO\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK2_SET3"); - gpio_request(pdata->buck1_set2, "MAX8998 BUCK1_SET2"); - gpio_direction_output(pdata->buck1_set2, - (max8998->buck1_idx >> 1) & 0x1); - + if (max8998->buck1_gpio1 && max8998->buck1_gpio2) { /* Set predefined values for BUCK1 registers */ for (v = 0; v < ARRAY_SIZE(pdata->buck1_voltage); ++v) { int index = MAX8998_BUCK1 - MAX8998_LDO2; @@ -742,18 +711,7 @@ static int max8998_pmic_probe(struct platform_device *pdev) } } - if (gpio_is_valid(pdata->buck2_set3)) { - /* Check if SET3 is not equal to 0 */ - if (!pdata->buck2_set3) { - dev_err(&pdev->dev, - "MAX8998 SET3 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck2_set3); - return -EIO; - } - gpio_request(pdata->buck2_set3, "MAX8998 BUCK2_SET3"); - gpio_direction_output(pdata->buck2_set3, - max8998->buck2_idx & 0x1); - + if (max8998->buck2_gpio) { /* Set predefined values for BUCK2 registers */ for (v = 0; v < ARRAY_SIZE(pdata->buck2_voltage); ++v) { int index = MAX8998_BUCK2 - MAX8998_LDO2; diff --git a/include/linux/mfd/max8998.h b/include/linux/mfd/max8998.h index 79c020bd0c70..a054e55c8646 100644 --- a/include/linux/mfd/max8998.h +++ b/include/linux/mfd/max8998.h @@ -65,10 +65,7 @@ struct max8998_regulator_data { * be other than the preset values. * @buck1_voltage: BUCK1 DVS mode 1 voltage registers * @buck2_voltage: BUCK2 DVS mode 2 voltage registers - * @buck1_set1: BUCK1 gpio pin 1 to set output voltage - * @buck1_set2: BUCK1 gpio pin 2 to set output voltage * @buck1_default_idx: Default for BUCK1 gpio pin 1, 2 - * @buck2_set3: BUCK2 gpio pin to set output voltage * @buck2_default_idx: Default for BUCK2 gpio pin. * @wakeup: Allow to wake up from suspend * @rtc_delay: LP3974 RTC chip bug that requires delay after a register @@ -91,10 +88,7 @@ struct max8998_platform_data { bool buck_voltage_lock; int buck1_voltage[4]; int buck2_voltage[2]; - int buck1_set1; - int buck1_set2; int buck1_default_idx; - int buck2_set3; int buck2_default_idx; bool wakeup; bool rtc_delay; From b65e9149bdb76e9b09f6fb76fea1f10bde256619 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:27 +0000 Subject: [PATCH 0796/1038] regulator: mp8859: Specify register accessibility and enable caching Avoid needless accesses to the hardware by caching register values that we know, marking status registers as volatile as appropriate. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-1-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index b820bd6043e5..689b56680d93 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -77,11 +77,47 @@ static const struct linear_range mp8859_dcdc_ranges[] = { REGULATOR_LINEAR_RANGE(0, VOL_MIN_IDX, VOL_MAX_IDX, 10000), }; +static bool mp8859_readable(struct device *dev, unsigned int reg) +{ + switch (reg) { + case MP8859_VOUT_L_REG: + case MP8859_VOUT_H_REG: + case MP8859_VOUT_GO_REG: + case MP8859_IOUT_LIM_REG: + case MP8859_CTL1_REG: + case MP8859_CTL2_REG: + case MP8859_STATUS_REG: + case MP8859_INTERRUPT_REG: + case MP8859_MASK_REG: + case MP8859_ID1_REG: + case MP8859_MFR_ID_REG: + case MP8859_DEV_ID_REG: + case MP8859_IC_REV_REG: + return true; + default: + return false; + } +} + +static bool mp8859_volatile(struct device *dev, unsigned int reg) +{ + switch (reg) { + case MP8859_VOUT_GO_REG: + case MP8859_STATUS_REG: + case MP8859_INTERRUPT_REG: + return true; + default: + return false; + } +} + static const struct regmap_config mp8859_regmap = { .reg_bits = 8, .val_bits = 8, .max_register = MP8859_MAX_REG, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, + .readable_reg = mp8859_readable, + .volatile_reg = mp8859_volatile, }; static const struct regulator_ops mp8859_ops = { From 6c848d772eee0f03b72542f35e1a66469030390f Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:28 +0000 Subject: [PATCH 0797/1038] regulator: mp8859: Validate and log device identifier information Ensure that we are talking to a device which reports the expected ID register information and log the OTP and revision information for diagnostic purposes. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-2-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index 689b56680d93..b07bc63a25cb 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -147,12 +147,46 @@ static int mp8859_i2c_probe(struct i2c_client *i2c) struct regulator_config config = {.dev = &i2c->dev}; struct regmap *regmap = devm_regmap_init_i2c(i2c, &mp8859_regmap); struct regulator_dev *rdev; + unsigned int val, rev; if (IS_ERR(regmap)) { ret = PTR_ERR(regmap); dev_err(&i2c->dev, "regmap init failed: %d\n", ret); return ret; } + + ret = regmap_read(regmap, MP8859_MFR_ID_REG, &val); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read manufacturer ID: %d\n", ret); + return ret; + } + if (val != 0x9) { + dev_err(&i2c->dev, "Manufacturer ID %x != 9\n", val); + return -EINVAL; + } + + ret = regmap_read(regmap, MP8859_DEV_ID_REG, &val); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read device ID: %d\n", ret); + return ret; + } + if (val != 0x58) { + dev_err(&i2c->dev, "Manufacturer ID %x != 0x58\n", val); + return -EINVAL; + } + + ret = regmap_read(regmap, MP8859_IC_REV_REG, &rev); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read device revision: %d\n", ret); + return ret; + } + ret = regmap_read(regmap, MP8859_ID1_REG, &val); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read device ID1: %d\n", ret); + return ret; + } + dev_info(&i2c->dev, "MP8859-%04d revision %d\n", val, rev); + rdev = devm_regulator_register(&i2c->dev, &mp8859_regulators[0], &config); From b79d93d99e084bf1abafba2b7aabff6a06defcd0 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:29 +0000 Subject: [PATCH 0798/1038] regulator: mp8859: Support enable control The MP8859 provides a software enable control, support it in the regulator driver. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-3-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index b07bc63a25cb..a443ebe927c5 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -35,6 +35,7 @@ #define MP8859_GO_BIT 0x01 +#define MP8859_ENABLE_MASK 0x80 static int mp8859_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel) { @@ -124,6 +125,9 @@ static const struct regulator_ops mp8859_ops = { .set_voltage_sel = mp8859_set_voltage_sel, .get_voltage_sel = mp8859_get_voltage_sel, .list_voltage = regulator_list_voltage_linear_range, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, }; static const struct regulator_desc mp8859_regulators[] = { @@ -136,6 +140,9 @@ static const struct regulator_desc mp8859_regulators[] = { .n_voltages = VOL_MAX_IDX + 1, .linear_ranges = mp8859_dcdc_ranges, .n_linear_ranges = 1, + .enable_reg = MP8859_CTL1_REG, + .enable_mask = MP8859_ENABLE_MASK, + .enable_val = MP8859_ENABLE_MASK, .ops = &mp8859_ops, .owner = THIS_MODULE, }, From 673d06a858864c7dca9dd5c36a78f5f7fda793af Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:30 +0000 Subject: [PATCH 0799/1038] regulator: mp8859: Support mode operations The MP8859 provides mode control, implement the relevant regulator API operations. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-4-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index a443ebe927c5..b97c00097148 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -36,6 +36,7 @@ #define MP8859_GO_BIT 0x01 #define MP8859_ENABLE_MASK 0x80 +#define MP8859_MODE_MASK 0x08 static int mp8859_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel) { @@ -74,6 +75,42 @@ static int mp8859_get_voltage_sel(struct regulator_dev *rdev) return val; } +static unsigned int mp8859_get_mode(struct regulator_dev *rdev) +{ + unsigned int val; + int ret; + + ret = regmap_read(rdev->regmap, MP8859_CTL1_REG, &val); + if (ret != 0) { + dev_err(&rdev->dev, "Failed to read mode: %d\n", ret); + return 0; + } + + if (val & MP8859_MODE_MASK) + return REGULATOR_MODE_FAST; + else + return REGULATOR_MODE_NORMAL; +} + +static int mp8859_set_mode(struct regulator_dev *rdev, unsigned int mode) +{ + unsigned int val; + + switch (mode) { + case REGULATOR_MODE_FAST: + val = MP8859_MODE_MASK; + break; + case REGULATOR_MODE_NORMAL: + val = 0; + break; + default: + return -EINVAL; + } + + return regmap_update_bits(rdev->regmap, MP8859_CTL1_REG, + MP8859_MODE_MASK, val); +} + static const struct linear_range mp8859_dcdc_ranges[] = { REGULATOR_LINEAR_RANGE(0, VOL_MIN_IDX, VOL_MAX_IDX, 10000), }; @@ -128,6 +165,8 @@ static const struct regulator_ops mp8859_ops = { .enable = regulator_enable_regmap, .disable = regulator_disable_regmap, .is_enabled = regulator_is_enabled_regmap, + .set_mode = mp8859_set_mode, + .get_mode = mp8859_get_mode, }; static const struct regulator_desc mp8859_regulators[] = { From d7217c91bbde48ee60d3ce67cda6557f56c6b639 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:31 +0000 Subject: [PATCH 0800/1038] regulator: mp8859: Support active discharge control The MP8859 can actively discharge the output when disabled, add support for controlling this feature. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-5-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index b97c00097148..87ef34875be1 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -36,6 +36,7 @@ #define MP8859_GO_BIT 0x01 #define MP8859_ENABLE_MASK 0x80 +#define MP8859_DISCHG_EN_MASK 0x10 #define MP8859_MODE_MASK 0x08 static int mp8859_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel) @@ -167,6 +168,7 @@ static const struct regulator_ops mp8859_ops = { .is_enabled = regulator_is_enabled_regmap, .set_mode = mp8859_set_mode, .get_mode = mp8859_get_mode, + .set_active_discharge = regulator_set_active_discharge_regmap, }; static const struct regulator_desc mp8859_regulators[] = { @@ -182,6 +184,9 @@ static const struct regulator_desc mp8859_regulators[] = { .enable_reg = MP8859_CTL1_REG, .enable_mask = MP8859_ENABLE_MASK, .enable_val = MP8859_ENABLE_MASK, + .active_discharge_reg = MP8859_CTL1_REG, + .active_discharge_on = MP8859_DISCHG_EN_MASK, + .active_discharge_mask = MP8859_DISCHG_EN_MASK, .ops = &mp8859_ops, .owner = THIS_MODULE, }, From 4317ecadbeeab5464a8c34b27b73e2d2f81ef718 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:32 +0000 Subject: [PATCH 0801/1038] regulator: mp8859: Support status and error readback The MP8859 can report if it is in regulation and detect over temperature conditions, report this information via the relevant regulator API calls. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-6-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index 87ef34875be1..fc1636d69bca 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -39,6 +39,11 @@ #define MP8859_DISCHG_EN_MASK 0x10 #define MP8859_MODE_MASK 0x08 +#define MP8859_PG_MASK 0x80 +#define MP8859_OTP_MASK 0x40 +#define MP8859_OTW_MASK 0x20 +#define MP8859_CC_CV_MASK 0x10 + static int mp8859_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel) { int ret; @@ -112,6 +117,58 @@ static int mp8859_set_mode(struct regulator_dev *rdev, unsigned int mode) MP8859_MODE_MASK, val); } +static int mp8859_get_status(struct regulator_dev *rdev) +{ + unsigned int val; + int ret; + + /* Output status is only meaingful when enabled */ + ret = regmap_read(rdev->regmap, MP8859_CTL1_REG, &val); + if (ret != 0) + return ret; + if (!(val & MP8859_ENABLE_MASK)) + return REGULATOR_STATUS_UNDEFINED; + + ret = regmap_read(rdev->regmap, MP8859_STATUS_REG, &val); + if (ret != 0) + return ret; + + if (val & MP8859_PG_MASK) + return REGULATOR_STATUS_ON; + else + return REGULATOR_STATUS_ERROR; +} + +static int mp8859_get_error_flags(struct regulator_dev *rdev, + unsigned int *flags) +{ + unsigned int status, enabled; + int ret; + + *flags = 0; + + /* Output status is only meaingful when enabled */ + ret = regmap_read(rdev->regmap, MP8859_CTL1_REG, &enabled); + if (ret != 0) + return ret; + enabled &= MP8859_ENABLE_MASK; + + ret = regmap_read(rdev->regmap, MP8859_STATUS_REG, &status); + if (ret != 0) + return ret; + + if (enabled && !(status & MP8859_PG_MASK)) + status |= REGULATOR_ERROR_FAIL; + if (status & MP8859_OTP_MASK) + status |= REGULATOR_ERROR_OVER_TEMP; + if (status & MP8859_OTW_MASK) + status |= REGULATOR_ERROR_OVER_TEMP_WARN; + if (status & MP8859_CC_CV_MASK) + status |= REGULATOR_ERROR_OVER_CURRENT; + + return 0; +} + static const struct linear_range mp8859_dcdc_ranges[] = { REGULATOR_LINEAR_RANGE(0, VOL_MIN_IDX, VOL_MAX_IDX, 10000), }; @@ -169,6 +226,8 @@ static const struct regulator_ops mp8859_ops = { .set_mode = mp8859_set_mode, .get_mode = mp8859_get_mode, .set_active_discharge = regulator_set_active_discharge_regmap, + .get_status = mp8859_get_status, + .get_error_flags = mp8859_get_error_flags, }; static const struct regulator_desc mp8859_regulators[] = { From 6df0921e9013622091c283aa2a5be8c5d1ca3642 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:33 +0000 Subject: [PATCH 0802/1038] regulator: mp8859: Report slew rate The MP8859 implements voltage changes at the rate of 1mV/us, tell the core about this so that it can provide appropriate delays on voltage changes. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-7-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index fc1636d69bca..3e849e1a172f 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -81,6 +81,20 @@ static int mp8859_get_voltage_sel(struct regulator_dev *rdev) return val; } +static int mp8859_set_voltage_time_sel(struct regulator_dev *rdev, + unsigned int from, unsigned int to) +{ + int change; + + /* The voltage ramps at 1mV/uS, selectors are 10mV */ + if (from > to) + change = from - to; + else + change = to - from; + + return change * 10 * 1000; +} + static unsigned int mp8859_get_mode(struct regulator_dev *rdev) { unsigned int val; @@ -220,6 +234,7 @@ static const struct regulator_ops mp8859_ops = { .set_voltage_sel = mp8859_set_voltage_sel, .get_voltage_sel = mp8859_get_voltage_sel, .list_voltage = regulator_list_voltage_linear_range, + .set_voltage_time_sel = mp8859_set_voltage_time_sel, .enable = regulator_enable_regmap, .disable = regulator_disable_regmap, .is_enabled = regulator_is_enabled_regmap, From c8e794cfb05344af7b9ae920b5548a25a7e91fe9 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sun, 25 Feb 2024 14:59:34 +0000 Subject: [PATCH 0803/1038] regulator: mp8859: Implement set_current_limit() The mp8859 implements support for current limiting, provide support for configuring this via the driver. The datasheet recommends that if the device has hit the current limit then any changes should be implemented via a ramp so we do so in the driver. Tested-by: Markus Reichl Signed-off-by: Mark Brown Link: https://msgid.link/r/20240225-regulator-mp8859-v1-8-68ee2c839ded@kernel.org Signed-off-by: Mark Brown --- drivers/regulator/mp8859.c | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index 3e849e1a172f..ab105ffd6a2e 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -35,6 +35,8 @@ #define MP8859_GO_BIT 0x01 +#define MP8859_IOUT_LIM_MASK 0x7f + #define MP8859_ENABLE_MASK 0x80 #define MP8859_DISCHG_EN_MASK 0x10 #define MP8859_MODE_MASK 0x08 @@ -131,6 +133,58 @@ static int mp8859_set_mode(struct regulator_dev *rdev, unsigned int mode) MP8859_MODE_MASK, val); } +static int mp8859_set_current_limit(struct regulator_dev *rdev, + int min_uA, int max_uA) +{ + unsigned int cur_val, new_val; + int ret, i; + + /* Steps of 50mA */ + new_val = max_uA / 50000; + if (new_val > MP8859_IOUT_LIM_MASK) + return -EINVAL; + if (new_val == 0) + return -EINVAL; + + /* + * If the regulator is limiting then ramp gradually as per + * datasheet, otherwise just set the value directly. + */ + ret = regmap_read(rdev->regmap, MP8859_STATUS_REG, &cur_val); + if (ret != 0) + return ret; + if (!(cur_val & MP8859_CC_CV_MASK)) { + return regmap_update_bits(rdev->regmap, MP8859_IOUT_LIM_REG, + MP8859_IOUT_LIM_MASK, new_val); + } + + ret = regmap_read(rdev->regmap, MP8859_IOUT_LIM_REG, &cur_val); + if (ret != 0) + return ret; + + if (cur_val >= new_val) { + for (i = cur_val; i >= new_val; i--) { + ret = regmap_update_bits(rdev->regmap, + MP8859_IOUT_LIM_REG, + MP8859_IOUT_LIM_MASK, + cur_val - i); + if (ret != 0) + return ret; + } + } else { + for (i = cur_val; i <= new_val; i++) { + ret = regmap_update_bits(rdev->regmap, + MP8859_IOUT_LIM_REG, + MP8859_IOUT_LIM_MASK, + cur_val + i); + if (ret != 0) + return ret; + } + } + + return 0; +} + static int mp8859_get_status(struct regulator_dev *rdev) { unsigned int val; @@ -241,6 +295,7 @@ static const struct regulator_ops mp8859_ops = { .set_mode = mp8859_set_mode, .get_mode = mp8859_get_mode, .set_active_discharge = regulator_set_active_discharge_regmap, + .set_current_limit = mp8859_set_current_limit, .get_status = mp8859_get_status, .get_error_flags = mp8859_get_error_flags, }; From efe9a6da2292e68d45cd9e0eb1c31b85841d0c2a Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 23 Feb 2024 17:07:51 -0700 Subject: [PATCH 0804/1038] regulator: dt-bindings: gpio-regulator: Fix "gpios-states" and "states" array bounds The minimum number of array entries for "gpios-states" was not not specified, so the the default is the same as the max (8). The minimum is also missing from "states", and the maximum is also wrong as it should be 2^(# of GPIO lines). Since there can be 1 to 8 GPIOs, the "states" range should be 2 to 256. Signed-off-by: Rob Herring Link: https://msgid.link/r/20240224000752.3830665-1-robh@kernel.org Signed-off-by: Mark Brown --- .../devicetree/bindings/regulator/gpio-regulator.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml index f4c1f36e52e9..a34e85754658 100644 --- a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml @@ -47,6 +47,7 @@ properties: 1: HIGH Default is LOW if nothing else is specified. $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 maxItems: 8 items: enum: [0, 1] @@ -57,7 +58,8 @@ properties: regulator and matching GPIO configurations to achieve them. If there are no states in the "states" array, use a fixed regulator instead. $ref: /schemas/types.yaml#/definitions/uint32-matrix - maxItems: 8 + minItems: 2 + maxItems: 256 items: items: - description: Voltage in microvolts From 7b1d87af14d9ae902ed0c5dc5fabf4eea5abdf02 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 19 Feb 2024 16:33:18 -0600 Subject: [PATCH 0805/1038] spi: add spi_optimize_message() APIs This adds a new spi_optimize_message() function that can be used to optimize SPI messages that are used more than once. Peripheral drivers that use the same message multiple times can use this API to perform SPI message validation and controller-specific optimizations once and then reuse the message while avoiding the overhead of revalidating the message on each spi_(a)sync() call. Internally, the SPI core will also call this function for each message if the peripheral driver did not explicitly call it. This is done to so that controller drivers don't have to have multiple code paths for optimized and non-optimized messages. A hook is provided for controller drivers to perform controller-specific optimizations. Suggested-by: Martin Sperl Link: https://lore.kernel.org/linux-spi/39DEC004-10A1-47EF-9D77-276188D2580C@martin.sperl.org/ Signed-off-by: David Lechner Link: https://msgid.link/r/20240219-mainline-spi-precook-message-v2-1-4a762c6701b9@baylibre.com Reviewed-by: Jonathan Cameron Signed-off-by: Mark Brown --- drivers/spi/spi.c | 151 ++++++++++++++++++++++++++++++++++++++-- include/linux/spi/spi.h | 20 ++++++ 2 files changed, 167 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index c2b10e2c75f0..f68d92b57543 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -2106,6 +2106,41 @@ struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr) } EXPORT_SYMBOL_GPL(spi_get_next_queued_message); +/* + * __spi_unoptimize_message - shared implementation of spi_unoptimize_message() + * and spi_maybe_unoptimize_message() + * @msg: the message to unoptimize + * + * Peripheral drivers should use spi_unoptimize_message() and callers inside + * core should use spi_maybe_unoptimize_message() rather than calling this + * function directly. + * + * It is not valid to call this on a message that is not currently optimized. + */ +static void __spi_unoptimize_message(struct spi_message *msg) +{ + struct spi_controller *ctlr = msg->spi->controller; + + if (ctlr->unoptimize_message) + ctlr->unoptimize_message(msg); + + msg->optimized = false; + msg->opt_state = NULL; +} + +/* + * spi_maybe_unoptimize_message - unoptimize msg not managed by a peripheral + * @msg: the message to unoptimize + * + * This function is used to unoptimize a message if and only if it was + * optimized by the core (via spi_maybe_optimize_message()). + */ +static void spi_maybe_unoptimize_message(struct spi_message *msg) +{ + if (!msg->pre_optimized && msg->optimized) + __spi_unoptimize_message(msg); +} + /** * spi_finalize_current_message() - the current message is complete * @ctlr: the controller to return the message to @@ -2153,6 +2188,8 @@ void spi_finalize_current_message(struct spi_controller *ctlr) mesg->prepared = false; + spi_maybe_unoptimize_message(mesg); + WRITE_ONCE(ctlr->cur_msg_incomplete, false); smp_mb(); /* See __spi_pump_transfer_message()... */ if (READ_ONCE(ctlr->cur_msg_need_completion)) @@ -4194,6 +4231,110 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) return 0; } +/* + * __spi_optimize_message - shared implementation for spi_optimize_message() + * and spi_maybe_optimize_message() + * @spi: the device that will be used for the message + * @msg: the message to optimize + * + * Peripheral drivers will call spi_optimize_message() and the spi core will + * call spi_maybe_optimize_message() instead of calling this directly. + * + * It is not valid to call this on a message that has already been optimized. + * + * Return: zero on success, else a negative error code + */ +static int __spi_optimize_message(struct spi_device *spi, + struct spi_message *msg) +{ + struct spi_controller *ctlr = spi->controller; + int ret; + + ret = __spi_validate(spi, msg); + if (ret) + return ret; + + if (ctlr->optimize_message) { + ret = ctlr->optimize_message(msg); + if (ret) + return ret; + } + + msg->optimized = true; + + return 0; +} + +/* + * spi_maybe_optimize_message - optimize message if it isn't already pre-optimized + * @spi: the device that will be used for the message + * @msg: the message to optimize + * Return: zero on success, else a negative error code + */ +static int spi_maybe_optimize_message(struct spi_device *spi, + struct spi_message *msg) +{ + if (msg->pre_optimized) + return 0; + + return __spi_optimize_message(spi, msg); +} + +/** + * spi_optimize_message - do any one-time validation and setup for a SPI message + * @spi: the device that will be used for the message + * @msg: the message to optimize + * + * Peripheral drivers that reuse the same message repeatedly may call this to + * perform as much message prep as possible once, rather than repeating it each + * time a message transfer is performed to improve throughput and reduce CPU + * usage. + * + * Once a message has been optimized, it cannot be modified with the exception + * of updating the contents of any xfer->tx_buf (the pointer can't be changed, + * only the data in the memory it points to). + * + * Calls to this function must be balanced with calls to spi_unoptimize_message() + * to avoid leaking resources. + * + * Context: can sleep + * Return: zero on success, else a negative error code + */ +int spi_optimize_message(struct spi_device *spi, struct spi_message *msg) +{ + int ret; + + ret = __spi_optimize_message(spi, msg); + if (ret) + return ret; + + /* + * This flag indicates that the peripheral driver called spi_optimize_message() + * and therefore we shouldn't unoptimize message automatically when finalizing + * the message but rather wait until spi_unoptimize_message() is called + * by the peripheral driver. + */ + msg->pre_optimized = true; + + return 0; +} +EXPORT_SYMBOL_GPL(spi_optimize_message); + +/** + * spi_unoptimize_message - releases any resources allocated by spi_optimize_message() + * @msg: the message to unoptimize + * + * Calls to this function must be balanced with calls to spi_optimize_message(). + * + * Context: can sleep + */ +void spi_unoptimize_message(struct spi_message *msg) +{ + __spi_unoptimize_message(msg); + msg->pre_optimized = false; +} +EXPORT_SYMBOL_GPL(spi_unoptimize_message); + static int __spi_async(struct spi_device *spi, struct spi_message *message) { struct spi_controller *ctlr = spi->controller; @@ -4258,8 +4399,8 @@ int spi_async(struct spi_device *spi, struct spi_message *message) int ret; unsigned long flags; - ret = __spi_validate(spi, message); - if (ret != 0) + ret = spi_maybe_optimize_message(spi, message); + if (ret) return ret; spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); @@ -4271,6 +4412,8 @@ int spi_async(struct spi_device *spi, struct spi_message *message) spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); + spi_maybe_unoptimize_message(message); + return ret; } EXPORT_SYMBOL_GPL(spi_async); @@ -4331,8 +4474,8 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) return -ESHUTDOWN; } - status = __spi_validate(spi, message); - if (status != 0) + status = spi_maybe_optimize_message(spi, message); + if (status) return status; SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync); diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 2b8e2746769a..ddfb66dd4caf 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -475,6 +475,8 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * * @set_cs: set the logic level of the chip select line. May be called * from interrupt context. + * @optimize_message: optimize the message for reuse + * @unoptimize_message: release resources allocated by optimize_message * @prepare_message: set up the controller to transfer a single message, * for example doing DMA mapping. Called from threaded * context. @@ -715,6 +717,8 @@ struct spi_controller { struct completion xfer_completion; size_t max_dma_len; + int (*optimize_message)(struct spi_message *msg); + int (*unoptimize_message)(struct spi_message *msg); int (*prepare_transfer_hardware)(struct spi_controller *ctlr); int (*transfer_one_message)(struct spi_controller *ctlr, struct spi_message *mesg); @@ -1111,6 +1115,8 @@ struct spi_transfer { * @spi: SPI device to which the transaction is queued * @is_dma_mapped: if true, the caller provided both DMA and CPU virtual * addresses for each transfer buffer + * @pre_optimized: peripheral driver pre-optimized the message + * @optimized: the message is in the optimized state * @prepared: spi_prepare_message was called for the this message * @status: zero for success, else negative errno * @complete: called to report transaction completions @@ -1120,6 +1126,7 @@ struct spi_transfer { * successful segments * @queue: for use by whichever driver currently owns the message * @state: for use by whichever driver currently owns the message + * @opt_state: for use by whichever driver currently owns the message * @resources: for resource management when the SPI message is processed * * A @spi_message is used to execute an atomic sequence of data transfers, @@ -1143,6 +1150,11 @@ struct spi_message { unsigned is_dma_mapped:1; + /* spi_optimize_message() was called for this message */ + bool pre_optimized; + /* __spi_optimize_message() was called for this message */ + bool optimized; + /* spi_prepare_message() was called for this message */ bool prepared; @@ -1172,6 +1184,11 @@ struct spi_message { */ struct list_head queue; void *state; + /* + * Optional state for use by controller driver between calls to + * __spi_optimize_message() and __spi_unoptimize_message(). + */ + void *opt_state; /* List of spi_res resources when the SPI message is processed */ struct list_head resources; @@ -1255,6 +1272,9 @@ static inline void spi_message_free(struct spi_message *m) kfree(m); } +extern int spi_optimize_message(struct spi_device *spi, struct spi_message *msg); +extern void spi_unoptimize_message(struct spi_message *msg); + extern int spi_setup(struct spi_device *spi); extern int spi_async(struct spi_device *spi, struct spi_message *message); extern int spi_slave_abort(struct spi_device *spi); From fab53fea21a909e4e0656764a8ee7c356fe89d6f Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 19 Feb 2024 16:33:19 -0600 Subject: [PATCH 0806/1038] spi: move splitting transfers to spi_optimize_message() Splitting transfers is an expensive operation so we can potentially optimize it by doing it only once per optimization of the message instead of repeating each time the message is transferred. The transfer splitting functions are currently the only user of spi_res_alloc() so spi_res_release() can be safely moved at this time from spi_finalize_current_message() to spi_unoptimize_message(). The doc comments of the public functions for splitting transfers are also updated so that callers will know when it is safe to call them to ensure proper resource management. Reviewed-by: Jonathan Cameron Signed-off-by: David Lechner Link: https://msgid.link/r/20240219-mainline-spi-precook-message-v2-2-4a762c6701b9@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 110 ++++++++++++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 42 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index f68d92b57543..ba4d3fde2054 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1747,38 +1747,6 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, trace_spi_message_start(msg); - /* - * If an SPI controller does not support toggling the CS line on each - * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO - * for the CS line, we can emulate the CS-per-word hardware function by - * splitting transfers into one-word transfers and ensuring that - * cs_change is set for each transfer. - */ - if ((msg->spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || - spi_is_csgpiod(msg->spi))) { - ret = spi_split_transfers_maxwords(ctlr, msg, 1); - if (ret) { - msg->status = ret; - spi_finalize_current_message(ctlr); - return ret; - } - - list_for_each_entry(xfer, &msg->transfers, transfer_list) { - /* Don't change cs_change on the last entry in the list */ - if (list_is_last(&xfer->transfer_list, &msg->transfers)) - break; - xfer->cs_change = 1; - } - } else { - ret = spi_split_transfers_maxsize(ctlr, msg, - spi_max_transfer_size(msg->spi)); - if (ret) { - msg->status = ret; - spi_finalize_current_message(ctlr); - return ret; - } - } - if (ctlr->prepare_message) { ret = ctlr->prepare_message(ctlr, msg); if (ret) { @@ -2124,6 +2092,8 @@ static void __spi_unoptimize_message(struct spi_message *msg) if (ctlr->unoptimize_message) ctlr->unoptimize_message(msg); + spi_res_release(ctlr, msg); + msg->optimized = false; msg->opt_state = NULL; } @@ -2169,15 +2139,6 @@ void spi_finalize_current_message(struct spi_controller *ctlr) spi_unmap_msg(ctlr, mesg); - /* - * In the prepare_messages callback the SPI bus has the opportunity - * to split a transfer to smaller chunks. - * - * Release the split transfers here since spi_map_msg() is done on - * the split transfers. - */ - spi_res_release(ctlr, mesg); - if (mesg->prepared && ctlr->unprepare_message) { ret = ctlr->unprepare_message(ctlr, mesg); if (ret) { @@ -3819,6 +3780,10 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, * @msg: the @spi_message to transform * @maxsize: the maximum when to apply this * + * This function allocates resources that are automatically freed during the + * spi message unoptimize phase so this function should only be called from + * optimize_message callbacks. + * * Return: status of transformation */ int spi_split_transfers_maxsize(struct spi_controller *ctlr, @@ -3857,6 +3822,10 @@ EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize); * @msg: the @spi_message to transform * @maxwords: the number of words to limit each transfer to * + * This function allocates resources that are automatically freed during the + * spi message unoptimize phase so this function should only be called from + * optimize_message callbacks. + * * Return: status of transformation */ int spi_split_transfers_maxwords(struct spi_controller *ctlr, @@ -4231,6 +4200,57 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) return 0; } +/* + * spi_split_transfers - generic handling of transfer splitting + * @msg: the message to split + * + * Under certain conditions, a SPI controller may not support arbitrary + * transfer sizes or other features required by a peripheral. This function + * will split the transfers in the message into smaller transfers that are + * supported by the controller. + * + * Controllers with special requirements not covered here can also split + * transfers in the optimize_message() callback. + * + * Context: can sleep + * Return: zero on success, else a negative error code + */ +static int spi_split_transfers(struct spi_message *msg) +{ + struct spi_controller *ctlr = msg->spi->controller; + struct spi_transfer *xfer; + int ret; + + /* + * If an SPI controller does not support toggling the CS line on each + * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO + * for the CS line, we can emulate the CS-per-word hardware function by + * splitting transfers into one-word transfers and ensuring that + * cs_change is set for each transfer. + */ + if ((msg->spi->mode & SPI_CS_WORD) && + (!(ctlr->mode_bits & SPI_CS_WORD) || spi_is_csgpiod(msg->spi))) { + ret = spi_split_transfers_maxwords(ctlr, msg, 1); + if (ret) + return ret; + + list_for_each_entry(xfer, &msg->transfers, transfer_list) { + /* Don't change cs_change on the last entry in the list */ + if (list_is_last(&xfer->transfer_list, &msg->transfers)) + break; + + xfer->cs_change = 1; + } + } else { + ret = spi_split_transfers_maxsize(ctlr, msg, + spi_max_transfer_size(msg->spi)); + if (ret) + return ret; + } + + return 0; +} + /* * __spi_optimize_message - shared implementation for spi_optimize_message() * and spi_maybe_optimize_message() @@ -4254,10 +4274,16 @@ static int __spi_optimize_message(struct spi_device *spi, if (ret) return ret; + ret = spi_split_transfers(msg); + if (ret) + return ret; + if (ctlr->optimize_message) { ret = ctlr->optimize_message(msg); - if (ret) + if (ret) { + spi_res_release(ctlr, msg); return ret; + } } msg->optimized = true; From c2bcfe7c6edf418d5adf731a7d60a8abd81e952f Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 19 Feb 2024 16:33:20 -0600 Subject: [PATCH 0807/1038] spi: stm32: move splitting transfers to optimize_message Since splitting transfers was moved to spi_optimize_message() in the core SPI code, we now need to use the optimize_message callback in the STM32 SPI driver to ensure that the operation is only performed once when spi_optimize_message() is used by peripheral drivers explicitly. Signed-off-by: David Lechner Link: https://msgid.link/r/20240219-mainline-spi-precook-message-v2-3-4a762c6701b9@baylibre.com Reviewed-by: Jonathan Cameron Signed-off-by: Mark Brown --- drivers/spi/spi-stm32.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index c32e57bb38bd..e4e7ddb7524a 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1118,6 +1118,21 @@ static irqreturn_t stm32h7_spi_irq_thread(int irq, void *dev_id) return IRQ_HANDLED; } +static int stm32_spi_optimize_message(struct spi_message *msg) +{ + struct spi_controller *ctrl = msg->spi->controller; + struct stm32_spi *spi = spi_controller_get_devdata(ctrl); + + /* On STM32H7, messages should not exceed a maximum size set + * later via the set_number_of_data function. In order to + * ensure that, split large messages into several messages + */ + if (spi->cfg->set_number_of_data) + return spi_split_transfers_maxwords(ctrl, msg, spi->t_size_max); + + return 0; +} + /** * stm32_spi_prepare_msg - set up the controller to transfer a single message * @ctrl: controller interface @@ -1163,18 +1178,6 @@ static int stm32_spi_prepare_msg(struct spi_controller *ctrl, !!(spi_dev->mode & SPI_LSB_FIRST), !!(spi_dev->mode & SPI_CS_HIGH)); - /* On STM32H7, messages should not exceed a maximum size setted - * afterward via the set_number_of_data function. In order to - * ensure that, split large messages into several messages - */ - if (spi->cfg->set_number_of_data) { - int ret; - - ret = spi_split_transfers_maxwords(ctrl, msg, spi->t_size_max); - if (ret) - return ret; - } - spin_lock_irqsave(&spi->lock, flags); /* CPOL, CPHA and LSB FIRST bits have common register */ @@ -2180,6 +2183,7 @@ static int stm32_spi_probe(struct platform_device *pdev) ctrl->max_speed_hz = spi->clk_rate / spi->cfg->baud_rate_div_min; ctrl->min_speed_hz = spi->clk_rate / spi->cfg->baud_rate_div_max; ctrl->use_gpio_descriptors = true; + ctrl->optimize_message = stm32_spi_optimize_message; ctrl->prepare_message = stm32_spi_prepare_msg; ctrl->transfer_one = stm32_spi_transfer_one; ctrl->unprepare_message = stm32_spi_unprepare_msg; From 7dba2adb063bcf7a293eacb88980e0975b1fb1fd Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 19 Feb 2024 16:33:21 -0600 Subject: [PATCH 0808/1038] spi: axi-spi-engine: move message compile to optimize_message In the AXI SPI Engine driver, compiling the message is an expensive operation. Previously, it was done per message transfer in the prepare_message hook. This patch moves the message compile to the optimize_message hook so that it is only done once per message in cases where the peripheral driver calls spi_optimize_message(). This can be a significant performance improvement for some peripherals. For example, the ad7380 driver saw a 13% improvement in throughput when using the AXI SPI Engine driver with this patch. Since we now need two message states, one for the optimization stage that doesn't change for the lifetime of the message and one that is reset on each transfer for managing the current transfer state, the old msg->state is split into msg->opt_state and spi_engine->msg_state. The latter is included in the driver struct now since there is only one current message at a time that can ever use it and it is in a hot path so avoiding allocating a new one on each message transfer saves a few cpu cycles and lets us get rid of the prepare_message callback. Signed-off-by: David Lechner Link: https://msgid.link/r/20240219-mainline-spi-precook-message-v2-4-4a762c6701b9@baylibre.com Reviewed-by: Jonathan Cameron Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 40 ++++++++++++++------------------ 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index ca66d202f0e2..6177c1a8d56e 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -109,6 +109,7 @@ struct spi_engine { spinlock_t lock; void __iomem *base; + struct spi_engine_message_state msg_state; struct completion msg_complete; unsigned int int_enable; }; @@ -499,17 +500,11 @@ static irqreturn_t spi_engine_irq(int irq, void *devid) return IRQ_HANDLED; } -static int spi_engine_prepare_message(struct spi_controller *host, - struct spi_message *msg) +static int spi_engine_optimize_message(struct spi_message *msg) { struct spi_engine_program p_dry, *p; - struct spi_engine_message_state *st; size_t size; - st = kzalloc(sizeof(*st), GFP_KERNEL); - if (!st) - return -ENOMEM; - spi_engine_precompile_message(msg); p_dry.length = 0; @@ -517,31 +512,22 @@ static int spi_engine_prepare_message(struct spi_controller *host, size = sizeof(*p->instructions) * (p_dry.length + 1); p = kzalloc(sizeof(*p) + size, GFP_KERNEL); - if (!p) { - kfree(st); + if (!p) return -ENOMEM; - } spi_engine_compile_message(msg, false, p); spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC( AXI_SPI_ENGINE_CUR_MSG_SYNC_ID)); - st->p = p; - st->cmd_buf = p->instructions; - st->cmd_length = p->length; - msg->state = st; + msg->opt_state = p; return 0; } -static int spi_engine_unprepare_message(struct spi_controller *host, - struct spi_message *msg) +static int spi_engine_unoptimize_message(struct spi_message *msg) { - struct spi_engine_message_state *st = msg->state; - - kfree(st->p); - kfree(st); + kfree(msg->opt_state); return 0; } @@ -550,10 +536,18 @@ static int spi_engine_transfer_one_message(struct spi_controller *host, struct spi_message *msg) { struct spi_engine *spi_engine = spi_controller_get_devdata(host); - struct spi_engine_message_state *st = msg->state; + struct spi_engine_message_state *st = &spi_engine->msg_state; + struct spi_engine_program *p = msg->opt_state; unsigned int int_enable = 0; unsigned long flags; + /* reinitialize message state for this transfer */ + memset(st, 0, sizeof(*st)); + st->p = p; + st->cmd_buf = p->instructions; + st->cmd_length = p->length; + msg->state = st; + reinit_completion(&spi_engine->msg_complete); spin_lock_irqsave(&spi_engine->lock, flags); @@ -658,8 +652,8 @@ static int spi_engine_probe(struct platform_device *pdev) host->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); host->max_speed_hz = clk_get_rate(spi_engine->ref_clk) / 2; host->transfer_one_message = spi_engine_transfer_one_message; - host->prepare_message = spi_engine_prepare_message; - host->unprepare_message = spi_engine_unprepare_message; + host->optimize_message = spi_engine_optimize_message; + host->unoptimize_message = spi_engine_unoptimize_message; host->num_chipselect = 8; if (host->max_speed_hz == 0) From e63aef9c9121e5061cbf5112d12cadc9da399692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Fri, 16 Feb 2024 17:42:19 +0100 Subject: [PATCH 0809/1038] spi: spi-mem: add statistics support to ->exec_op() calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Current behavior is that spi-mem operations do not increment statistics, neither per-controller nor per-device, if ->exec_op() is used. For operations that do NOT use ->exec_op(), stats are increased as the usual spi_sync() is called. The newly implemented spi_mem_add_op_stats() function is strongly inspired by spi_statistics_add_transfer_stats(); locking logic and l2len computation comes from there. Statistics that are being filled: bytes{,_rx,_tx}, messages, transfers, errors, timedout, transfer_bytes_histo_*. Note about messages & transfers counters: in the fallback to spi_sync() case, there are from 1 to 4 transfers per message. We only register one big transfer in the ->exec_op() case as that is closer to reality. This patch is NOT touching: - spi_async, spi_sync, spi_sync_immediate: those counters describe precise function calls, incrementing them would be lying. I believe comparing the messages counter to spi_async+spi_sync is a good way to detect ->exec_op() calls, but I might be missing edge cases knowledge. - transfers_split_maxsize: splitting cannot happen if ->exec_op() is provided. Reviewed-by: Dhruva Gole Signed-off-by: Théo Lebrun Reviewed-by: Tudor Ambarus Link: https://msgid.link/r/20240216-spi-mem-stats-v2-1-9256dfe4887d@bootlin.com Signed-off-by: Mark Brown --- drivers/spi/spi-mem.c | 49 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index 2dc8ceb85374..c9d6d42a88f5 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -297,6 +297,49 @@ static void spi_mem_access_end(struct spi_mem *mem) pm_runtime_put(ctlr->dev.parent); } +static void spi_mem_add_op_stats(struct spi_statistics __percpu *pcpu_stats, + const struct spi_mem_op *op, int exec_op_ret) +{ + struct spi_statistics *stats; + u64 len, l2len; + + get_cpu(); + stats = this_cpu_ptr(pcpu_stats); + u64_stats_update_begin(&stats->syncp); + + /* + * We do not have the concept of messages or transfers. Let's consider + * that one operation is equivalent to one message and one transfer. + */ + u64_stats_inc(&stats->messages); + u64_stats_inc(&stats->transfers); + + /* Use the sum of all lengths as bytes count and histogram value. */ + len = op->cmd.nbytes + op->addr.nbytes; + len += op->dummy.nbytes + op->data.nbytes; + u64_stats_add(&stats->bytes, len); + l2len = min(fls(len), SPI_STATISTICS_HISTO_SIZE) - 1; + u64_stats_inc(&stats->transfer_bytes_histo[l2len]); + + /* Only account for data bytes as transferred bytes. */ + if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT) + u64_stats_add(&stats->bytes_tx, op->data.nbytes); + if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_IN) + u64_stats_add(&stats->bytes_rx, op->data.nbytes); + + /* + * A timeout is not an error, following the same behavior as + * spi_transfer_one_message(). + */ + if (exec_op_ret == -ETIMEDOUT) + u64_stats_inc(&stats->timedout); + else if (exec_op_ret) + u64_stats_inc(&stats->errors); + + u64_stats_update_end(&stats->syncp); + put_cpu(); +} + /** * spi_mem_exec_op() - Execute a memory operation * @mem: the SPI memory @@ -339,8 +382,12 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) * read path) and expect the core to use the regular SPI * interface in other cases. */ - if (!ret || ret != -ENOTSUPP || ret != -EOPNOTSUPP) + if (!ret || ret != -ENOTSUPP || ret != -EOPNOTSUPP) { + spi_mem_add_op_stats(ctlr->pcpu_statistics, op, ret); + spi_mem_add_op_stats(mem->spi->pcpu_statistics, op, ret); + return ret; + } } tmpbufsize = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; From 11498d99008fbe6a23e827773d9ee47728f23aa6 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 14 Feb 2024 11:45:06 +0100 Subject: [PATCH 0810/1038] gpio: sim: add lockdep asserts We have three functions in gpio-sim that are called with the device lock already held. We use the "_unlocked" suffix in their names to indicate that. This has proven to be confusing though as the naming convention in the kernel varies between using "_locked" or "_unlocked" for this purpose. Naming convention also doesn't enforce anything. Let's remove the suffix and add lockdep annotation at the top of these functions. This makes it clear the function requires a lock to be held (and which one specifically!) as well as results in a warning if it's not the case. The only place where the information is lost is the place where the function is called but the caller doesn't care about that information anyway. Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-sim.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c index c4106e37e6db..db42dc5616e4 100644 --- a/drivers/gpio/gpio-sim.c +++ b/drivers/gpio/gpio-sim.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -697,8 +698,10 @@ static struct gpio_sim_device *gpio_sim_hog_get_device(struct gpio_sim_hog *hog) return gpio_sim_line_get_device(line); } -static bool gpio_sim_device_is_live_unlocked(struct gpio_sim_device *dev) +static bool gpio_sim_device_is_live(struct gpio_sim_device *dev) { + lockdep_assert_held(&dev->lock); + return !!dev->pdev; } @@ -737,7 +740,7 @@ gpio_sim_device_config_live_show(struct config_item *item, char *page) bool live; scoped_guard(mutex, &dev->lock) - live = gpio_sim_device_is_live_unlocked(dev); + live = gpio_sim_device_is_live(dev); return sprintf(page, "%c\n", live ? '1' : '0'); } @@ -926,7 +929,7 @@ static bool gpio_sim_bank_labels_non_unique(struct gpio_sim_device *dev) return false; } -static int gpio_sim_device_activate_unlocked(struct gpio_sim_device *dev) +static int gpio_sim_device_activate(struct gpio_sim_device *dev) { struct platform_device_info pdevinfo; struct fwnode_handle *swnode; @@ -934,6 +937,8 @@ static int gpio_sim_device_activate_unlocked(struct gpio_sim_device *dev) struct gpio_sim_bank *bank; int ret; + lockdep_assert_held(&dev->lock); + if (list_empty(&dev->bank_list)) return -ENODATA; @@ -998,10 +1003,12 @@ static int gpio_sim_device_activate_unlocked(struct gpio_sim_device *dev) return 0; } -static void gpio_sim_device_deactivate_unlocked(struct gpio_sim_device *dev) +static void gpio_sim_device_deactivate(struct gpio_sim_device *dev) { struct fwnode_handle *swnode; + lockdep_assert_held(&dev->lock); + swnode = dev_fwnode(&dev->pdev->dev); platform_device_unregister(dev->pdev); gpio_sim_remove_hogs(dev); @@ -1023,12 +1030,12 @@ gpio_sim_device_config_live_store(struct config_item *item, guard(mutex)(&dev->lock); - if (live == gpio_sim_device_is_live_unlocked(dev)) + if (live == gpio_sim_device_is_live(dev)) ret = -EPERM; else if (live) - ret = gpio_sim_device_activate_unlocked(dev); + ret = gpio_sim_device_activate(dev); else - gpio_sim_device_deactivate_unlocked(dev); + gpio_sim_device_deactivate(dev); return ret ?: count; } @@ -1069,7 +1076,7 @@ static ssize_t gpio_sim_bank_config_chip_name_show(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return device_for_each_child(&dev->pdev->dev, &ctx, gpio_sim_emit_chip_name); @@ -1098,7 +1105,7 @@ static ssize_t gpio_sim_bank_config_label_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; trimmed = gpio_sim_strdup_trimmed(page, count); @@ -1142,7 +1149,7 @@ gpio_sim_bank_config_num_lines_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; bank->num_lines = num_lines; @@ -1179,7 +1186,7 @@ static ssize_t gpio_sim_line_config_name_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; trimmed = gpio_sim_strdup_trimmed(page, count); @@ -1219,7 +1226,7 @@ static ssize_t gpio_sim_hog_config_name_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; trimmed = gpio_sim_strdup_trimmed(page, count); @@ -1274,7 +1281,7 @@ gpio_sim_hog_config_direction_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; if (sysfs_streq(page, "input")) @@ -1392,7 +1399,7 @@ gpio_sim_bank_config_make_line_group(struct config_group *group, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return ERR_PTR(-EBUSY); line = kzalloc(sizeof(*line), GFP_KERNEL); @@ -1445,7 +1452,7 @@ gpio_sim_device_config_make_bank_group(struct config_group *group, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return ERR_PTR(-EBUSY); bank = kzalloc(sizeof(*bank), GFP_KERNEL); @@ -1467,8 +1474,8 @@ static void gpio_sim_device_config_group_release(struct config_item *item) struct gpio_sim_device *dev = to_gpio_sim_device(item); scoped_guard(mutex, &dev->lock) { - if (gpio_sim_device_is_live_unlocked(dev)) - gpio_sim_device_deactivate_unlocked(dev); + if (gpio_sim_device_is_live(dev)) + gpio_sim_device_deactivate(dev); } mutex_destroy(&dev->lock); From 840a97e2fbaf5315f9705b97e99aa97e5e5fd6cb Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 20 Feb 2024 14:54:31 +0100 Subject: [PATCH 0811/1038] gpio: sim: delimit the fwnode name with a ":" when generating labels Typically, whenever a human-readable name is created for objects using a software node, its name is delimited with ":" as dashes are often used in other parts of the name. Make gpio-sim use the same pattern. This results in better looking default names: gpio-sim.0:node0 gpio-sim.0:node1 gpio-sim.1:node0 Signed-off-by: Bartosz Golaszewski Acked-by: Linus Walleij --- drivers/gpio/gpio-sim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c index db42dc5616e4..a2706fe1340a 100644 --- a/drivers/gpio/gpio-sim.c +++ b/drivers/gpio/gpio-sim.c @@ -421,7 +421,7 @@ static int gpio_sim_add_bank(struct fwnode_handle *swnode, struct device *dev) ret = fwnode_property_read_string(swnode, "gpio-sim,label", &label); if (ret) { - label = devm_kasprintf(dev, GFP_KERNEL, "%s-%pfwP", + label = devm_kasprintf(dev, GFP_KERNEL, "%s:%pfwP", dev_name(dev), swnode); if (!label) return -ENOMEM; @@ -836,7 +836,7 @@ static int gpio_sim_add_hogs(struct gpio_sim_device *dev) GFP_KERNEL); else hog->chip_label = kasprintf(GFP_KERNEL, - "gpio-sim.%u-%pfwP", + "gpio-sim.%u:%pfwP", dev->id, bank->swnode); if (!hog->chip_label) { From 5d60c1e61fda628619605f5917aba62f071b7106 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Wed, 21 Feb 2024 11:21:03 +0100 Subject: [PATCH 0812/1038] gpio: don't warn about removing GPIO chips with active users anymore With SRCU we can now correctly handle the situation when a GPIO provider is removed while having users still holding references to GPIO descriptors. Remove all warnings emitted in this situation. Suggested-by: Kent Gibson Signed-off-by: Bartosz Golaszewski Reviewed-by: Herve Codina --- drivers/gpio/gpiolib.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 3c22920bd201..63e793a410e3 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1106,7 +1106,6 @@ EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key); void gpiochip_remove(struct gpio_chip *gc) { struct gpio_device *gdev = gc->gpiodev; - unsigned int i; /* FIXME: should the legacy sysfs handling be moved to gpio_device? */ gpiochip_sysfs_unregister(gdev); @@ -1130,15 +1129,6 @@ void gpiochip_remove(struct gpio_chip *gc) */ gpiochip_set_data(gc, NULL); - for (i = 0; i < gdev->ngpio; i++) { - if (test_bit(FLAG_REQUESTED, &gdev->descs[i].flags)) - break; - } - - if (i != gdev->ngpio) - dev_crit(&gdev->dev, - "REMOVING GPIOCHIP WITH GPIOS STILL REQUESTED\n"); - /* * The gpiochip side puts its use of the device to rest here: * if there are no userspace clients, the chardev and device will @@ -2329,10 +2319,9 @@ int gpiod_request(struct gpio_desc *desc, const char *label) return ret; } -static bool gpiod_free_commit(struct gpio_desc *desc) +static void gpiod_free_commit(struct gpio_desc *desc) { unsigned long flags; - bool ret = false; might_sleep(); @@ -2357,23 +2346,18 @@ static bool gpiod_free_commit(struct gpio_desc *desc) #ifdef CONFIG_OF_DYNAMIC WRITE_ONCE(desc->hog, NULL); #endif - ret = true; desc_set_label(desc, NULL); WRITE_ONCE(desc->flags, flags); gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED); } - - return ret; } void gpiod_free(struct gpio_desc *desc) { VALIDATE_DESC_VOID(desc); - if (!gpiod_free_commit(desc)) - WARN_ON(1); - + gpiod_free_commit(desc); module_put(desc->gdev->owner); gpio_device_put(desc->gdev); } From 3d8bb3d3080d1609c2a6bef007ede2d8f8ffea5b Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 22 Feb 2024 11:25:12 +0100 Subject: [PATCH 0813/1038] gpio: provide for_each_hwgpio() We only provide iterators for requested GPIOs to provider drivers. In order to allow them to display debug information about all GPIOs, let's provide a variant for iterating over all GPIOs. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij --- include/linux/gpio/driver.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 9c1fbfaebaa8..175129a92656 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -551,6 +551,21 @@ DEFINE_CLASS(_gpiochip_for_each_data, }), const char **label, int *i) +/** + * for_each_hwgpio - Iterates over all GPIOs for given chip. + * @_chip: Chip to iterate over. + * @_i: Loop counter. + * @_label: Place to store the address of the label if the GPIO is requested. + * Set to NULL for unused GPIOs. + */ +#define for_each_hwgpio(_chip, _i, _label) \ + for (CLASS(_gpiochip_for_each_data, _data)(&_label, &_i); \ + *_data.i < _chip->ngpio; \ + (*_data.i)++, kfree(*(_data.label)), *_data.label = NULL) \ + if (IS_ERR(*_data.label = \ + gpiochip_dup_line_label(_chip, *_data.i))) {} \ + else + /** * for_each_requested_gpio_in_range - iterates over requested GPIOs in a given range * @_chip: the chip to query From ebb03f692f5192ca2da554e97c8461ec7498d3bf Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 22 Feb 2024 11:25:13 +0100 Subject: [PATCH 0814/1038] gpio: sim: use for_each_hwgpio() Display debugfs information about all simulated GPIOs, not only the requested ones. Signed-off-by: Bartosz Golaszewski Reviewed-by: Linus Walleij --- drivers/gpio/gpio-sim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c index a2706fe1340a..2ed5cbe7c8a8 100644 --- a/drivers/gpio/gpio-sim.c +++ b/drivers/gpio/gpio-sim.c @@ -235,10 +235,10 @@ static void gpio_sim_dbg_show(struct seq_file *seq, struct gpio_chip *gc) guard(mutex)(&chip->lock); - for_each_requested_gpio(gc, i, label) + for_each_hwgpio(gc, i, label) seq_printf(seq, " gpio-%-3d (%s) %s,%s\n", gc->base + i, - label, + label ?: "", test_bit(i, chip->direction_map) ? "input" : test_bit(i, chip->value_map) ? "output-high" : "output-low", From 8d60902f16e2a36e9138a7ac0e90eb30f35a33b5 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Thu, 22 Feb 2024 16:13:21 +0200 Subject: [PATCH 0815/1038] ACPI: property: Polish ignoring bad data nodes Clean up ignoring data nodes in mipi-disco-img.c: use { } initialiser, fix a kernel-doc warning and use isdigit(). Fixes: 5bd4edbbf920 ("ACPI: property: Ignore bad graph port nodes on Dell XPS 9315") Signed-off-by: Sakari Ailus Signed-off-by: Rafael J. Wysocki --- drivers/acpi/mipi-disco-img.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/mipi-disco-img.c b/drivers/acpi/mipi-disco-img.c index da71eb4bf6a6..d05413a0672a 100644 --- a/drivers/acpi/mipi-disco-img.c +++ b/drivers/acpi/mipi-disco-img.c @@ -732,7 +732,7 @@ static const struct dmi_system_id dmi_ignore_port_nodes[] = { DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 9315"), }, }, - { 0 } + { } }; static const char *strnext(const char *s1, const char *s2) @@ -749,7 +749,7 @@ static const char *strnext(const char *s1, const char *s2) * acpi_graph_ignore_port - Tell whether a port node should be ignored * @handle: The ACPI handle of the node (which may be a port node) * - * Returns true if a port node should be ignored and the data to that should + * Return: true if a port node should be ignored and the data to that should * come from other sources instead (Windows ACPI definitions and * ipu-bridge). This is currently used to ignore bad port nodes related to IPU6 * ("IPU?") and camera sensor devices ("LNK?") in certain Dell systems with @@ -778,12 +778,12 @@ bool acpi_graph_ignore_port(acpi_handle handle) if (!path) goto out_free; - if (!(path[0] >= '0' && path[0] <= '9' && path[1] == '.')) + if (!(isdigit(path[0]) && path[1] == '.')) goto out_free; /* Check if the node has a "PRT" prefix. */ path = strnext(path, "PRT"); - if (path && path[0] >= '0' && path[0] <= '9' && !path[1]) { + if (path && isdigit(path[0]) && !path[1]) { acpi_handle_debug(handle, "ignoring data node\n"); kfree(orig_path); From 531a0c0cdbff9cecf41073220a826f8b1132f9ab Mon Sep 17 00:00:00 2001 From: John Keeping Date: Mon, 26 Feb 2024 16:05:53 +0000 Subject: [PATCH 0816/1038] regulator: userspace-consumer: add module device table The userspace consumer can be built as a module but it cannot be automatically probed as there is no device table to match it up with device tree nodes. Add the missing macro so that the module can load automatically. Fixes: 5c51d4afcf3fd ("regulator: userspace-consumer: Handle regulator-output DT nodes") Signed-off-by: John Keeping Link: https://msgid.link/r/20240226160554.1453283-1-jkeeping@inmusicbrands.com Signed-off-by: Mark Brown --- drivers/regulator/userspace-consumer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/regulator/userspace-consumer.c b/drivers/regulator/userspace-consumer.c index 53d1b9d6f69c..86a626a4f610 100644 --- a/drivers/regulator/userspace-consumer.c +++ b/drivers/regulator/userspace-consumer.c @@ -208,6 +208,7 @@ static const struct of_device_id regulator_userspace_consumer_of_match[] = { { .compatible = "regulator-output", }, {}, }; +MODULE_DEVICE_TABLE(of, regulator_userspace_consumer_of_match); static struct platform_driver regulator_userspace_consumer_driver = { .probe = regulator_userspace_consumer_probe, From da1983355ccefcfb3f8eb410fff82e250fa87e39 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 26 Feb 2024 17:54:58 -0700 Subject: [PATCH 0817/1038] thermal: core: Move initial num_trips assignment before memcpy() When booting a CONFIG_FORTIFY_SOURCE=y kernel compiled with a toolchain that supports __counted_by() (such as clang-18 and newer), there is a panic on boot: [ 2.913770] memcpy: detected buffer overflow: 72 byte write of buffer size 0 [ 2.920834] WARNING: CPU: 2 PID: 1 at lib/string_helpers.c:1027 __fortify_report+0x5c/0x74 ... [ 3.039208] Call trace: [ 3.041643] __fortify_report+0x5c/0x74 [ 3.045469] __fortify_panic+0x18/0x20 [ 3.049209] thermal_zone_device_register_with_trips+0x4c8/0x4f8 This panic occurs because trips is counted by num_trips but num_trips is assigned after the call to memcpy(), so the fortify checks think the buffer size is zero because tz was allocated with kzalloc(). Move the num_trips assignment before the memcpy() to resolve the panic and ensure that the fortify checks work properly. Fixes: 9b0a62758665 ("thermal: core: Store zone trips table in struct thermal_zone_device") Signed-off-by: Nathan Chancellor Reviewed-by: Kees Cook Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index bb21f78b4bfa..1eabc8ebe27d 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1354,8 +1354,8 @@ thermal_zone_device_register_with_trips(const char *type, tz->device.class = thermal_class; tz->devdata = devdata; - memcpy(tz->trips, trips, num_trips * sizeof(*trips)); tz->num_trips = num_trips; + memcpy(tz->trips, trips, num_trips * sizeof(*trips)); thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); From 5340f7647294fa8ff8cf5a1bee326b2bd8340e27 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:30:49 +0100 Subject: [PATCH 0818/1038] thermal: core: Add flags to struct thermal_trip In order to allow thermal zone creators to specify the writability of trip point temperature and hysteresis on a per-trip basis, add a flags field to struct thermal_trip and define flags to represent the desired trip properties. Also make thermal_zone_device_register_with_trips() set the THERMAL_TRIP_FLAG_RW_TEMP flag for all trips covered by the writable trips mask passed to it and modify the thermal sysfs code to look at the trip flags instead of using the writable trips mask directly or checking the presence of the .set_trip_hyst() zone callback. Additionally, make trip_point_temp_store() and trip_point_hyst_store() fail with an error code if the trip passed to one of them has THERMAL_TRIP_FLAG_RW_TEMP or THERMAL_TRIP_FLAG_RW_HYST, respectively, clear in its flags. No intentional functional impact. Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 9 ++++++++- drivers/thermal/thermal_core.h | 2 +- drivers/thermal/thermal_sysfs.c | 18 +++++++++--------- include/linux/thermal.h | 8 ++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 1eabc8ebe27d..2ab495220c6e 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1278,6 +1278,7 @@ thermal_zone_device_register_with_trips(const char *type, int passive_delay, int polling_delay) { struct thermal_zone_device *tz; + struct thermal_trip *trip; int id; int result; struct thermal_governor *governor; @@ -1356,13 +1357,19 @@ thermal_zone_device_register_with_trips(const char *type, tz->devdata = devdata; tz->num_trips = num_trips; memcpy(tz->trips, trips, num_trips * sizeof(*trips)); + for_each_trip(tz, trip) { + if (mask & 1) + trip->flags |= THERMAL_TRIP_FLAG_RW_TEMP; + + mask >>= 1; + } thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); /* sys I/F */ /* Add nodes that are always present via .groups */ - result = thermal_zone_create_device_groups(tz, mask); + result = thermal_zone_create_device_groups(tz); if (result) goto remove_id; diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index e9c099ecdd0f..0d8a42bb7ce8 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -131,7 +131,7 @@ void thermal_zone_trip_updated(struct thermal_zone_device *tz, int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp); /* sysfs I/F */ -int thermal_zone_create_device_groups(struct thermal_zone_device *, int); +int thermal_zone_create_device_groups(struct thermal_zone_device *tz); void thermal_zone_destroy_device_groups(struct thermal_zone_device *); void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *); void thermal_cooling_device_destroy_sysfs(struct thermal_cooling_device *cdev); diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 83914aa2248b..39dbbdac75d0 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -392,17 +392,16 @@ static const struct attribute_group *thermal_zone_attribute_groups[] = { /** * create_trip_attrs() - create attributes for trip points * @tz: the thermal zone device - * @mask: Writeable trip point bitmap. * * helper function to instantiate sysfs entries for every trip * point and its properties of a struct thermal_zone_device. * * Return: 0 on success, the proper error value otherwise. */ -static int create_trip_attrs(struct thermal_zone_device *tz, int mask) +static int create_trip_attrs(struct thermal_zone_device *tz) { + const struct thermal_trip *trip; struct attribute **attrs; - int indx; /* This function works only for zones with at least one trip */ if (tz->num_trips <= 0) @@ -437,7 +436,9 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) return -ENOMEM; } - for (indx = 0; indx < tz->num_trips; indx++) { + for_each_trip(tz, trip) { + int indx = thermal_zone_trip_id(tz, trip); + /* create trip type attribute */ snprintf(tz->trip_type_attrs[indx].name, THERMAL_NAME_LENGTH, "trip_point_%d_type", indx); @@ -458,7 +459,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_temp_attrs[indx].name; tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show; - if (mask & (1 << indx)) { + if (trip->flags & THERMAL_TRIP_FLAG_RW_TEMP) { tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_temp_attrs[indx].attr.store = trip_point_temp_store; @@ -473,7 +474,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_hyst_attrs[indx].name; tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show; - if (tz->ops.set_trip_hyst) { + if (trip->flags & THERMAL_TRIP_FLAG_RW_HYST) { tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_hyst_attrs[indx].attr.store = trip_point_hyst_store; @@ -505,8 +506,7 @@ static void destroy_trip_attrs(struct thermal_zone_device *tz) kfree(tz->trips_attribute_group.attrs); } -int thermal_zone_create_device_groups(struct thermal_zone_device *tz, - int mask) +int thermal_zone_create_device_groups(struct thermal_zone_device *tz) { const struct attribute_group **groups; int i, size, result; @@ -522,7 +522,7 @@ int thermal_zone_create_device_groups(struct thermal_zone_device *tz, groups[i] = thermal_zone_attribute_groups[i]; if (tz->num_trips) { - result = create_trip_attrs(tz, mask); + result = create_trip_attrs(tz); if (result) { kfree(groups); diff --git a/include/linux/thermal.h b/include/linux/thermal.h index ec0559e98d6f..6eb6f3297ea0 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -64,15 +64,23 @@ enum thermal_notify_event { * @threshold: trip crossing notification threshold miliCelsius * @type: trip point type * @priv: pointer to driver data associated with this trip + * @flags: flags representing binary properties of the trip */ struct thermal_trip { int temperature; int hysteresis; int threshold; enum thermal_trip_type type; + u8 flags; void *priv; }; +#define THERMAL_TRIP_FLAG_RW_TEMP BIT(0) +#define THERMAL_TRIP_FLAG_RW_HYST BIT(1) + +#define THERMAL_TRIP_FLAG_RW (THERMAL_TRIP_FLAG_RW_TEMP | \ + THERMAL_TRIP_FLAG_RW_HYST) + struct thermal_zone_device_ops { int (*bind) (struct thermal_zone_device *, struct thermal_cooling_device *); From 46f5bef8ec2e1e05ad2fda0bcf5ac32e191ec694 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:32:03 +0100 Subject: [PATCH 0819/1038] thermal: core: Drop the .set_trip_hyst() thermal zone operation None of the users of the thermal core provides a .set_trip_hyst() thermal zone operation, so drop that callback from struct thermal_zone_device_ops and update trip_point_hyst_store() accordingly. No functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_sysfs.c | 7 ------- include/linux/thermal.h | 1 - 2 files changed, 8 deletions(-) diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 39dbbdac75d0..7c02d35384ce 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -174,18 +174,11 @@ trip_point_hyst_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (hyst != trip->hysteresis) { - if (tz->ops.set_trip_hyst) { - ret = tz->ops.set_trip_hyst(tz, trip_id, hyst); - if (ret) - goto unlock; - } - trip->hysteresis = hyst; thermal_zone_trip_updated(tz, trip); } -unlock: mutex_unlock(&tz->lock); return ret ? ret : count; diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 6eb6f3297ea0..eb4145a64c7e 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -91,7 +91,6 @@ struct thermal_zone_device_ops { int (*change_mode) (struct thermal_zone_device *, enum thermal_device_mode); int (*set_trip_temp) (struct thermal_zone_device *, int, int); - int (*set_trip_hyst) (struct thermal_zone_device *, int, int); int (*get_crit_temp) (struct thermal_zone_device *, int *); int (*set_emul_temp) (struct thermal_zone_device *, int); int (*get_trend) (struct thermal_zone_device *, From cca52f696952962f020c7e1393740d0ba07c3dc2 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 19:06:05 +0100 Subject: [PATCH 0820/1038] thermal: intel: Set THERMAL_TRIP_FLAG_RW_TEMP directly Some Intel thermal drivers need/want the temperature of their trip points to be set by user space via sysfs and so they pass nonzero writable trip masks during thermal zone registration for this purpose. It is now possible to achieve the same result by setting the THERMAL_TRIP_FLAG_RW_TEMP trip flag directly, so modify the drivers in question to do that instead of using a nonzero writable trips mask. No intentional functional impact. Note that this change is requisite for dropping the mask argument from thermal_zone_device_register_with_trips() going forward. Signed-off-by: Rafael J. Wysocki Acked-by: Daniel Lezcano --- .../int340x_thermal/int340x_thermal_zone.c | 8 +-- .../processor_thermal_device_pci.c | 3 +- .../thermal/intel/intel_quark_dts_thermal.c | 20 ++----- drivers/thermal/intel/intel_soc_dts_iosf.c | 57 ++++++++----------- drivers/thermal/intel/intel_soc_dts_iosf.h | 1 - drivers/thermal/intel/x86_pkg_temp_thermal.c | 4 +- 6 files changed, 38 insertions(+), 55 deletions(-) diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index 1b0a9e9206c4..0e8148afb490 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -129,7 +129,6 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, struct thermal_trip *zone_trips; unsigned long long trip_cnt = 0; unsigned long long hyst; - int trip_mask = 0; acpi_status status; int i, ret; @@ -140,10 +139,8 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->adev = adev; status = acpi_evaluate_integer(adev->handle, "PATC", NULL, &trip_cnt); - if (ACPI_SUCCESS(status)) { + if (ACPI_SUCCESS(status)) int34x_zone->aux_trip_nr = trip_cnt; - trip_mask = BIT(trip_cnt) - 1; - } zone_trips = kzalloc(sizeof(*zone_trips) * (trip_cnt + INT340X_THERMAL_MAX_TRIP_COUNT), GFP_KERNEL); @@ -155,6 +152,7 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, for (i = 0; i < trip_cnt; i++) { zone_trips[i].type = THERMAL_TRIP_PASSIVE; zone_trips[i].temperature = THERMAL_TEMP_INVALID; + zone_trips[i].flags |= THERMAL_TRIP_FLAG_RW_TEMP; } trip_cnt = int340x_thermal_read_trips(adev, zone_trips, trip_cnt); @@ -173,7 +171,7 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->zone = thermal_zone_device_register_with_trips( acpi_device_bid(adev), zone_trips, trip_cnt, - trip_mask, int34x_zone, + 0, int34x_zone, &zone_ops, &int340x_thermal_params, 0, 0); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index 8ff7e914ecf8..7572f3a73c54 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -249,6 +249,7 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ struct proc_thermal_pci *pci_info; struct thermal_trip psv_trip = { .type = THERMAL_TRIP_PASSIVE, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }; int irq_flag = 0, irq, ret; bool msi_irq = false; @@ -289,7 +290,7 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ psv_trip.temperature = get_trip_temp(pci_info); pci_info->tzone = thermal_zone_device_register_with_trips("TCPU_PCI", &psv_trip, - 1, 1, pci_info, + 1, 0, pci_info, &tzone_ops, &tzone_params, 0, 0); if (IS_ERR(pci_info->tzone)) { diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c index 210b822bc988..55a031fc3a9a 100644 --- a/drivers/thermal/intel/intel_quark_dts_thermal.c +++ b/drivers/thermal/intel/intel_quark_dts_thermal.c @@ -93,10 +93,6 @@ /* Quark DTS has 2 trip points: hot & catastrophic */ #define QRK_MAX_DTS_TRIPS 2 -/* If DTS not locked, all trip points are configurable */ -#define QRK_DTS_WR_MASK_SET 0x3 -/* If DTS locked, all trip points are not configurable */ -#define QRK_DTS_WR_MASK_CLR 0 #define DEFAULT_POLL_DELAY 2000 @@ -323,7 +319,6 @@ static struct soc_sensor_entry *alloc_soc_dts(void) struct soc_sensor_entry *aux_entry; int err; u32 out; - int wr_mask; aux_entry = kzalloc(sizeof(*aux_entry), GFP_KERNEL); if (!aux_entry) { @@ -337,13 +332,7 @@ static struct soc_sensor_entry *alloc_soc_dts(void) if (err) goto err_ret; - if (out & QRK_DTS_LOCK_BIT) { - aux_entry->locked = true; - wr_mask = QRK_DTS_WR_MASK_CLR; - } else { - aux_entry->locked = false; - wr_mask = QRK_DTS_WR_MASK_SET; - } + aux_entry->locked = !!(out & QRK_DTS_LOCK_BIT); /* Store DTS default state if DTS registers are not locked */ if (!aux_entry->locked) { @@ -360,6 +349,9 @@ static struct soc_sensor_entry *alloc_soc_dts(void) &aux_entry->store_ptps); if (err) goto err_ret; + + trips[QRK_DTS_ID_TP_CRITICAL].flags |= THERMAL_TRIP_FLAG_RW_TEMP; + trips[QRK_DTS_ID_TP_HOT].flags |= THERMAL_TRIP_FLAG_RW_TEMP; } trips[QRK_DTS_ID_TP_CRITICAL].temperature = get_trip_temp(QRK_DTS_ID_TP_CRITICAL); @@ -371,8 +363,8 @@ static struct soc_sensor_entry *alloc_soc_dts(void) aux_entry->tzone = thermal_zone_device_register_with_trips("quark_dts", trips, QRK_MAX_DTS_TRIPS, - wr_mask, - aux_entry, &tzone_ops, + 0, aux_entry, + &tzone_ops, NULL, 0, polling_delay); if (IS_ERR(aux_entry->tzone)) { err = PTR_ERR(aux_entry->tzone); diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c index c008eb42d82e..5a84c9166274 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.c +++ b/drivers/thermal/intel/intel_soc_dts_iosf.c @@ -202,16 +202,10 @@ static void remove_dts_thermal_zone(struct intel_soc_dts_sensor_entry *dts) } static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, - struct thermal_trip *trips, - bool critical_trip) + struct thermal_trip *trips) { - int writable_trip_cnt = SOC_MAX_DTS_TRIPS; char name[10]; - unsigned long trip; - int trip_mask; - unsigned long ptps; u32 store_ptps; - unsigned long i; int ret; /* Store status to restor on exit */ @@ -222,27 +216,21 @@ static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, dts->id = id; - if (critical_trip) - writable_trip_cnt--; - - trip_mask = GENMASK(writable_trip_cnt - 1, 0); - /* Check if the writable trip we provide is not used by BIOS */ ret = iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, SOC_DTS_OFFSET_PTPS, &store_ptps); - if (ret) - trip_mask = 0; - else { - ptps = store_ptps; - for_each_set_clump8(i, trip, &ptps, writable_trip_cnt * 8) - trip_mask &= ~BIT(i / 8); + if (!ret) { + int i; + + for (i = 0; i <= 1; i++) { + if (store_ptps & (0xFFU << i * 8)) + trips[i].flags &= ~THERMAL_TRIP_FLAG_RW_TEMP; + } } - dts->trip_mask = trip_mask; snprintf(name, sizeof(name), "soc_dts%d", id); dts->tzone = thermal_zone_device_register_with_trips(name, trips, SOC_MAX_DTS_TRIPS, - trip_mask, - dts, &tzone_ops, + 0, dts, &tzone_ops, NULL, 0, 0); if (IS_ERR(dts->tzone)) { ret = PTR_ERR(dts->tzone); @@ -304,6 +292,14 @@ static void dts_trips_reset(struct intel_soc_dts_sensors *sensors, int dts_index update_trip_temp(sensors, 1, 0); } +static void set_trip(struct thermal_trip *trip, enum thermal_trip_type type, + u8 flags, int temp) +{ + trip->type = type; + trip->flags = flags; + trip->temperature = temp; +} + struct intel_soc_dts_sensors * intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, bool critical_trip, int crit_offset) @@ -331,36 +327,33 @@ intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, sensors->tj_max = tj_max * 1000; for (i = 0; i < SOC_MAX_DTS_SENSORS; ++i) { - enum thermal_trip_type trip_type; int temp; sensors->soc_dts[i].sensors = sensors; + set_trip(&trips[i][0], THERMAL_TRIP_PASSIVE, + THERMAL_TRIP_FLAG_RW_TEMP, 0); + ret = update_trip_temp(sensors, 0, 0); if (ret) goto err_reset_trips; - trips[i][0].type = THERMAL_TRIP_PASSIVE; - trips[i][0].temperature = 0; - if (critical_trip) { - trip_type = THERMAL_TRIP_CRITICAL; temp = sensors->tj_max - crit_offset; + set_trip(&trips[i][1], THERMAL_TRIP_CRITICAL, 0, temp); } else { - trip_type = THERMAL_TRIP_PASSIVE; + set_trip(&trips[i][1], THERMAL_TRIP_PASSIVE, + THERMAL_TRIP_FLAG_RW_TEMP, 0); temp = 0; } + ret = update_trip_temp(sensors, 1, temp); if (ret) goto err_reset_trips; - - trips[i][1].type = trip_type; - trips[i][1].temperature = temp; } for (i = 0; i < SOC_MAX_DTS_SENSORS; ++i) { - ret = add_dts_thermal_zone(i, &sensors->soc_dts[i], trips[i], - critical_trip); + ret = add_dts_thermal_zone(i, &sensors->soc_dts[i], trips[i]); if (ret) goto err_remove_zone; } diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.h b/drivers/thermal/intel/intel_soc_dts_iosf.h index da59d719eb9d..44eee844ab3c 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.h +++ b/drivers/thermal/intel/intel_soc_dts_iosf.h @@ -28,7 +28,6 @@ struct intel_soc_dts_sensors; struct intel_soc_dts_sensor_entry { int id; u32 store_status; - u32 trip_mask; struct thermal_zone_device *tzone; struct intel_soc_dts_sensors *sensors; }; diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 20cb21cb97c1..5a48b73a0951 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -295,6 +295,7 @@ static int pkg_temp_thermal_trips_init(int cpu, int tj_max, tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID; trips[i].type = THERMAL_TRIP_PASSIVE; + trips[i].flags |= THERMAL_TRIP_FLAG_RW_TEMP; pr_debug("%s: cpu=%d, trip=%d, temp=%d\n", __func__, cpu, i, trips[i].temperature); @@ -337,8 +338,7 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn); zonedev->cpu = cpu; zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp", - trips, thres_count, - (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01, + trips, thres_count, 0, zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0); if (IS_ERR(zonedev->tzone)) { err = PTR_ERR(zonedev->tzone); From c7ebf8e5d0681198347d515ff0702669b565d423 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 12 Feb 2024 19:35:56 +0100 Subject: [PATCH 0821/1038] mlxsw: core_thermal: Set THERMAL_TRIP_FLAG_RW_TEMP directly It is now possible to flag trip points with THERMAL_TRIP_FLAG_RW_TEMP to allow their temperature to be set from user space via sysfs instead of using a nonzero writable trips mask during thermal zone registration, so make the mlxsw code do that. No intentional functional impact. Note that this change is requisite for dropping the mask argument from thermal_zone_device_register_with_trips() going forward. Signed-off-by: Rafael J. Wysocki Reviewed-by: Ido Schimmel Reviewed-by: Daniel Lezcano --- .../net/ethernet/mellanox/mlxsw/core_thermal.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index f1b48d6615f6..29ec0c122139 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -44,16 +44,19 @@ static const struct thermal_trip default_thermal_trips[] = { .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_ASIC_TEMP_NORM, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* In range - 40-100% PWM */ .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_ASIC_TEMP_HIGH, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* Warning */ .type = THERMAL_TRIP_HOT, .temperature = MLXSW_THERMAL_ASIC_TEMP_HOT, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, }; @@ -62,16 +65,19 @@ static const struct thermal_trip default_thermal_module_trips[] = { .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_MODULE_TEMP_NORM, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* In range - 40-100% PWM */ .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_MODULE_TEMP_HIGH, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* Warning */ .type = THERMAL_TRIP_HOT, .temperature = MLXSW_THERMAL_MODULE_TEMP_HOT, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, }; @@ -92,9 +98,6 @@ static const struct mlxsw_cooling_states default_cooling_states[] = { #define MLXSW_THERMAL_NUM_TRIPS ARRAY_SIZE(default_thermal_trips) -/* Make sure all trips are writable */ -#define MLXSW_THERMAL_TRIP_MASK (BIT(MLXSW_THERMAL_NUM_TRIPS) - 1) - struct mlxsw_thermal; struct mlxsw_thermal_module { @@ -420,7 +423,7 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) module_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, module_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, + 0, module_tz, &mlxsw_thermal_module_ops, &mlxsw_thermal_params, @@ -548,7 +551,7 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) gearbox_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, gearbox_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, + 0, gearbox_tz, &mlxsw_thermal_gearbox_ops, &mlxsw_thermal_params, 0, @@ -773,7 +776,7 @@ int mlxsw_thermal_init(struct mlxsw_core *core, thermal->tzdev = thermal_zone_device_register_with_trips("mlxsw", thermal->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, + 0, thermal, &mlxsw_thermal_ops, &mlxsw_thermal_params, 0, From 96c5330bf75839a31d6fdf953fade6e9fbf30bde Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 12 Feb 2024 19:38:07 +0100 Subject: [PATCH 0822/1038] wifi: iwlwifi: mvm: Set THERMAL_TRIP_FLAG_RW_TEMP directly It is now possible to flag trip points with THERMAL_TRIP_FLAG_RW_TEMP to allow their temperature to be set from user space via sysfs instead of using a nonzero writable trips mask during thermal zone registration, so make the iwlwifi code do that. No intentional functional impact. Note that this change is requisite for dropping the mask argument from thermal_zone_device_register_with_trips() going forward. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/net/wireless/intel/iwlwifi/mvm/tt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index d2a00cbcbc42..2353177f2dfa 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -667,9 +667,6 @@ static struct thermal_zone_device_ops tzone_ops = { .set_trip_temp = iwl_mvm_tzone_set_trip_temp, }; -/* make all trips writable */ -#define IWL_WRITABLE_TRIPS_MSK (BIT(IWL_MAX_DTS_TRIPS) - 1) - static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) { int i, ret; @@ -692,11 +689,12 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) for (i = 0 ; i < IWL_MAX_DTS_TRIPS; i++) { mvm->tz_device.trips[i].temperature = THERMAL_TEMP_INVALID; mvm->tz_device.trips[i].type = THERMAL_TRIP_PASSIVE; + mvm->tz_device.trips[i].flags = THERMAL_TRIP_FLAG_RW_TEMP; } mvm->tz_device.tzone = thermal_zone_device_register_with_trips(name, mvm->tz_device.trips, IWL_MAX_DTS_TRIPS, - IWL_WRITABLE_TRIPS_MSK, + 0, mvm, &tzone_ops, NULL, 0, 0); if (IS_ERR(mvm->tz_device.tzone)) { From 68e9c60353d263a5920b6378bcb9ba183241ce92 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 12 Feb 2024 19:39:20 +0100 Subject: [PATCH 0823/1038] thermal: imx: Set THERMAL_TRIP_FLAG_RW_TEMP directly It is now possible to flag trip points with THERMAL_TRIP_FLAG_RW_TEMP to allow their temperature to be set from user space via sysfs instead of using a nonzero writable trips mask during thermal zone registration, so make the imx thermal code do that. No intentional functional impact. Note that this change is requisite for dropping the mask argument from thermal_zone_device_register_with_trips() going forward. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/thermal/imx_thermal.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 499eff858f3f..0dbede6bec54 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -115,7 +115,8 @@ struct thermal_soc_data { }; static struct thermal_trip trips[] = { - [IMX_TRIP_PASSIVE] = { .type = THERMAL_TRIP_PASSIVE }, + [IMX_TRIP_PASSIVE] = { .type = THERMAL_TRIP_PASSIVE, + .flags = THERMAL_TRIP_FLAG_RW_TEMP }, [IMX_TRIP_CRITICAL] = { .type = THERMAL_TRIP_CRITICAL }, }; @@ -700,7 +701,7 @@ static int imx_thermal_probe(struct platform_device *pdev) data->tz = thermal_zone_device_register_with_trips("imx_thermal_zone", trips, ARRAY_SIZE(trips), - BIT(IMX_TRIP_PASSIVE), data, + 0, data, &imx_tz_ops, NULL, IMX_PASSIVE_DELAY, IMX_POLLING_DELAY); From 83c2d444ed9da7dd3536abb7596de309b8a9991f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 18:38:21 +0100 Subject: [PATCH 0824/1038] thermal: of: Set THERMAL_TRIP_FLAG_RW_TEMP directly It is now possible to flag trip points with THERMAL_TRIP_FLAG_RW_TEMP to allow their temperature to be set from user space via sysfs instead of using a nonzero writable trips mask during thermal zone registration, so make the OF thermal code do that. No intentional functional impact. Note that this change is requisite for dropping the mask argument from thermal_zone_device_register_with_trips() going forward. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/thermal/thermal_of.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 405743e90531..c3bd5fb15c7e 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -117,6 +117,8 @@ static int thermal_of_populate_trip(struct device_node *np, return ret; } + trip->flags = THERMAL_TRIP_FLAG_RW_TEMP; + return 0; } @@ -472,7 +474,7 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * struct device_node *np; const char *action; int delay, pdelay; - int ntrips, mask; + int ntrips; int ret; np = of_thermal_zone_find(sensor, id); @@ -499,15 +501,13 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * of_ops.bind = thermal_of_bind; of_ops.unbind = thermal_of_unbind; - mask = GENMASK_ULL((ntrips) - 1, 0); - ret = of_property_read_string(np, "critical-action", &action); if (!ret) if (!of_ops.critical && !strcasecmp(action, "reboot")) of_ops.critical = thermal_zone_device_critical_reboot; tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, - mask, data, &of_ops, &tzp, + 0, data, &of_ops, &tzp, pdelay, delay); if (IS_ERR(tz)) { ret = PTR_ERR(tz); From 4a62d588a84e13c68017bd16bc9c2531a2cde08f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 22 Feb 2024 19:09:16 +0100 Subject: [PATCH 0825/1038] thermal: core: Eliminate writable trip points masks All of the thermal_zone_device_register_with_trips() callers pass zero writable trip points masks to it, so drop the mask argument from that function and update all of its callers accordingly. This also removes the artificial trip points per zone limit of 32, related to using writable trip points masks. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano --- drivers/acpi/thermal.c | 2 +- .../ethernet/chelsio/cxgb4/cxgb4_thermal.c | 2 +- .../ethernet/mellanox/mlxsw/core_thermal.c | 3 -- drivers/net/wireless/intel/iwlwifi/mvm/tt.c | 1 - drivers/platform/x86/acerhdf.c | 2 +- drivers/thermal/da9062-thermal.c | 2 +- drivers/thermal/imx_thermal.c | 2 +- .../int340x_thermal/int340x_thermal_zone.c | 2 +- .../processor_thermal_device_pci.c | 2 +- drivers/thermal/intel/intel_pch_thermal.c | 2 +- .../thermal/intel/intel_quark_dts_thermal.c | 2 +- drivers/thermal/intel/intel_soc_dts_iosf.c | 2 +- drivers/thermal/intel/x86_pkg_temp_thermal.c | 2 +- drivers/thermal/rcar_thermal.c | 2 +- drivers/thermal/st/st_thermal.c | 2 +- drivers/thermal/thermal_core.c | 28 ++----------------- drivers/thermal/thermal_of.c | 2 +- include/linux/thermal.h | 6 ++-- 18 files changed, 19 insertions(+), 47 deletions(-) diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 8042e96016e6..302dce0b2b50 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -665,7 +665,7 @@ static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz, tz->thermal_zone = thermal_zone_device_register_with_trips("acpitz", trip_table, trip_count, - 0, tz, + tz, &acpi_thermal_zone_ops, NULL, passive_delay, diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c index dea9d2907666..b08356060fb4 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c @@ -60,7 +60,7 @@ int cxgb4_thermal_init(struct adapter *adap) snprintf(ch_tz_name, sizeof(ch_tz_name), "cxgb4_%s", adap->name); ch_thermal->tzdev = thermal_zone_device_register_with_trips(ch_tz_name, &trip, num_trip, - 0, adap, + adap, &cxgb4_thermal_ops, NULL, 0, 0); if (IS_ERR(ch_thermal->tzdev)) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index 29ec0c122139..5c511e1a8efa 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -423,7 +423,6 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) module_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, module_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - 0, module_tz, &mlxsw_thermal_module_ops, &mlxsw_thermal_params, @@ -551,7 +550,6 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) gearbox_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, gearbox_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - 0, gearbox_tz, &mlxsw_thermal_gearbox_ops, &mlxsw_thermal_params, 0, @@ -776,7 +774,6 @@ int mlxsw_thermal_init(struct mlxsw_core *core, thermal->tzdev = thermal_zone_device_register_with_trips("mlxsw", thermal->trips, MLXSW_THERMAL_NUM_TRIPS, - 0, thermal, &mlxsw_thermal_ops, &mlxsw_thermal_params, 0, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index 2353177f2dfa..61a4638d1be2 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -694,7 +694,6 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) mvm->tz_device.tzone = thermal_zone_device_register_with_trips(name, mvm->tz_device.trips, IWL_MAX_DTS_TRIPS, - 0, mvm, &tzone_ops, NULL, 0, 0); if (IS_ERR(mvm->tz_device.tzone)) { diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c index 74bcb3d13104..018c48429616 100644 --- a/drivers/platform/x86/acerhdf.c +++ b/drivers/platform/x86/acerhdf.c @@ -678,7 +678,7 @@ static int __init acerhdf_register_thermal(void) return -EINVAL; thz_dev = thermal_zone_device_register_with_trips("acerhdf", trips, ARRAY_SIZE(trips), - 0, NULL, &acerhdf_dev_ops, + NULL, &acerhdf_dev_ops, &acerhdf_zone_params, 0, (kernelmode) ? interval*1000 : 0); if (IS_ERR(thz_dev)) diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c index 160d64913057..a27aff88cd96 100644 --- a/drivers/thermal/da9062-thermal.c +++ b/drivers/thermal/da9062-thermal.c @@ -197,7 +197,7 @@ static int da9062_thermal_probe(struct platform_device *pdev) mutex_init(&thermal->lock); thermal->zone = thermal_zone_device_register_with_trips(thermal->config->name, - trips, ARRAY_SIZE(trips), 0, thermal, + trips, ARRAY_SIZE(trips), thermal, &da9062_thermal_ops, NULL, pp_tmp, 0); if (IS_ERR(thermal->zone)) { diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 0dbede6bec54..83eaae5ca3b8 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -701,7 +701,7 @@ static int imx_thermal_probe(struct platform_device *pdev) data->tz = thermal_zone_device_register_with_trips("imx_thermal_zone", trips, ARRAY_SIZE(trips), - 0, data, + data, &imx_tz_ops, NULL, IMX_PASSIVE_DELAY, IMX_POLLING_DELAY); diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index 0e8148afb490..e8182f0929d3 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -171,7 +171,7 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->zone = thermal_zone_device_register_with_trips( acpi_device_bid(adev), zone_trips, trip_cnt, - 0, int34x_zone, + int34x_zone, &zone_ops, &int340x_thermal_params, 0, 0); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index 7572f3a73c54..7dcf6006816e 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -290,7 +290,7 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ psv_trip.temperature = get_trip_temp(pci_info); pci_info->tzone = thermal_zone_device_register_with_trips("TCPU_PCI", &psv_trip, - 1, 0, pci_info, + 1, pci_info, &tzone_ops, &tzone_params, 0, 0); if (IS_ERR(pci_info->tzone)) { diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c index 8bfb221e57e7..f5be2c389351 100644 --- a/drivers/thermal/intel/intel_pch_thermal.c +++ b/drivers/thermal/intel/intel_pch_thermal.c @@ -237,7 +237,7 @@ read_trips: ptd->tzd = thermal_zone_device_register_with_trips(board_names[board_id], ptd_trips, nr_trips, - 0, ptd, &tzd_ops, + ptd, &tzd_ops, NULL, 0, 0); if (IS_ERR(ptd->tzd)) { dev_err(&pdev->dev, "Failed to register thermal zone %s\n", diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c index 55a031fc3a9a..ec6ad26027bc 100644 --- a/drivers/thermal/intel/intel_quark_dts_thermal.c +++ b/drivers/thermal/intel/intel_quark_dts_thermal.c @@ -363,7 +363,7 @@ static struct soc_sensor_entry *alloc_soc_dts(void) aux_entry->tzone = thermal_zone_device_register_with_trips("quark_dts", trips, QRK_MAX_DTS_TRIPS, - 0, aux_entry, + aux_entry, &tzone_ops, NULL, 0, polling_delay); if (IS_ERR(aux_entry->tzone)) { diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c index 5a84c9166274..2ab943b66f7a 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.c +++ b/drivers/thermal/intel/intel_soc_dts_iosf.c @@ -230,7 +230,7 @@ static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, snprintf(name, sizeof(name), "soc_dts%d", id); dts->tzone = thermal_zone_device_register_with_trips(name, trips, SOC_MAX_DTS_TRIPS, - 0, dts, &tzone_ops, + dts, &tzone_ops, NULL, 0, 0); if (IS_ERR(dts->tzone)) { ret = PTR_ERR(dts->tzone); diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 5a48b73a0951..22a1e3268a98 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -338,7 +338,7 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn); zonedev->cpu = cpu; zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp", - trips, thres_count, 0, + trips, thres_count, zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0); if (IS_ERR(zonedev->tzone)) { err = PTR_ERR(zonedev->tzone); diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index feb848d595fa..925183753fcb 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -489,7 +489,7 @@ static int rcar_thermal_probe(struct platform_device *pdev) &rcar_thermal_zone_ops); } else { priv->zone = thermal_zone_device_register_with_trips( - "rcar_thermal", trips, ARRAY_SIZE(trips), 0, priv, + "rcar_thermal", trips, ARRAY_SIZE(trips), priv, &rcar_thermal_zone_ops, NULL, 0, idle); diff --git a/drivers/thermal/st/st_thermal.c b/drivers/thermal/st/st_thermal.c index 0d6249b36609..2a105409864e 100644 --- a/drivers/thermal/st/st_thermal.c +++ b/drivers/thermal/st/st_thermal.c @@ -203,7 +203,7 @@ int st_thermal_register(struct platform_device *pdev, trip.type = THERMAL_TRIP_CRITICAL; sensor->thermal_dev = - thermal_zone_device_register_with_trips(dev_name(dev), &trip, 1, 0, sensor, + thermal_zone_device_register_with_trips(dev_name(dev), &trip, 1, sensor, &st_tz_ops, NULL, 0, polling_delay); if (IS_ERR(sensor->thermal_dev)) { dev_err(dev, "failed to register thermal zone device\n"); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 2ab495220c6e..f7a7d43809e7 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1248,7 +1248,6 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); * @type: the thermal zone device type * @trips: a pointer to an array of thermal trips * @num_trips: the number of trip points the thermal zone support - * @mask: a bit string indicating the writeablility of trip points * @devdata: private device data * @ops: standard thermal zone device callbacks * @tzp: thermal zone platform parameters @@ -1271,14 +1270,12 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); struct thermal_zone_device * thermal_zone_device_register_with_trips(const char *type, const struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, + int num_trips, void *devdata, const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { struct thermal_zone_device *tz; - struct thermal_trip *trip; int id; int result; struct thermal_governor *governor; @@ -1294,20 +1291,7 @@ thermal_zone_device_register_with_trips(const char *type, return ERR_PTR(-EINVAL); } - /* - * Max trip count can't exceed 31 as the "mask >> num_trips" condition. - * For example, shifting by 32 will result in compiler warning: - * warning: right shift count >= width of type [-Wshift-count- overflow] - * - * Also "mask >> num_trips" will always be true with 32 bit shift. - * E.g. mask = 0x80000000 for trip id 31 to be RW. Then - * mask >> 32 = 0x80000000 - * This will result in failure for the below condition. - * - * Check will be true when the bit 31 of the mask is set. - * 32 bit shift will cause overflow of 4 byte integer. - */ - if (num_trips > (BITS_PER_TYPE(int) - 1) || num_trips < 0 || mask >> num_trips) { + if (num_trips < 0) { pr_err("Incorrect number of thermal trips\n"); return ERR_PTR(-EINVAL); } @@ -1357,12 +1341,6 @@ thermal_zone_device_register_with_trips(const char *type, tz->devdata = devdata; tz->num_trips = num_trips; memcpy(tz->trips, trips, num_trips * sizeof(*trips)); - for_each_trip(tz, trip) { - if (mask & 1) - trip->flags |= THERMAL_TRIP_FLAG_RW_TEMP; - - mask >>= 1; - } thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); @@ -1447,7 +1425,7 @@ struct thermal_zone_device *thermal_tripless_zone_device_register( const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp) { - return thermal_zone_device_register_with_trips(type, NULL, 0, 0, devdata, + return thermal_zone_device_register_with_trips(type, NULL, 0, devdata, ops, tzp, 0, 0); } EXPORT_SYMBOL_GPL(thermal_tripless_zone_device_register); diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index c3bd5fb15c7e..f1cbf9aa62cf 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -507,7 +507,7 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * of_ops.critical = thermal_zone_device_critical_reboot; tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, - 0, data, &of_ops, &tzp, + data, &of_ops, &tzp, pdelay, delay); if (IS_ERR(tz)) { ret = PTR_ERR(tz); diff --git a/include/linux/thermal.h b/include/linux/thermal.h index eb4145a64c7e..c33f50177f51 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -323,8 +323,7 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp); struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, const struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, + int num_trips, void *devdata, const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay); @@ -383,8 +382,7 @@ void thermal_zone_device_critical(struct thermal_zone_device *tz); static inline struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, const struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, + int num_trips, void *devdata, const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) From 4c5e3efae66d04af3fc745621acaa47713d853a2 Mon Sep 17 00:00:00 2001 From: Martin Blumenstingl Date: Sat, 17 Feb 2024 11:01:59 +0100 Subject: [PATCH 0826/1038] mmc: meson-mx-sdhc: Use devm_clk_hw_get_clk() for clock retrieval Now that devm_clk_hw_get_clk() has been available for a while we can resolve an older TODO where this API did not exist yet. No functional changes intended. Signed-off-by: Martin Blumenstingl Link: https://lore.kernel.org/r/20240217100200.1494980-2-martin.blumenstingl@googlemail.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/meson-mx-sdhc-clkc.c | 43 ++++++++++++++------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/drivers/mmc/host/meson-mx-sdhc-clkc.c b/drivers/mmc/host/meson-mx-sdhc-clkc.c index 19200b7079a6..cbd17a596cd2 100644 --- a/drivers/mmc/host/meson-mx-sdhc-clkc.c +++ b/drivers/mmc/host/meson-mx-sdhc-clkc.c @@ -71,12 +71,23 @@ static int meson_mx_sdhc_clk_hw_register(struct device *dev, static int meson_mx_sdhc_gate_clk_hw_register(struct device *dev, const char *name_suffix, struct clk_hw *parent, - struct clk_hw *hw) + struct clk_hw *hw, + struct clk_bulk_data *clk_bulk_data, + u8 bulk_index) { struct clk_parent_data parent_data = { .hw = parent }; + int ret; - return meson_mx_sdhc_clk_hw_register(dev, name_suffix, &parent_data, 1, - &clk_gate_ops, hw); + ret = meson_mx_sdhc_clk_hw_register(dev, name_suffix, &parent_data, 1, + &clk_gate_ops, hw); + if (ret) + return ret; + + clk_bulk_data[bulk_index].clk = devm_clk_hw_get_clk(dev, hw, name_suffix); + if (IS_ERR(clk_bulk_data[bulk_index].clk)) + return PTR_ERR(clk_bulk_data[bulk_index].clk); + + return 0; } int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, @@ -115,7 +126,8 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->mod_clk_en.bit_idx = 15; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "mod_clk_on", &clkc_data->div.hw, - &clkc_data->mod_clk_en.hw); + &clkc_data->mod_clk_en.hw, + clk_bulk_data, 0); if (ret) return ret; @@ -123,7 +135,8 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->tx_clk_en.bit_idx = 14; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "tx_clk_on", &clkc_data->div.hw, - &clkc_data->tx_clk_en.hw); + &clkc_data->tx_clk_en.hw, + clk_bulk_data, 1); if (ret) return ret; @@ -131,7 +144,8 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->rx_clk_en.bit_idx = 13; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "rx_clk_on", &clkc_data->div.hw, - &clkc_data->rx_clk_en.hw); + &clkc_data->rx_clk_en.hw, + clk_bulk_data, 2); if (ret) return ret; @@ -139,18 +153,7 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->sd_clk_en.bit_idx = 12; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "sd_clk_on", &clkc_data->div.hw, - &clkc_data->sd_clk_en.hw); - if (ret) - return ret; - - /* - * TODO: Replace clk_hw.clk with devm_clk_hw_get_clk() once that is - * available. - */ - clk_bulk_data[0].clk = clkc_data->mod_clk_en.hw.clk; - clk_bulk_data[1].clk = clkc_data->sd_clk_en.hw.clk; - clk_bulk_data[2].clk = clkc_data->tx_clk_en.hw.clk; - clk_bulk_data[3].clk = clkc_data->rx_clk_en.hw.clk; - - return 0; + &clkc_data->sd_clk_en.hw, + clk_bulk_data, 3); + return ret; } From c0200efad3306435009fa2b00c02d4af45317a6f Mon Sep 17 00:00:00 2001 From: Martin Blumenstingl Date: Sat, 17 Feb 2024 11:02:00 +0100 Subject: [PATCH 0827/1038] mmc: meson-mx-sdhc: Remove .card_hw_reset callback Commit 32f18e596141 ("mmc: improve API to make clear hw_reset callback is for cards") made it clear that the hw_reset callback is intended for resetting the card. Remove the .card_hw_reset callback from the meson-mx-sdhc-mmc driver because it's purpose is to reset the SDHC controller (FIFOs, PHY, DMA interface, ...). While here also rename and change the argument of meson_mx_sdhc_hw_reset so it cannot be called by accident as a replacement for card_hw_reset in the future. Signed-off-by: Martin Blumenstingl Link: https://lore.kernel.org/r/20240217100200.1494980-3-martin.blumenstingl@googlemail.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/meson-mx-sdhc-mmc.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/mmc/host/meson-mx-sdhc-mmc.c b/drivers/mmc/host/meson-mx-sdhc-mmc.c index 1ed9731e77ef..31f750301dc1 100644 --- a/drivers/mmc/host/meson-mx-sdhc-mmc.c +++ b/drivers/mmc/host/meson-mx-sdhc-mmc.c @@ -65,10 +65,8 @@ static const struct regmap_config meson_mx_sdhc_regmap_config = { .max_register = MESON_SDHC_CLK2, }; -static void meson_mx_sdhc_hw_reset(struct mmc_host *mmc) +static void meson_mx_sdhc_reset(struct meson_mx_sdhc_host *host) { - struct meson_mx_sdhc_host *host = mmc_priv(mmc); - regmap_write(host->regmap, MESON_SDHC_SRST, MESON_SDHC_SRST_MAIN_CTRL | MESON_SDHC_SRST_RXFIFO | MESON_SDHC_SRST_TXFIFO | MESON_SDHC_SRST_DPHY_RX | MESON_SDHC_SRST_DPHY_TX | @@ -116,7 +114,7 @@ static void meson_mx_sdhc_wait_cmd_ready(struct mmc_host *mmc) dev_warn(mmc_dev(mmc), "Failed to poll for CMD_BUSY while processing CMD%d\n", host->cmd->opcode); - meson_mx_sdhc_hw_reset(mmc); + meson_mx_sdhc_reset(host); } ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_ESTA, esta, @@ -127,7 +125,7 @@ static void meson_mx_sdhc_wait_cmd_ready(struct mmc_host *mmc) dev_warn(mmc_dev(mmc), "Failed to poll for ESTA[13:11] while processing CMD%d\n", host->cmd->opcode); - meson_mx_sdhc_hw_reset(mmc); + meson_mx_sdhc_reset(host); } } @@ -495,7 +493,6 @@ static int meson_mx_sdhc_execute_tuning(struct mmc_host *mmc, u32 opcode) } static const struct mmc_host_ops meson_mx_sdhc_ops = { - .card_hw_reset = meson_mx_sdhc_hw_reset, .request = meson_mx_sdhc_request, .set_ios = meson_mx_sdhc_set_ios, .card_busy = meson_mx_sdhc_card_busy, @@ -618,7 +615,7 @@ static irqreturn_t meson_mx_sdhc_irq_thread(int irq, void *irq_data) } if (cmd->error == -EIO || cmd->error == -ETIMEDOUT) - meson_mx_sdhc_hw_reset(host->mmc); + meson_mx_sdhc_reset(host); else if (cmd->data) /* * Clear the FIFOs after completing data transfers to prevent @@ -728,7 +725,7 @@ static void meson_mx_sdhc_init_hw(struct mmc_host *mmc) { struct meson_mx_sdhc_host *host = mmc_priv(mmc); - meson_mx_sdhc_hw_reset(mmc); + meson_mx_sdhc_reset(host); regmap_write(host->regmap, MESON_SDHC_CTRL, FIELD_PREP(MESON_SDHC_CTRL_RX_PERIOD, 0xf) | From 849e81817b9b0658a65b668cd65849fa1fb054e0 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sat, 17 Feb 2024 22:42:01 +0800 Subject: [PATCH 0828/1038] dt-bindings: mmc: sdhci-of-dwcmhsc: Add Sophgo CV1800B and SG2002 support Add compatible value for the dwcmshc controller in Sophgo's CV1800B and SG2002. Signed-off-by: Jisheng Zhang Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240217144202.3808-2-jszhang@kernel.org Signed-off-by: Ulf Hansson --- Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml index 42804d955293..4d3031d9965f 100644 --- a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml +++ b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml @@ -19,6 +19,8 @@ properties: - rockchip,rk3568-dwcmshc - rockchip,rk3588-dwcmshc - snps,dwcmshc-sdhci + - sophgo,cv1800b-dwcmshc + - sophgo,sg2002-dwcmshc - thead,th1520-dwcmshc reg: From 017199c2849c3d6d417791ec1cf5521005d663a1 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sat, 17 Feb 2024 22:42:02 +0800 Subject: [PATCH 0829/1038] mmc: sdhci-of-dwcmshc: Add support for Sophgo CV1800B and SG2002 Add support for the mmc controller in the Sophgo CV1800B and SG2002 with corresponding new compatible strings. Implement custom sdhci_ops. Signed-off-by: Jisheng Zhang Link: https://lore.kernel.org/r/20240217144202.3808-3-jszhang@kernel.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-of-dwcmshc.c | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index a1f57af6acfb..ab4b964d4058 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -52,6 +52,20 @@ #define AT_CTRL_SWIN_TH_VAL_MASK GENMASK(31, 24) /* bits [31:24] */ #define AT_CTRL_SWIN_TH_VAL 0x9 /* sampling window threshold */ +/* Sophgo CV18XX specific Registers */ +#define CV18XX_SDHCI_MSHC_CTRL 0x00 +#define CV18XX_EMMC_FUNC_EN BIT(0) +#define CV18XX_LATANCY_1T BIT(1) +#define CV18XX_SDHCI_PHY_TX_RX_DLY 0x40 +#define CV18XX_PHY_TX_DLY_MSK GENMASK(6, 0) +#define CV18XX_PHY_TX_SRC_MSK GENMASK(9, 8) +#define CV18XX_PHY_TX_SRC_INVERT_CLK_TX 0x1 +#define CV18XX_PHY_RX_DLY_MSK GENMASK(22, 16) +#define CV18XX_PHY_RX_SRC_MSK GENMASK(25, 24) +#define CV18XX_PHY_RX_SRC_INVERT_RX_CLK 0x1 +#define CV18XX_SDHCI_PHY_CONFIG 0x4c +#define CV18XX_PHY_TX_BPS BIT(0) + /* Rockchip specific Registers */ #define DWCMSHC_EMMC_DLL_CTRL 0x800 #define DWCMSHC_EMMC_DLL_RXCLK 0x804 @@ -642,6 +656,35 @@ static void th1520_sdhci_reset(struct sdhci_host *host, u8 mask) } } +static void cv18xx_sdhci_reset(struct sdhci_host *host, u8 mask) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host); + u32 val, emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO; + + sdhci_reset(host, mask); + + if ((host->mmc->caps2 & emmc_caps) == emmc_caps) { + val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + val |= CV18XX_EMMC_FUNC_EN; + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + } + + val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + val |= CV18XX_LATANCY_1T; + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + + val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_PHY_CONFIG); + val |= CV18XX_PHY_TX_BPS; + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_PHY_CONFIG); + + val = (FIELD_PREP(CV18XX_PHY_TX_DLY_MSK, 0) | + FIELD_PREP(CV18XX_PHY_TX_SRC_MSK, CV18XX_PHY_TX_SRC_INVERT_CLK_TX) | + FIELD_PREP(CV18XX_PHY_RX_DLY_MSK, 0) | + FIELD_PREP(CV18XX_PHY_RX_SRC_MSK, CV18XX_PHY_RX_SRC_INVERT_RX_CLK)); + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_PHY_TX_RX_DLY); +} + static const struct sdhci_ops sdhci_dwcmshc_ops = { .set_clock = sdhci_set_clock, .set_bus_width = sdhci_set_bus_width, @@ -671,6 +714,15 @@ static const struct sdhci_ops sdhci_dwcmshc_th1520_ops = { .platform_execute_tuning = &th1520_execute_tuning, }; +static const struct sdhci_ops sdhci_dwcmshc_cv18xx_ops = { + .set_clock = sdhci_set_clock, + .set_bus_width = sdhci_set_bus_width, + .set_uhs_signaling = dwcmshc_set_uhs_signaling, + .get_max_clock = dwcmshc_get_max_clock, + .reset = cv18xx_sdhci_reset, + .adma_write_desc = dwcmshc_adma_write_desc, +}; + static const struct sdhci_pltfm_data sdhci_dwcmshc_pdata = { .ops = &sdhci_dwcmshc_ops, .quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN, @@ -700,6 +752,12 @@ static const struct sdhci_pltfm_data sdhci_dwcmshc_th1520_pdata = { .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, }; +static const struct sdhci_pltfm_data sdhci_dwcmshc_cv18xx_pdata = { + .ops = &sdhci_dwcmshc_cv18xx_ops, + .quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN, + .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, +}; + static int dwcmshc_rk35xx_init(struct sdhci_host *host, struct dwcmshc_priv *dwc_priv) { int err; @@ -768,6 +826,14 @@ static const struct of_device_id sdhci_dwcmshc_dt_ids[] = { .compatible = "snps,dwcmshc-sdhci", .data = &sdhci_dwcmshc_pdata, }, + { + .compatible = "sophgo,cv1800b-dwcmshc", + .data = &sdhci_dwcmshc_cv18xx_pdata, + }, + { + .compatible = "sophgo,sg2002-dwcmshc", + .data = &sdhci_dwcmshc_cv18xx_pdata, + }, { .compatible = "thead,th1520-dwcmshc", .data = &sdhci_dwcmshc_th1520_pdata, From 68f5630ae15407d8d3daaadbbfbafbd319e3ecd4 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Mon, 19 Feb 2024 09:43:57 -0300 Subject: [PATCH 0830/1038] mmc: core: constify the struct device_type usage Since commit aed65af1cc2f ("drivers: make device_type const"), the driver core can properly handle constant struct device_type. Move the sdio_type, sd_type and mmc_type variables to be constant structures as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Link: https://lore.kernel.org/r/20240219-device_cleanup-mmc-v1-1-1910e283cf5a@marliere.net Signed-off-by: Ulf Hansson --- drivers/mmc/core/bus.c | 2 +- drivers/mmc/core/bus.h | 2 +- drivers/mmc/core/mmc.c | 2 +- drivers/mmc/core/sd.c | 2 +- drivers/mmc/core/sd.h | 2 +- drivers/mmc/core/sdio.c | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c index 48daeafdff7a..0ddaee0eae54 100644 --- a/drivers/mmc/core/bus.c +++ b/drivers/mmc/core/bus.c @@ -272,7 +272,7 @@ static void mmc_release_card(struct device *dev) /* * Allocate and initialise a new MMC card structure. */ -struct mmc_card *mmc_alloc_card(struct mmc_host *host, struct device_type *type) +struct mmc_card *mmc_alloc_card(struct mmc_host *host, const struct device_type *type) { struct mmc_card *card; diff --git a/drivers/mmc/core/bus.h b/drivers/mmc/core/bus.h index 3996b191b68d..cfd0d02d3420 100644 --- a/drivers/mmc/core/bus.h +++ b/drivers/mmc/core/bus.h @@ -23,7 +23,7 @@ static ssize_t mmc_##name##_show (struct device *dev, struct device_attribute *a static DEVICE_ATTR(name, S_IRUGO, mmc_##name##_show, NULL) struct mmc_card *mmc_alloc_card(struct mmc_host *host, - struct device_type *type); + const struct device_type *type); int mmc_add_card(struct mmc_card *card); void mmc_remove_card(struct mmc_card *card); diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c index 58ed7193a3ca..5b2f7c285461 100644 --- a/drivers/mmc/core/mmc.c +++ b/drivers/mmc/core/mmc.c @@ -883,7 +883,7 @@ static struct attribute *mmc_std_attrs[] = { }; ATTRIBUTE_GROUPS(mmc_std); -static struct device_type mmc_type = { +static const struct device_type mmc_type = { .groups = mmc_std_groups, }; diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index c3e554344c99..1c8148cdda50 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -805,7 +805,7 @@ static const struct attribute_group sd_std_group = { }; __ATTRIBUTE_GROUPS(sd_std); -struct device_type sd_type = { +const struct device_type sd_type = { .groups = sd_std_groups, }; diff --git a/drivers/mmc/core/sd.h b/drivers/mmc/core/sd.h index 1af5a038bae9..fe6dd46927a4 100644 --- a/drivers/mmc/core/sd.h +++ b/drivers/mmc/core/sd.h @@ -4,7 +4,7 @@ #include -extern struct device_type sd_type; +extern const struct device_type sd_type; struct mmc_host; struct mmc_card; diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index 5914516df2f7..4fb247fde5c0 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -66,7 +66,7 @@ static struct attribute *sdio_std_attrs[] = { }; ATTRIBUTE_GROUPS(sdio_std); -static struct device_type sdio_type = { +static const struct device_type sdio_type = { .groups = sdio_std_groups, }; From 0d776cfd5e5b559fdf2e38285c2aea4b7048acbd Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Thu, 22 Feb 2024 22:52:53 -0800 Subject: [PATCH 0831/1038] gpiolib: Pass consumer device through to core in devm_fwnode_gpiod_get_index() This devm API takes a consumer device as an argument to setup the devm action, but throws it away when calling further into gpiolib. This leads to odd debug messages like this: (NULL device *): using DT '/gpio-keys/switch-pen-insert' for '(null)' GPIO lookup Let's pass the consumer device down, by directly calling what fwnode_gpiod_get_index() calls but pass the device used for devm. This changes the message to look like this instead: gpio-keys gpio-keys: using DT '/gpio-keys/switch-pen-insert' for '(null)' GPIO lookup Note that callers of fwnode_gpiod_get_index() will still see the NULL device pointer debug message, but there's not much we can do about that because the API doesn't take a struct device. Cc: Dmitry Torokhov Fixes: 8eb1f71e7acc ("gpiolib: consolidate GPIO lookups") Signed-off-by: Stephen Boyd Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-devres.c | 2 +- drivers/gpio/gpiolib.c | 14 +++++++------- drivers/gpio/gpiolib.h | 8 ++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/drivers/gpio/gpiolib-devres.c b/drivers/gpio/gpiolib-devres.c index fe9ce6b19f15..4987e62dcb3d 100644 --- a/drivers/gpio/gpiolib-devres.c +++ b/drivers/gpio/gpiolib-devres.c @@ -158,7 +158,7 @@ struct gpio_desc *devm_fwnode_gpiod_get_index(struct device *dev, if (!dr) return ERR_PTR(-ENOMEM); - desc = fwnode_gpiod_get_index(fwnode, con_id, index, flags, label); + desc = gpiod_find_and_request(dev, fwnode, con_id, index, flags, label, false); if (IS_ERR(desc)) { devres_free(dr); return desc; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 63e793a410e3..3e2cd85673b6 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4171,13 +4171,13 @@ static struct gpio_desc *gpiod_find_by_fwnode(struct fwnode_handle *fwnode, return desc; } -static struct gpio_desc *gpiod_find_and_request(struct device *consumer, - struct fwnode_handle *fwnode, - const char *con_id, - unsigned int idx, - enum gpiod_flags flags, - const char *label, - bool platform_lookup_allowed) +struct gpio_desc *gpiod_find_and_request(struct device *consumer, + struct fwnode_handle *fwnode, + const char *con_id, + unsigned int idx, + enum gpiod_flags flags, + const char *label, + bool platform_lookup_allowed) { unsigned long lookupflags = GPIO_LOOKUP_FLAGS_DEFAULT; /* diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index ada36aa0f81a..f67d5991ab1c 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -223,6 +223,14 @@ static inline int gpiod_request_user(struct gpio_desc *desc, const char *label) return ret; } +struct gpio_desc *gpiod_find_and_request(struct device *consumer, + struct fwnode_handle *fwnode, + const char *con_id, + unsigned int idx, + enum gpiod_flags flags, + const char *label, + bool platform_lookup_allowed); + int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, unsigned long lflags, enum gpiod_flags dflags); int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce); From f2f212f36a8cd3cd6763a19611d0edd2a19df51a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 20 Nov 2023 18:30:54 +0100 Subject: [PATCH 0832/1038] ACPI: APEI: GHES: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Instead of returning an error code, emit a better error message than the core. Apart from the improved error message this patch has no effects for the driver. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/apei/ghes.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index ab2a82cb1b0b..512067cac170 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -1455,7 +1455,7 @@ err: return rc; } -static int ghes_remove(struct platform_device *ghes_dev) +static void ghes_remove(struct platform_device *ghes_dev) { int rc; struct ghes *ghes; @@ -1492,8 +1492,15 @@ static int ghes_remove(struct platform_device *ghes_dev) break; case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED: rc = apei_sdei_unregister_ghes(ghes); - if (rc) - return rc; + if (rc) { + /* + * Returning early results in a resource leak, but we're + * only here if stopping the hardware failed. + */ + dev_err(&ghes_dev->dev, "Failed to unregister ghes (%pe)\n", + ERR_PTR(rc)); + return; + } break; default: BUG(); @@ -1507,8 +1514,6 @@ static int ghes_remove(struct platform_device *ghes_dev) mutex_unlock(&ghes_devs_mutex); kfree(ghes); - - return 0; } static struct platform_driver ghes_platform_driver = { @@ -1516,7 +1521,7 @@ static struct platform_driver ghes_platform_driver = { .name = "GHES", }, .probe = ghes_probe, - .remove = ghes_remove, + .remove_new = ghes_remove, }; void __init acpi_ghes_init(void) From 9a7897a2b03183f15ea4c7645416daf3ae1f634e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:33 +0100 Subject: [PATCH 0833/1038] ACPI: TAD: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_tad.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index 33c3b16af556..1d670dbe4d1d 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -554,7 +554,7 @@ static int acpi_tad_disable_timer(struct device *dev, u32 timer_id) return acpi_tad_wake_set(dev, "_STV", timer_id, ACPI_TAD_WAKE_DISABLED); } -static int acpi_tad_remove(struct platform_device *pdev) +static void acpi_tad_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; acpi_handle handle = ACPI_HANDLE(dev); @@ -579,7 +579,6 @@ static int acpi_tad_remove(struct platform_device *pdev) pm_runtime_put_sync(dev); pm_runtime_disable(dev); acpi_remove_cmos_rtc_space_handler(handle); - return 0; } static int acpi_tad_probe(struct platform_device *pdev) @@ -684,7 +683,7 @@ static struct platform_driver acpi_tad_driver = { .acpi_match_table = acpi_tad_ids, }, .probe = acpi_tad_probe, - .remove = acpi_tad_remove, + .remove_new = acpi_tad_remove, }; MODULE_DEVICE_TABLE(acpi, acpi_tad_ids); From 10ff709a68ccd985cbbbdd04beac468724ca3c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:34 +0100 Subject: [PATCH 0834/1038] ACPI: AGDI: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Acked-by: Hanjun Guo Signed-off-by: Rafael J. Wysocki --- drivers/acpi/arm64/agdi.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/arm64/agdi.c b/drivers/acpi/arm64/agdi.c index 8b3c7d42b41b..f5f21dd0d277 100644 --- a/drivers/acpi/arm64/agdi.c +++ b/drivers/acpi/arm64/agdi.c @@ -58,7 +58,7 @@ static int agdi_probe(struct platform_device *pdev) return agdi_sdei_probe(pdev, adata); } -static int agdi_remove(struct platform_device *pdev) +static void agdi_remove(struct platform_device *pdev) { struct agdi_data *adata = dev_get_platdata(&pdev->dev); int err, i; @@ -67,7 +67,7 @@ static int agdi_remove(struct platform_device *pdev) if (err) { dev_err(&pdev->dev, "Failed to disable sdei-event #%d (%pe)\n", adata->sdei_event, ERR_PTR(err)); - return 0; + return; } for (i = 0; i < 3; i++) { @@ -81,8 +81,6 @@ static int agdi_remove(struct platform_device *pdev) if (err) dev_err(&pdev->dev, "Failed to unregister sdei-event #%d (%pe)\n", adata->sdei_event, ERR_PTR(err)); - - return 0; } static struct platform_driver agdi_driver = { @@ -90,7 +88,7 @@ static struct platform_driver agdi_driver = { .name = "agdi", }, .probe = agdi_probe, - .remove = agdi_remove, + .remove_new = agdi_remove, }; void __init acpi_agdi_init(void) From da22084d586670fa5cfdc92fd7e350cf1fc737de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:35 +0100 Subject: [PATCH 0835/1038] ACPI: DPTF: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert these two drivers from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/dptf/dptf_pch_fivr.c | 6 ++---- drivers/acpi/dptf/dptf_power.c | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/acpi/dptf/dptf_pch_fivr.c b/drivers/acpi/dptf/dptf_pch_fivr.c index 4919e7abe93f..654aaa53c67f 100644 --- a/drivers/acpi/dptf/dptf_pch_fivr.c +++ b/drivers/acpi/dptf/dptf_pch_fivr.c @@ -141,11 +141,9 @@ static int pch_fivr_add(struct platform_device *pdev) return 0; } -static int pch_fivr_remove(struct platform_device *pdev) +static void pch_fivr_remove(struct platform_device *pdev) { sysfs_remove_group(&pdev->dev.kobj, &pch_fivr_attribute_group); - - return 0; } static const struct acpi_device_id pch_fivr_device_ids[] = { @@ -159,7 +157,7 @@ MODULE_DEVICE_TABLE(acpi, pch_fivr_device_ids); static struct platform_driver pch_fivr_driver = { .probe = pch_fivr_add, - .remove = pch_fivr_remove, + .remove_new = pch_fivr_remove, .driver = { .name = "dptf_pch_fivr", .acpi_match_table = pch_fivr_device_ids, diff --git a/drivers/acpi/dptf/dptf_power.c b/drivers/acpi/dptf/dptf_power.c index 86561eda939f..b8187babbbbb 100644 --- a/drivers/acpi/dptf/dptf_power.c +++ b/drivers/acpi/dptf/dptf_power.c @@ -209,7 +209,7 @@ static int dptf_power_add(struct platform_device *pdev) return 0; } -static int dptf_power_remove(struct platform_device *pdev) +static void dptf_power_remove(struct platform_device *pdev) { struct acpi_device *acpi_dev = platform_get_drvdata(pdev); @@ -221,8 +221,6 @@ static int dptf_power_remove(struct platform_device *pdev) sysfs_remove_group(&pdev->dev.kobj, &dptf_battery_attribute_group); else sysfs_remove_group(&pdev->dev.kobj, &dptf_power_attribute_group); - - return 0; } static const struct acpi_device_id int3407_device_ids[] = { @@ -242,7 +240,7 @@ MODULE_DEVICE_TABLE(acpi, int3407_device_ids); static struct platform_driver dptf_power_driver = { .probe = dptf_power_add, - .remove = dptf_power_remove, + .remove_new = dptf_power_remove, .driver = { .name = "dptf_power", .acpi_match_table = int3407_device_ids, From c21f50e1f39408d43b0fd034764ea63985db2d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:36 +0100 Subject: [PATCH 0836/1038] ACPI: GED: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/evged.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/evged.c b/drivers/acpi/evged.c index fe6b6792c8bb..11778c93254b 100644 --- a/drivers/acpi/evged.c +++ b/drivers/acpi/evged.c @@ -173,10 +173,9 @@ static void ged_shutdown(struct platform_device *pdev) } } -static int ged_remove(struct platform_device *pdev) +static void ged_remove(struct platform_device *pdev) { ged_shutdown(pdev); - return 0; } static const struct acpi_device_id ged_acpi_ids[] = { @@ -186,7 +185,7 @@ static const struct acpi_device_id ged_acpi_ids[] = { static struct platform_driver ged_driver = { .probe = ged_probe, - .remove = ged_remove, + .remove_new = ged_remove, .shutdown = ged_shutdown, .driver = { .name = MODULE_NAME, From 24fd13c0824f1fc3cb2db0f96d8ac78302b5beb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:37 +0100 Subject: [PATCH 0837/1038] ACPI: fan: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/fan_core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c index 9dccbae9e8ea..ff72e4ef8738 100644 --- a/drivers/acpi/fan_core.c +++ b/drivers/acpi/fan_core.c @@ -387,7 +387,7 @@ err_end: return result; } -static int acpi_fan_remove(struct platform_device *pdev) +static void acpi_fan_remove(struct platform_device *pdev) { struct acpi_fan *fan = platform_get_drvdata(pdev); @@ -399,8 +399,6 @@ static int acpi_fan_remove(struct platform_device *pdev) sysfs_remove_link(&pdev->dev.kobj, "thermal_cooling"); sysfs_remove_link(&fan->cdev->device.kobj, "device"); thermal_cooling_device_unregister(fan->cdev); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -446,7 +444,7 @@ static const struct dev_pm_ops acpi_fan_pm = { static struct platform_driver acpi_fan_driver = { .probe = acpi_fan_probe, - .remove = acpi_fan_remove, + .remove_new = acpi_fan_remove, .driver = { .name = "acpi-fan", .acpi_match_table = fan_device_ids, From b4a48c50589e4096d4cc5fefbcea4ebc9a7f31ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:38 +0100 Subject: [PATCH 0838/1038] ACPI: pfr_telemetry: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/pfr_telemetry.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/pfr_telemetry.c b/drivers/acpi/pfr_telemetry.c index 843f678ade0c..998264a7333d 100644 --- a/drivers/acpi/pfr_telemetry.c +++ b/drivers/acpi/pfr_telemetry.c @@ -347,13 +347,11 @@ static const struct file_operations acpi_pfrt_log_fops = { .llseek = noop_llseek, }; -static int acpi_pfrt_log_remove(struct platform_device *pdev) +static void acpi_pfrt_log_remove(struct platform_device *pdev) { struct pfrt_log_device *pfrt_log_dev = platform_get_drvdata(pdev); misc_deregister(&pfrt_log_dev->miscdev); - - return 0; } static void pfrt_log_put_idx(void *data) @@ -427,7 +425,7 @@ static struct platform_driver acpi_pfrt_log_driver = { .acpi_match_table = acpi_pfrt_log_ids, }, .probe = acpi_pfrt_log_probe, - .remove = acpi_pfrt_log_remove, + .remove_new = acpi_pfrt_log_remove, }; module_platform_driver(acpi_pfrt_log_driver); From 74550b070d0b40f5b9aaa8c1161e7b587e391289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 Feb 2024 19:52:39 +0100 Subject: [PATCH 0839/1038] ACPI: pfr_update: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Signed-off-by: Rafael J. Wysocki --- drivers/acpi/pfr_update.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c index 98267f163e2b..8b2910995fc1 100644 --- a/drivers/acpi/pfr_update.c +++ b/drivers/acpi/pfr_update.c @@ -489,13 +489,11 @@ static const struct file_operations acpi_pfru_fops = { .llseek = noop_llseek, }; -static int acpi_pfru_remove(struct platform_device *pdev) +static void acpi_pfru_remove(struct platform_device *pdev) { struct pfru_device *pfru_dev = platform_get_drvdata(pdev); misc_deregister(&pfru_dev->miscdev); - - return 0; } static void pfru_put_idx(void *data) @@ -567,7 +565,7 @@ static struct platform_driver acpi_pfru_driver = { .acpi_match_table = acpi_pfru_ids, }, .probe = acpi_pfru_probe, - .remove = acpi_pfru_remove, + .remove_new = acpi_pfru_remove, }; module_platform_driver(acpi_pfru_driver); From d748b48eeba8e1a10c822109252b75ca30288ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Sat, 10 Feb 2024 17:40:07 +0100 Subject: [PATCH 0840/1038] spi: ppc4xx: Fix fallout from rename in struct spi_bitbang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I failed to adapt this driver because it's not enabled in a powerpc allmodconfig build and also wasn't hit by my grep expertise. Fix accordingly. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402100815.XQXw9XCF-lkp@intel.com/ Fixes: 2259233110d9 ("spi: bitbang: Follow renaming of SPI "master" to "controller"") Signed-off-by: Uwe Kleine-König Link: https://msgid.link/r/20240210164006.208149-7-u.kleine-koenig@pengutronix.de Signed-off-by: Mark Brown --- drivers/spi/spi-ppc4xx.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/spi/spi-ppc4xx.c b/drivers/spi/spi-ppc4xx.c index 03aab661be9d..b07bb5e5811e 100644 --- a/drivers/spi/spi-ppc4xx.c +++ b/drivers/spi/spi-ppc4xx.c @@ -362,22 +362,22 @@ static int spi_ppc4xx_of_probe(struct platform_device *op) /* Setup the state for the bitbang driver */ bbp = &hw->bitbang; - bbp->master = hw->host; + bbp->ctlr = hw->host; bbp->setup_transfer = spi_ppc4xx_setupxfer; bbp->txrx_bufs = spi_ppc4xx_txrx; bbp->use_dma = 0; - bbp->master->setup = spi_ppc4xx_setup; - bbp->master->cleanup = spi_ppc4xx_cleanup; - bbp->master->bits_per_word_mask = SPI_BPW_MASK(8); - bbp->master->use_gpio_descriptors = true; + bbp->ctlr->setup = spi_ppc4xx_setup; + bbp->ctlr->cleanup = spi_ppc4xx_cleanup; + bbp->ctlr->bits_per_word_mask = SPI_BPW_MASK(8); + bbp->ctlr->use_gpio_descriptors = true; /* * The SPI core will count the number of GPIO descriptors to figure * out the number of chip selects available on the platform. */ - bbp->master->num_chipselect = 0; + bbp->ctlr->num_chipselect = 0; /* the spi->mode bits understood by this driver: */ - bbp->master->mode_bits = + bbp->ctlr->mode_bits = SPI_CPHA | SPI_CPOL | SPI_CS_HIGH | SPI_LSB_FIRST; /* Get the clock for the OPB */ From 021a67d096154893cd1d883c7be0097e2ee327fd Mon Sep 17 00:00:00 2001 From: Maxim Kudinov Date: Fri, 23 Feb 2024 19:24:08 +0300 Subject: [PATCH 0841/1038] ACPI: resource: Add MAIBENBEN X577 to irq1_edge_low_force_override A known issue on some Zen laptops, keyboard stopped working due to commit 9946e39fe8d0 fael@kernel.org("ACPI: resource: skip IRQ override on AMD Zen platforms") on kernel 5.19.10. The ACPI IRQ override is required for this board due to buggy DSDT, thus adding the board vendor and name to irq1_edge_low_force_override fixes the issue. Fixes: 9946e39fe8d0 ("ACPI: resource: skip IRQ override on AMD Zen platforms") Link: https://bugzilla.kernel.org/show_bug.cgi?id=217394 Link: https://lore.kernel.org/linux-acpi/20231006123304.32686-1-hdegoede@redhat.com/ Tested-by: Maxim Trofimov Signed-off-by: Maxim Kudinov Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index cc5b59fbc714..4fae9af32f69 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -616,6 +616,13 @@ static const struct dmi_system_id irq1_edge_low_force_override[] = { DMI_MATCH(DMI_BOARD_NAME, "LL6FA"), }, }, + { + /* MAIBENBEN X577 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "MAIBENBEN"), + DMI_MATCH(DMI_BOARD_NAME, "X577"), + }, + }, { } }; From 793551c965116d9dfaf0550dacae1396a20efa69 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 26 Feb 2024 17:35:27 +0100 Subject: [PATCH 0842/1038] ACPI: scan: Fix device check notification handling It is generally invalid to fail a Device Check notification if the scan handler has not been attached to the given device after a bus rescan, because there may be valid reasons for the scan handler to refuse attaching to the device (for example, the device is not ready). For this reason, modify acpi_scan_device_check() to return 0 in that case without printing a warning. While at it, reduce the log level of the "already enumerated" message in the same function, because it is only interesting when debugging notification handling Fixes: 443fc8202272 ("ACPI / hotplug: Rework generic code to handle suprise removals") Signed-off-by: Rafael J. Wysocki Reviewed-by: Jonathan Cameron --- drivers/acpi/scan.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index f32a2c738c8b..dd817e9ed2e5 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -314,18 +314,14 @@ static int acpi_scan_device_check(struct acpi_device *adev) * again). */ if (adev->handler) { - dev_warn(&adev->dev, "Already enumerated\n"); - return -EALREADY; + dev_dbg(&adev->dev, "Already enumerated\n"); + return 0; } error = acpi_bus_scan(adev->handle); if (error) { dev_warn(&adev->dev, "Namespace scan failure\n"); return error; } - if (!adev->handler) { - dev_warn(&adev->dev, "Enumeration failure\n"); - error = -ENODEV; - } } else { error = acpi_scan_device_not_enumerated(adev); } From 514bcabc05387369ee491b8359f34fcd45f25f55 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 26 Feb 2024 17:36:22 +0100 Subject: [PATCH 0843/1038] ACPI: scan: Relocate acpi_bus_trim_one() Relocate acpi_bus_trim_one() (without modifications) so as to avoid the need to add a forward declaration of it in a subsequent patch. No functional changes. Signed-off-by: Rafael J. Wysocki Reviewed-by: Jonathan Cameron --- drivers/acpi/scan.c | 52 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index dd817e9ed2e5..e641a7c87358 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -244,6 +244,32 @@ static int acpi_scan_try_to_offline(struct acpi_device *device) return 0; } +static int acpi_bus_trim_one(struct acpi_device *adev, void *not_used) +{ + struct acpi_scan_handler *handler = adev->handler; + + acpi_dev_for_each_child_reverse(adev, acpi_bus_trim_one, NULL); + + adev->flags.match_driver = false; + if (handler) { + if (handler->detach) + handler->detach(adev); + + adev->handler = NULL; + } else { + device_release_driver(&adev->dev); + } + /* + * Most likely, the device is going away, so put it into D3cold before + * that. + */ + acpi_device_set_power(adev, ACPI_STATE_D3_COLD); + adev->flags.initialized = false; + acpi_device_clear_enumerated(adev); + + return 0; +} + static int acpi_scan_hot_remove(struct acpi_device *device) { acpi_handle handle = device->handle; @@ -2547,32 +2573,6 @@ int acpi_bus_scan(acpi_handle handle) } EXPORT_SYMBOL(acpi_bus_scan); -static int acpi_bus_trim_one(struct acpi_device *adev, void *not_used) -{ - struct acpi_scan_handler *handler = adev->handler; - - acpi_dev_for_each_child_reverse(adev, acpi_bus_trim_one, NULL); - - adev->flags.match_driver = false; - if (handler) { - if (handler->detach) - handler->detach(adev); - - adev->handler = NULL; - } else { - device_release_driver(&adev->dev); - } - /* - * Most likely, the device is going away, so put it into D3cold before - * that. - */ - acpi_device_set_power(adev, ACPI_STATE_D3_COLD); - adev->flags.initialized = false; - acpi_device_clear_enumerated(adev); - - return 0; -} - /** * acpi_bus_trim - Detach scan handlers and drivers from ACPI device objects. * @adev: Root of the ACPI namespace scope to walk. From 1b4f02a34f095b96de36fa16755f5598ba380939 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 26 Feb 2024 17:40:52 +0100 Subject: [PATCH 0844/1038] ACPI: scan: Make acpi_processor_add() check the device enabled bit Modify acpi_processor_add() return an error if _STA returns the enabled bit clear for the given processor device, so as to avoid using processors that don't decode their resources, as per the ACPI specification. [1] Link: https://uefi.org/specs/ACPI/6.5/06_Device_Configuration.html#sta-device-status # [1] Signed-off-by: Rafael J. Wysocki Reviewed-by: Jonathan Cameron --- drivers/acpi/acpi_processor.c | 3 +++ drivers/acpi/internal.h | 1 + drivers/acpi/scan.c | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 4fe2ef54088c..99c28b7307c3 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -381,6 +381,9 @@ static int acpi_processor_add(struct acpi_device *device, struct device *dev; int result = 0; + if (!acpi_device_is_enabled(device)) + return -ENODEV; + pr = kzalloc(sizeof(struct acpi_processor), GFP_KERNEL); if (!pr) return -ENOMEM; diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 6588525c45ef..895aa2c0ad58 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -121,6 +121,7 @@ int acpi_device_setup_files(struct acpi_device *dev); void acpi_device_remove_files(struct acpi_device *dev); void acpi_device_add_finalize(struct acpi_device *device); void acpi_free_pnp_ids(struct acpi_device_pnp *pnp); +bool acpi_device_is_enabled(const struct acpi_device *adev); bool acpi_device_is_present(const struct acpi_device *adev); bool acpi_device_is_battery(struct acpi_device *adev); bool acpi_device_is_first_physical_node(struct acpi_device *adev, diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index e641a7c87358..fddcb1691fd3 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -1945,6 +1945,11 @@ bool acpi_device_is_present(const struct acpi_device *adev) return adev->status.present || adev->status.functional; } +bool acpi_device_is_enabled(const struct acpi_device *adev) +{ + return adev->status.present && adev->status.enabled; +} + static bool acpi_scan_handler_matching(struct acpi_scan_handler *handler, const char *idstr, const struct acpi_device_id **matchid) From 520c2286c222c0a6c9b366e9c2a6c42c3fc091ed Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 26 Feb 2024 17:45:11 +0100 Subject: [PATCH 0845/1038] ACPI: scan: Rework Device Check and Bus Check notification handling The underlying problem is the handling of the enabled bit in device status (bit 1 of _STA return value) which is required by the ACPI specification to be observed in addition to the present bit (bit 0 of _STA return value) [1], but Linux does not observe it. Since Linux has not looked at that bit for a long time, it is generally risky to start obseving it in all device enumeration cases, especially at the system initialization time, but it can be observed when the kernel receives a Bus Check or Device Check notification indicating a change in device configuration. In those cases, seeing the enabled bit clear may be regarded as an indication that the device at hand should not be used any more. For this reason, rework the handling of Device Check and Bus Check notifications in the ACPI core device enumeration code in the following way: 1. Rename acpi_bus_trim_one() to acpi_scan_check_and_detach() and make it check device status if its second argument is not NULL, in which case it will detach scan handlers or ACPI drivers from devices whose _STA returns the enabled bit clear. 2. Make acpi_scan_device_check() and acpi_scan_bus_check() invoke acpi_scan_check_and_detach() with a non-NULL second argument unconditionally, so scan handlers and ACPI drivers are detached from the target device and its ancestors if their _STA returns the enabled bit clear. Link: https://uefi.org/specs/ACPI/6.5/06_Device_Configuration.html#sta-device-status # [1] Signed-off-by: Rafael J. Wysocki Reviewed-by: Jonathan Cameron --- drivers/acpi/scan.c | 84 ++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index fddcb1691fd3..b35722b650ab 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -244,11 +244,27 @@ static int acpi_scan_try_to_offline(struct acpi_device *device) return 0; } -static int acpi_bus_trim_one(struct acpi_device *adev, void *not_used) +static int acpi_scan_check_and_detach(struct acpi_device *adev, void *check) { struct acpi_scan_handler *handler = adev->handler; - acpi_dev_for_each_child_reverse(adev, acpi_bus_trim_one, NULL); + acpi_dev_for_each_child_reverse(adev, acpi_scan_check_and_detach, check); + + if (check) { + acpi_bus_get_status(adev); + /* + * Skip devices that are still there and take the enabled + * flag into account. + */ + if (acpi_device_is_enabled(adev)) + return 0; + + /* Skip device that have not been enumerated. */ + if (!acpi_device_enumerated(adev)) { + dev_dbg(&adev->dev, "Still not enumerated\n"); + return 0; + } + } adev->flags.match_driver = false; if (handler) { @@ -270,6 +286,11 @@ static int acpi_bus_trim_one(struct acpi_device *adev, void *not_used) return 0; } +static void acpi_scan_check_subtree(struct acpi_device *adev) +{ + acpi_scan_check_and_detach(adev, (void *)true); +} + static int acpi_scan_hot_remove(struct acpi_device *device) { acpi_handle handle = device->handle; @@ -315,42 +336,30 @@ static int acpi_scan_hot_remove(struct acpi_device *device) return 0; } -static int acpi_scan_device_not_enumerated(struct acpi_device *adev) -{ - if (!acpi_device_enumerated(adev)) { - dev_warn(&adev->dev, "Still not enumerated\n"); - return -EALREADY; - } - acpi_bus_trim(adev); - return 0; -} - static int acpi_scan_device_check(struct acpi_device *adev) { int error; - acpi_bus_get_status(adev); - if (acpi_device_is_present(adev)) { - /* - * This function is only called for device objects for which - * matching scan handlers exist. The only situation in which - * the scan handler is not attached to this device object yet - * is when the device has just appeared (either it wasn't - * present at all before or it was removed and then added - * again). - */ - if (adev->handler) { - dev_dbg(&adev->dev, "Already enumerated\n"); - return 0; - } - error = acpi_bus_scan(adev->handle); - if (error) { - dev_warn(&adev->dev, "Namespace scan failure\n"); - return error; - } - } else { - error = acpi_scan_device_not_enumerated(adev); + acpi_scan_check_subtree(adev); + + if (!acpi_device_is_present(adev)) + return 0; + + /* + * This function is only called for device objects for which matching + * scan handlers exist. The only situation in which the scan handler + * is not attached to this device object yet is when the device has + * just appeared (either it wasn't present at all before or it was + * removed and then added again). + */ + if (adev->handler) { + dev_dbg(&adev->dev, "Already enumerated\n"); + return 0; } + error = acpi_bus_scan(adev->handle); + if (error) + dev_warn(&adev->dev, "Namespace scan failure\n"); + return error; } @@ -359,11 +368,8 @@ static int acpi_scan_bus_check(struct acpi_device *adev, void *not_used) struct acpi_scan_handler *handler = adev->handler; int error; - acpi_bus_get_status(adev); - if (!acpi_device_is_present(adev)) { - acpi_scan_device_not_enumerated(adev); - return 0; - } + acpi_scan_check_subtree(adev); + if (handler && handler->hotplug.scan_dependent) return handler->hotplug.scan_dependent(adev); @@ -2586,7 +2592,7 @@ EXPORT_SYMBOL(acpi_bus_scan); */ void acpi_bus_trim(struct acpi_device *adev) { - acpi_bus_trim_one(adev, NULL); + acpi_scan_check_and_detach(adev, NULL); } EXPORT_SYMBOL_GPL(acpi_bus_trim); From 4f4a335acfbb72bd11185c97394b3c0890e72453 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 26 Feb 2024 17:46:41 +0100 Subject: [PATCH 0846/1038] ACPI: scan: Consolidate Device Check and Bus Check notification handling There is no particular reason why device object subtree rescans in acpi_scan_device_check() and acpi_scan_device_check() should be carried out differently, so move the rescan code into a new function called acpi_scan_rescan_bus() and make both the functions above invoke it. While at it, in the Device Check case, start the device object subtree rescan mentioned above from the target device's parent, as per the specification. [1] Link: https://uefi.org/specs/ACPI/6.5/05_ACPI_Software_Programming_Model.html#device-object-notification-values # [1] Signed-off-by: Rafael J. Wysocki Reviewed-by: Jonathan Cameron --- drivers/acpi/scan.c | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index b35722b650ab..3b722e4c0f06 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -336,9 +336,25 @@ static int acpi_scan_hot_remove(struct acpi_device *device) return 0; } +static int acpi_scan_rescan_bus(struct acpi_device *adev) +{ + struct acpi_scan_handler *handler = adev->handler; + int ret; + + if (handler && handler->hotplug.scan_dependent) + ret = handler->hotplug.scan_dependent(adev); + else + ret = acpi_bus_scan(adev->handle); + + if (ret) + dev_info(&adev->dev, "Namespace scan failure\n"); + + return ret; +} + static int acpi_scan_device_check(struct acpi_device *adev) { - int error; + struct acpi_device *parent; acpi_scan_check_subtree(adev); @@ -356,36 +372,26 @@ static int acpi_scan_device_check(struct acpi_device *adev) dev_dbg(&adev->dev, "Already enumerated\n"); return 0; } - error = acpi_bus_scan(adev->handle); - if (error) - dev_warn(&adev->dev, "Namespace scan failure\n"); - return error; + parent = acpi_dev_parent(adev); + if (!parent) + parent = adev; + + return acpi_scan_rescan_bus(parent); } -static int acpi_scan_bus_check(struct acpi_device *adev, void *not_used) +static int acpi_scan_bus_check(struct acpi_device *adev) { - struct acpi_scan_handler *handler = adev->handler; - int error; - acpi_scan_check_subtree(adev); - if (handler && handler->hotplug.scan_dependent) - return handler->hotplug.scan_dependent(adev); - - error = acpi_bus_scan(adev->handle); - if (error) { - dev_warn(&adev->dev, "Namespace scan failure\n"); - return error; - } - return acpi_dev_for_each_child(adev, acpi_scan_bus_check, NULL); + return acpi_scan_rescan_bus(adev); } static int acpi_generic_hotplug_event(struct acpi_device *adev, u32 type) { switch (type) { case ACPI_NOTIFY_BUS_CHECK: - return acpi_scan_bus_check(adev, NULL); + return acpi_scan_bus_check(adev); case ACPI_NOTIFY_DEVICE_CHECK: return acpi_scan_device_check(adev); case ACPI_NOTIFY_EJECT_REQUEST: From 6f3d56882a3f210e7d1b6a169c4b32d926283f39 Mon Sep 17 00:00:00 2001 From: Marco Felsch Date: Wed, 21 Feb 2024 19:46:02 +0100 Subject: [PATCH 0847/1038] dt-bindings: mmc: fsl-imx-esdhc: add default and 100mhz state Some devices support only the default and the 100MHz case, add the support for this to the binding to avoid warnings. Signed-off-by: Marco Felsch Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240221184602.3639619-1-m.felsch@pengutronix.de Signed-off-by: Ulf Hansson --- Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml index 3f6dcc773092..82f7ee8702cb 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml @@ -177,6 +177,11 @@ properties: - const: state_100mhz - const: state_200mhz - const: sleep + - minItems: 2 + items: + - const: default + - const: state_100mhz + - const: sleep - minItems: 1 items: - const: default From 44d41bf65a4bd74e00300fda0b3220921128f722 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 28 Feb 2024 10:07:27 +0100 Subject: [PATCH 0848/1038] mmc: sdhci-esdhc-mcf: Flag the sg_miter as atomic The sg_miter used to loop over the returned sglist from a transfer in the esdhc subdriver for SDHCI can be called from atomic context so the miter needs to be atomic. sdhci_request_done() is always called from process context, either as a work or as part of the threaded interrupt handler, but the one case when we are actually calling .request_done() from an atomic context is in sdhci_irq(). Fix this by flagging the miter atomic so we always use kmap_atomic(). Fixes: e8a167b84886 ("mmc: sdhci-esdhc-mcf: Use sg_miter for swapping") Reported-by: Adrian Hunter Signed-off-by: Linus Walleij Acked-by: Adrian Hunter Link: https://lore.kernel.org/r/20240228-fix-sdhci-esdhc-mcf-2-v2-1-4ebb3fd691ea@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-esdhc-mcf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/host/sdhci-esdhc-mcf.c b/drivers/mmc/host/sdhci-esdhc-mcf.c index 1909a11fd065..c97363e2d86c 100644 --- a/drivers/mmc/host/sdhci-esdhc-mcf.c +++ b/drivers/mmc/host/sdhci-esdhc-mcf.c @@ -313,7 +313,7 @@ static void esdhc_mcf_request_done(struct sdhci_host *host, * transfer endiannes. A swap after the transfer is needed. */ sg_miter_start(&sgm, mrq->data->sg, mrq->data->sg_len, - SG_MITER_TO_SG | SG_MITER_FROM_SG); + SG_MITER_ATOMIC | SG_MITER_TO_SG | SG_MITER_FROM_SG); while (sg_miter_next(&sgm)) { buffer = sgm.addr; esdhc_mcf_buffer_swap32(buffer, sgm.length); From 666db8fd4265f938795004838d2a9335ce7b9da1 Mon Sep 17 00:00:00 2001 From: Varshini Rajendran Date: Fri, 23 Feb 2024 22:56:38 +0530 Subject: [PATCH 0849/1038] spi: dt-bindings: atmel,at91rm9200-spi: remove 9x60 compatible from list Remove microchip,sam9x60-spi compatible from the list as the driver used has the compatible atmel,at91rm9200-spi and sam9x60 devices also use the same compatible as fallback. So removing the microchip,sam9x60-spi compatible from the list since it is not needed. Signed-off-by: Varshini Rajendran Reviewed-by: Tudor Ambarus Link: https://msgid.link/r/20240223172638.672366-1-varshini.rajendran@microchip.com Signed-off-by: Mark Brown --- Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml index 58367587bfbc..32e7c14033c2 100644 --- a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml +++ b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml @@ -22,7 +22,6 @@ properties: - const: atmel,at91rm9200-spi - items: - const: microchip,sam9x7-spi - - const: microchip,sam9x60-spi - const: atmel,at91rm9200-spi reg: From 727cba706d38bcd1460b21239992e917be7da8fb Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 21 Feb 2024 22:23:00 +0100 Subject: [PATCH 0850/1038] mmc: sh_mmcif: sg_miter must not be atomic All the sglist iterations happen in the *threaded* interrupt handler and that context is not atomic, so don't request an atomic sglist miter. Using an atomic miter results in "BUG: scheduling while atomic" splats. Reported-by: Geert Uytterhoeven Fixes: 27b57277d9ba ("mmc: sh_mmcif: Use sg_miter for PIO") Tested-by: Geert Uytterhoeven Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20240221-fix-sh-mmcif-v2-1-5e521eb25ae4@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/sh_mmcif.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c index 1ef6e153e5a3..669555b5e8fa 100644 --- a/drivers/mmc/host/sh_mmcif.c +++ b/drivers/mmc/host/sh_mmcif.c @@ -607,7 +607,7 @@ static void sh_mmcif_single_read(struct sh_mmcif_host *host, BLOCK_SIZE_MASK) + 3; sg_miter_start(&host->sg_miter, data->sg, data->sg_len, - SG_MITER_ATOMIC | SG_MITER_TO_SG); + SG_MITER_TO_SG); host->wait_for = MMCIF_WAIT_FOR_READ; @@ -662,7 +662,7 @@ static void sh_mmcif_multi_read(struct sh_mmcif_host *host, BLOCK_SIZE_MASK; sg_miter_start(&host->sg_miter, data->sg, data->sg_len, - SG_MITER_ATOMIC | SG_MITER_TO_SG); + SG_MITER_TO_SG); host->wait_for = MMCIF_WAIT_FOR_MREAD; @@ -710,7 +710,7 @@ static void sh_mmcif_single_write(struct sh_mmcif_host *host, BLOCK_SIZE_MASK) + 3; sg_miter_start(&host->sg_miter, data->sg, data->sg_len, - SG_MITER_ATOMIC | SG_MITER_FROM_SG); + SG_MITER_FROM_SG); host->wait_for = MMCIF_WAIT_FOR_WRITE; @@ -765,7 +765,7 @@ static void sh_mmcif_multi_write(struct sh_mmcif_host *host, BLOCK_SIZE_MASK; sg_miter_start(&host->sg_miter, data->sg, data->sg_len, - SG_MITER_ATOMIC | SG_MITER_FROM_SG); + SG_MITER_FROM_SG); host->wait_for = MMCIF_WAIT_FOR_MWRITE; From 4d4a2c71b862b9e696006fd264dbf5dbd6484d77 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 21 Feb 2024 22:23:01 +0100 Subject: [PATCH 0851/1038] mmc: sh_mmcif: Advance sg_miter before reading blocks The introduction of sg_miter was a bit sloppy as it didn't exactly mimic the semantics of the old code on multiblock reads and writes: these like you to: - Advance to the first sglist entry *before* starting to read any blocks from the card. - Advance and check availability of the next entry *right after* processing one block. Not checking if we have more sglist entries right after reading a block will lead to this not being checked until we return to the callback to read out more blocks, i.e. until the next interrupt arrives. Since the last block is the last one (no more data will arrive) there will not be a next interrupt, and we will be waiting forever resulting in a timeout for command 18 when reading multiple blocks. The same bug was fixed also in the writing of multiple blocks. Reported-by: Geert Uytterhoeven Fixes: 27b57277d9ba ("mmc: sh_mmcif: Use sg_miter for PIO") Signed-off-by: Linus Walleij Tested-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240221-fix-sh-mmcif-v2-2-5e521eb25ae4@linaro.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/sh_mmcif.c | 38 +++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c index 669555b5e8fa..08b4312af94e 100644 --- a/drivers/mmc/host/sh_mmcif.c +++ b/drivers/mmc/host/sh_mmcif.c @@ -653,6 +653,7 @@ static bool sh_mmcif_read_block(struct sh_mmcif_host *host) static void sh_mmcif_multi_read(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct mmc_data *data = mrq->data; if (!data->sg_len || !data->sg->length) @@ -661,9 +662,15 @@ static void sh_mmcif_multi_read(struct sh_mmcif_host *host, host->blocksize = sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK; - sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + sg_miter_start(sgm, data->sg, data->sg_len, SG_MITER_TO_SG); + /* Advance to the first sglist entry */ + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return; + } + host->wait_for = MMCIF_WAIT_FOR_MREAD; sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFREN); @@ -684,11 +691,6 @@ static bool sh_mmcif_mread_block(struct sh_mmcif_host *host) return false; } - if (!sg_miter_next(sgm)) { - sg_miter_stop(sgm); - return false; - } - p = sgm->addr; for (i = 0; i < host->blocksize / 4; i++) @@ -698,6 +700,11 @@ static bool sh_mmcif_mread_block(struct sh_mmcif_host *host) sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFREN); + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return false; + } + return true; } @@ -756,6 +763,7 @@ static bool sh_mmcif_write_block(struct sh_mmcif_host *host) static void sh_mmcif_multi_write(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct mmc_data *data = mrq->data; if (!data->sg_len || !data->sg->length) @@ -764,9 +772,15 @@ static void sh_mmcif_multi_write(struct sh_mmcif_host *host, host->blocksize = sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK; - sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + sg_miter_start(sgm, data->sg, data->sg_len, SG_MITER_FROM_SG); + /* Advance to the first sglist entry */ + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return; + } + host->wait_for = MMCIF_WAIT_FOR_MWRITE; sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFWEN); @@ -787,11 +801,6 @@ static bool sh_mmcif_mwrite_block(struct sh_mmcif_host *host) return false; } - if (!sg_miter_next(sgm)) { - sg_miter_stop(sgm); - return false; - } - p = sgm->addr; for (i = 0; i < host->blocksize / 4; i++) @@ -799,6 +808,11 @@ static bool sh_mmcif_mwrite_block(struct sh_mmcif_host *host) sgm->consumed = host->blocksize; + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return false; + } + sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFWEN); return true; From a119357feff8afd96af5d8e9bd47d8f3735110f6 Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Thu, 22 Feb 2024 11:39:11 -0300 Subject: [PATCH 0852/1038] dt-bindings: mmc: fsl-imx-mmc: Document the required clocks The fsl-imx-mmc hardware needs two clocks to operate: ipg and per. Document these required clocks. This fixes the following schema warning: imx27-apf27dev.dtb: mmc@10014000: Unevaluated properties are not allowed ('clock-names', 'clocks' were unexpected) from schema $id: http://devicetree.org/schemas/mmc/fsl-imx-mmc.yaml# Signed-off-by: Fabio Estevam Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240222143911.979058-1-festevam@gmail.com Signed-off-by: Ulf Hansson --- .../devicetree/bindings/mmc/fsl-imx-mmc.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml index 221f5bc047bd..7911316fbd6a 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml @@ -24,6 +24,14 @@ properties: reg: maxItems: 1 + clocks: + maxItems: 2 + + clock-names: + items: + - const: ipg + - const: per + interrupts: maxItems: 1 @@ -34,6 +42,8 @@ properties: const: rx-tx required: + - clocks + - clock-names - compatible - reg - interrupts @@ -46,6 +56,8 @@ examples: compatible = "fsl,imx27-mmc", "fsl,imx21-mmc"; reg = <0x10014000 0x1000>; interrupts = <11>; + clocks = <&clks 29>, <&clks 60>; + clock-names = "ipg", "per"; dmas = <&dma 7>; dma-names = "rx-tx"; bus-width = <4>; From 02e765697038c596dc4a1126a13b018608365d81 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Tue, 9 Jan 2024 15:04:56 +0800 Subject: [PATCH 0853/1038] swiotlb: add debugfs to track swiotlb transient pool usage Introduce a new debugfs interface io_tlb_transient_nslabs. The device driver can create a new swiotlb transient memory pool once default memory pool is full. To export the swiotlb transient memory pool usage via debugfs would help the user estimate the size of transient swiotlb memory pool or analyze device driver memory leak issue. Signed-off-by: ZhangPeng Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 3 +++ kernel/dma/swiotlb.c | 46 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index ecde0312dd52..ea23097e351f 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -120,6 +120,8 @@ struct io_tlb_pool { * debugfs. * @used_hiwater: The high water mark for total_used. Used only for reporting * in debugfs. + * @transient_nslabs: The total number of slots in all transient pools that + * are currently used across all areas. */ struct io_tlb_mem { struct io_tlb_pool defpool; @@ -137,6 +139,7 @@ struct io_tlb_mem { #ifdef CONFIG_DEBUG_FS atomic_long_t total_used; atomic_long_t used_hiwater; + atomic_long_t transient_nslabs; #endif }; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index b079a9a8e087..77974cea3e69 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -956,6 +956,28 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) } #endif /* CONFIG_DEBUG_FS */ +#ifdef CONFIG_SWIOTLB_DYNAMIC +#ifdef CONFIG_DEBUG_FS +static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_add(nslots, &mem->transient_nslabs); +} + +static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_sub(nslots, &mem->transient_nslabs); +} + +#else /* !CONFIG_DEBUG_FS */ +static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_SWIOTLB_DYNAMIC */ + /** * swiotlb_search_pool_area() - search one memory area in one pool * @dev: Device which maps the buffer. @@ -1170,6 +1192,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); list_add_rcu(&pool->node, &dev->dma_io_tlb_pools); spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); + inc_transient_used(mem, pool->nslabs); found: WRITE_ONCE(dev->dma_uses_io_tlb, true); @@ -1415,6 +1438,7 @@ static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr) dec_used(dev->dma_io_tlb_mem, pool->nslabs); swiotlb_del_pool(dev, pool); + dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs); return true; } @@ -1557,6 +1581,23 @@ phys_addr_t default_swiotlb_limit(void) } #ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_SWIOTLB_DYNAMIC +static unsigned long mem_transient_used(struct io_tlb_mem *mem) +{ + return atomic_long_read(&mem->transient_nslabs); +} + +static int io_tlb_transient_used_get(void *data, u64 *val) +{ + struct io_tlb_mem *mem = data; + + *val = mem_transient_used(mem); + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get, + NULL, "%llu\n"); +#endif /* CONFIG_SWIOTLB_DYNAMIC */ static int io_tlb_used_get(void *data, u64 *val) { @@ -1605,6 +1646,11 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, &fops_io_tlb_used); debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem, &fops_io_tlb_hiwater); +#ifdef CONFIG_SWIOTLB_DYNAMIC + atomic_long_set(&mem->transient_nslabs, 0); + debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs, + mem, &fops_io_tlb_transient_used); +#endif } static int __init swiotlb_create_default_debugfs(void) From b9fa16949d18e06bdf728a560f5c8af56d2bdcaf Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Wed, 21 Feb 2024 16:17:21 -0800 Subject: [PATCH 0854/1038] dma-direct: Leak pages on dma_set_decrypted() failure On TDX it is possible for the untrusted host to cause set_memory_encrypted() or set_memory_decrypted() to fail such that an error is returned and the resulting memory is shared. Callers need to take care to handle these errors to avoid returning decrypted (shared) memory to the page allocator, which could lead to functional or security issues. DMA could free decrypted/shared pages if dma_set_decrypted() fails. This should be a rare case. Just leak the pages in this case instead of freeing them. Signed-off-by: Rick Edgecombe Signed-off-by: Christoph Hellwig --- kernel/dma/direct.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 98b2e192fd69..4d543b1e9d57 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -286,7 +286,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, } else { ret = page_address(page); if (dma_set_decrypted(dev, ret, size)) - goto out_free_pages; + goto out_leak_pages; } memset(ret, 0, size); @@ -307,6 +307,8 @@ out_encrypt_pages: out_free_pages: __dma_direct_free_pages(dev, page, size); return NULL; +out_leak_pages: + return NULL; } void dma_direct_free(struct device *dev, size_t size, @@ -367,12 +369,11 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, ret = page_address(page); if (dma_set_decrypted(dev, ret, size)) - goto out_free_pages; + goto out_leak_pages; memset(ret, 0, size); *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); return page; -out_free_pages: - __dma_direct_free_pages(dev, page, size); +out_leak_pages: return NULL; } From f1f0c445226c8f03ad329cc2d8072556198fb24a Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Tue, 27 Feb 2024 11:41:12 -0800 Subject: [PATCH 0855/1038] thermal: int340x: processor_thermal: Add Lunar Lake-M PCI ID Add Lunar Lake-M PCI ID for processor thermal device. Signed-off-by: Srinivas Pandruvada Signed-off-by: Rafael J. Wysocki --- drivers/thermal/intel/int340x_thermal/processor_thermal_device.h | 1 + .../thermal/intel/int340x_thermal/processor_thermal_device_pci.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h index 95c6013a33fb..674f3c85dfbc 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h @@ -25,6 +25,7 @@ #define PCI_DEVICE_ID_INTEL_HSB_THERMAL 0x0A03 #define PCI_DEVICE_ID_INTEL_ICL_THERMAL 0x8a03 #define PCI_DEVICE_ID_INTEL_JSL_THERMAL 0x4E03 +#define PCI_DEVICE_ID_INTEL_LNLM_THERMAL 0x641D #define PCI_DEVICE_ID_INTEL_MTLP_THERMAL 0x7D03 #define PCI_DEVICE_ID_INTEL_RPL_THERMAL 0xA71D #define PCI_DEVICE_ID_INTEL_SKL_THERMAL 0x1903 diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index d7495571dd5d..4e1dfd283651 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -407,6 +407,7 @@ static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, proc_thermal_pci_suspend, static const struct pci_device_id proc_thermal_pci_ids[] = { { PCI_DEVICE_DATA(INTEL, ADL_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_WT_REQ) }, + { PCI_DEVICE_DATA(INTEL, LNLM_THERMAL, PROC_THERMAL_FEATURE_RAPL) }, { PCI_DEVICE_DATA(INTEL, MTLP_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_DLVR | PROC_THERMAL_FEATURE_WT_HINT | PROC_THERMAL_FEATURE_POWER_FLOOR) }, From d73709646cafc0ea7e6cfa0bbc6fcd8a6c0f89c3 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 28 Feb 2024 21:57:30 +0200 Subject: [PATCH 0856/1038] regulator: da9121: Remove unused of_gpio.h of_gpio.h is deprecated and subject to remove. The driver doesn't use it, simply remove the unused header. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240228195730.3607154-1-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/regulator/da9121-regulator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c index b551a400bdd1..5ee76b533576 100644 --- a/drivers/regulator/da9121-regulator.c +++ b/drivers/regulator/da9121-regulator.c @@ -14,7 +14,6 @@ // Copyright (C) 2020 Dialog Semiconductor #include -#include #include #include #include From 18ebe6f6fccfb093eec85bcdf3e77d48cc03a592 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 28 Feb 2024 21:48:08 +0200 Subject: [PATCH 0857/1038] spi: pic32: Replace of_gpio.h by proper one of_gpio.h is deprecated and subject to remove. The driver doesn't use it directly, replace it with what is really being used. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240228194818.3606841-1-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi-pic32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c index f55b38c577e4..709edb70ad7d 100644 --- a/drivers/spi/spi-pic32.c +++ b/drivers/spi/spi-pic32.c @@ -11,13 +11,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include From bc9c0a9967fea2c0333bea26ab1bbb66c2bff31a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 28 Feb 2024 21:46:32 +0200 Subject: [PATCH 0858/1038] spi: stm32-qspi: Replace of_gpio.h by proper one of_gpio.h is deprecated and subject to remove. The driver doesn't use it directly, replace it with what is really being used. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240228194632.3606563-1-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi-stm32-qspi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c index 385832030459..f1e922fd362a 100644 --- a/drivers/spi/spi-stm32-qspi.c +++ b/drivers/spi/spi-stm32-qspi.c @@ -8,13 +8,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include From 7da51af9125c624318c8099de13c5ddefd47e9e8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:12 -0400 Subject: [PATCH 0859/1038] iommu/arm-smmu-v3: Make STE programming independent of the callers As the comment in arm_smmu_write_strtab_ent() explains, this routine has been limited to only work correctly in certain scenarios that the caller must ensure. Generally the caller must put the STE into ABORT or BYPASS before attempting to program it to something else. The iommu core APIs would ideally expect the driver to do a hitless change of iommu_domain in a number of cases: - RESV_DIRECT support wants IDENTITY -> DMA -> IDENTITY to be hitless for the RESV ranges - PASID upgrade has IDENTIY on the RID with no PASID then a PASID paging domain installed. The RID should not be impacted - PASID downgrade has IDENTIY on the RID and all PASID's removed. The RID should not be impacted - RID does PAGING -> BLOCKING with active PASID, PASID's should not be impacted - NESTING -> NESTING for carrying all the above hitless cases in a VM into the hypervisor. To comprehensively emulate the HW in a VM we should assume the VM OS is running logic like this and expecting hitless updates to be relayed to real HW. For CD updates arm_smmu_write_ctx_desc() has a similar comment explaining how limited it is, and the driver does have a need for hitless CD updates: - SMMUv3 BTM S1 ASID re-label - SVA mm release should change the CD to answert not-present to all requests without allowing logging (EPD0) The next patches/series are going to start removing some of this logic from the callers, and add more complex state combinations than currently. At the end everything that can be hitless will be hitless, including all of the above. Introduce arm_smmu_write_ste() which will run through the multi-qword programming sequence to avoid creating an incoherent 'torn' STE in the HW caches. It automatically detects which of two algorithms to use: 1) The disruptive V=0 update described in the spec which disrupts the entry and does three syncs to make the change: - Write V=0 to QWORD 0 - Write the entire STE except QWORD 0 - Write QWORD 0 2) A hitless update algorithm that follows the same rational that the driver already uses. It is safe to change IGNORED bits that HW doesn't use: - Write the target value into all currently unused bits - Write a single QWORD, this makes the new STE live atomically - Ensure now unused bits are 0 The detection of which path to use and the implementation of the hitless update rely on a "used bitmask" describing what bits the HW is actually using based on the V/CFG/etc bits. This flows from the spec language, typically indicated as IGNORED. Knowing which bits the HW is using we can update the bits it does not use and then compute how many QWORDS need to be changed. If only one qword needs to be updated the hitless algorithm is possible. Later patches will include CD updates in this mechanism so make the implementation generic using a struct arm_smmu_entry_writer and struct arm_smmu_entry_writer_ops to abstract the differences between STE and CD to be plugged in. At this point it generates the same sequence of updates as the current code, except that zeroing the VMID on entry to BYPASS/ABORT will do an extra sync (this seems to be an existing bug). Going forward this will use a V=0 transition instead of cycling through ABORT if a hitfull change is required. This seems more appropriate as ABORT will fail DMAs without any logging, but dropping a DMA due to transient V=0 is probably signaling a bug, so the C_BAD_STE is valuable. Add STRTAB_STE_1_SHCFG_INCOMING to s2_cfg, this was editing the STE in place and subtly inherited the value of data[1] from abort/bypass. Signed-off-by: Michael Shavit Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/1-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 275 +++++++++++++++----- 1 file changed, 211 insertions(+), 64 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 0ffb1cf17e0b..9805d989dafd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -48,6 +48,9 @@ enum arm_smmu_msi_index { ARM_SMMU_MAX_MSIS, }; +static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, + ioasid_t sid); + static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { [EVTQ_MSI_INDEX] = { ARM_SMMU_EVTQ_IRQ_CFG0, @@ -971,6 +974,199 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); } +/* + * Based on the value of ent report which bits of the STE the HW will access. It + * would be nice if this was complete according to the spec, but minimally it + * has to capture the bits this driver uses. + */ +static void arm_smmu_get_ste_used(const struct arm_smmu_ste *ent, + struct arm_smmu_ste *used_bits) +{ + unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent->data[0])); + + used_bits->data[0] = cpu_to_le64(STRTAB_STE_0_V); + if (!(ent->data[0] & cpu_to_le64(STRTAB_STE_0_V))) + return; + + used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_CFG); + + /* S1 translates */ + if (cfg & BIT(0)) { + used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT | + STRTAB_STE_0_S1CTXPTR_MASK | + STRTAB_STE_0_S1CDMAX); + used_bits->data[1] |= + cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR | + STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH | + STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | + STRTAB_STE_1_EATS); + used_bits->data[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); + } + + /* S2 translates */ + if (cfg & BIT(1)) { + used_bits->data[1] |= + cpu_to_le64(STRTAB_STE_1_EATS | STRTAB_STE_1_SHCFG); + used_bits->data[2] |= + cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR | + STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI | + STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2R); + used_bits->data[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK); + } + + if (cfg == STRTAB_STE_0_CFG_BYPASS) + used_bits->data[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); +} + +/* + * Figure out if we can do a hitless update of entry to become target. Returns a + * bit mask where 1 indicates that qword needs to be set disruptively. + * unused_update is an intermediate value of entry that has unused bits set to + * their new values. + */ +static u8 arm_smmu_entry_qword_diff(const struct arm_smmu_ste *entry, + const struct arm_smmu_ste *target, + struct arm_smmu_ste *unused_update) +{ + struct arm_smmu_ste target_used = {}; + struct arm_smmu_ste cur_used = {}; + u8 used_qword_diff = 0; + unsigned int i; + + arm_smmu_get_ste_used(entry, &cur_used); + arm_smmu_get_ste_used(target, &target_used); + + for (i = 0; i != ARRAY_SIZE(target_used.data); i++) { + /* + * Check that masks are up to date, the make functions are not + * allowed to set a bit to 1 if the used function doesn't say it + * is used. + */ + WARN_ON_ONCE(target->data[i] & ~target_used.data[i]); + + /* Bits can change because they are not currently being used */ + unused_update->data[i] = (entry->data[i] & cur_used.data[i]) | + (target->data[i] & ~cur_used.data[i]); + /* + * Each bit indicates that a used bit in a qword needs to be + * changed after unused_update is applied. + */ + if ((unused_update->data[i] & target_used.data[i]) != + target->data[i]) + used_qword_diff |= 1 << i; + } + return used_qword_diff; +} + +static bool entry_set(struct arm_smmu_device *smmu, ioasid_t sid, + struct arm_smmu_ste *entry, + const struct arm_smmu_ste *target, unsigned int start, + unsigned int len) +{ + bool changed = false; + unsigned int i; + + for (i = start; len != 0; len--, i++) { + if (entry->data[i] != target->data[i]) { + WRITE_ONCE(entry->data[i], target->data[i]); + changed = true; + } + } + + if (changed) + arm_smmu_sync_ste_for_sid(smmu, sid); + return changed; +} + +/* + * Update the STE/CD to the target configuration. The transition from the + * current entry to the target entry takes place over multiple steps that + * attempts to make the transition hitless if possible. This function takes care + * not to create a situation where the HW can perceive a corrupted entry. HW is + * only required to have a 64 bit atomicity with stores from the CPU, while + * entries are many 64 bit values big. + * + * The difference between the current value and the target value is analyzed to + * determine which of three updates are required - disruptive, hitless or no + * change. + * + * In the most general disruptive case we can make any update in three steps: + * - Disrupting the entry (V=0) + * - Fill now unused qwords, execpt qword 0 which contains V + * - Make qword 0 have the final value and valid (V=1) with a single 64 + * bit store + * + * However this disrupts the HW while it is happening. There are several + * interesting cases where a STE/CD can be updated without disturbing the HW + * because only a small number of bits are changing (S1DSS, CONFIG, etc) or + * because the used bits don't intersect. We can detect this by calculating how + * many 64 bit values need update after adjusting the unused bits and skip the + * V=0 process. This relies on the IGNORED behavior described in the + * specification. + */ +static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, + struct arm_smmu_ste *entry, + const struct arm_smmu_ste *target) +{ + unsigned int num_entry_qwords = ARRAY_SIZE(target->data); + struct arm_smmu_device *smmu = master->smmu; + struct arm_smmu_ste unused_update; + u8 used_qword_diff; + + used_qword_diff = + arm_smmu_entry_qword_diff(entry, target, &unused_update); + if (hweight8(used_qword_diff) == 1) { + /* + * Only one qword needs its used bits to be changed. This is a + * hitless update, update all bits the current STE is ignoring + * to their new values, then update a single "critical qword" to + * change the STE and finally 0 out any bits that are now unused + * in the target configuration. + */ + unsigned int critical_qword_index = ffs(used_qword_diff) - 1; + + /* + * Skip writing unused bits in the critical qword since we'll be + * writing it in the next step anyways. This can save a sync + * when the only change is in that qword. + */ + unused_update.data[critical_qword_index] = + entry->data[critical_qword_index]; + entry_set(smmu, sid, entry, &unused_update, 0, num_entry_qwords); + entry_set(smmu, sid, entry, target, critical_qword_index, 1); + entry_set(smmu, sid, entry, target, 0, num_entry_qwords); + } else if (used_qword_diff) { + /* + * At least two qwords need their inuse bits to be changed. This + * requires a breaking update, zero the V bit, write all qwords + * but 0, then set qword 0 + */ + unused_update.data[0] = entry->data[0] & (~STRTAB_STE_0_V); + entry_set(smmu, sid, entry, &unused_update, 0, 1); + entry_set(smmu, sid, entry, target, 1, num_entry_qwords - 1); + entry_set(smmu, sid, entry, target, 0, 1); + } else { + /* + * No inuse bit changed. Sanity check that all unused bits are 0 + * in the entry. The target was already sanity checked by + * compute_qword_diff(). + */ + WARN_ON_ONCE( + entry_set(smmu, sid, entry, target, 0, num_entry_qwords)); + } + + /* It's likely that we'll want to use the new STE soon */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { + struct arm_smmu_cmdq_ent + prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + } }; + + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + } +} + static void arm_smmu_sync_cd(struct arm_smmu_master *master, int ssid, bool leaf) { @@ -1254,34 +1450,12 @@ static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, struct arm_smmu_ste *dst) { - /* - * This is hideously complicated, but we only really care about - * three cases at the moment: - * - * 1. Invalid (all zero) -> bypass/fault (init) - * 2. Bypass/fault -> translation/bypass (attach) - * 3. Translation/bypass -> bypass/fault (detach) - * - * Given that we can't update the STE atomically and the SMMU - * doesn't read the thing in a defined order, that leaves us - * with the following maintenance requirements: - * - * 1. Update Config, return (init time STEs aren't live) - * 2. Write everything apart from dword 0, sync, write dword 0, sync - * 3. Update Config, sync - */ - u64 val = le64_to_cpu(dst->data[0]); - bool ste_live = false; + u64 val; struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_ctx_desc_cfg *cd_table = NULL; struct arm_smmu_s2_cfg *s2_cfg = NULL; struct arm_smmu_domain *smmu_domain = master->domain; - struct arm_smmu_cmdq_ent prefetch_cmd = { - .opcode = CMDQ_OP_PREFETCH_CFG, - .prefetch = { - .sid = sid, - }, - }; + struct arm_smmu_ste target = {}; if (smmu_domain) { switch (smmu_domain->stage) { @@ -1296,22 +1470,6 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, } } - if (val & STRTAB_STE_0_V) { - switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { - case STRTAB_STE_0_CFG_BYPASS: - break; - case STRTAB_STE_0_CFG_S1_TRANS: - case STRTAB_STE_0_CFG_S2_TRANS: - ste_live = true; - break; - case STRTAB_STE_0_CFG_ABORT: - BUG_ON(!disable_bypass); - break; - default: - BUG(); /* STE corruption */ - } - } - /* Nuke the existing STE_0 value, as we're going to rewrite it */ val = STRTAB_STE_0_V; @@ -1322,16 +1480,11 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, else val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); - dst->data[0] = cpu_to_le64(val); - dst->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, + target.data[0] = cpu_to_le64(val); + target.data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); - dst->data[2] = 0; /* Nuke the VMID */ - /* - * The SMMU can perform negative caching, so we must sync - * the STE regardless of whether the old value was live. - */ - if (smmu) - arm_smmu_sync_ste_for_sid(smmu, sid); + target.data[2] = 0; /* Nuke the VMID */ + arm_smmu_write_ste(master, sid, dst, &target); return; } @@ -1339,8 +1492,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; - BUG_ON(ste_live); - dst->data[1] = cpu_to_le64( + target.data[1] = cpu_to_le64( FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | @@ -1349,7 +1501,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, if (smmu->features & ARM_SMMU_FEAT_STALLS && !master->stall_enabled) - dst->data[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); + target.data[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); val |= (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | @@ -1358,8 +1510,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, } if (s2_cfg) { - BUG_ON(ste_live); - dst->data[2] = cpu_to_le64( + target.data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, + STRTAB_STE_1_SHCFG_INCOMING)); + target.data[2] = cpu_to_le64( FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | #ifdef __BIG_ENDIAN @@ -1368,23 +1521,17 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2R); - dst->data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); + target.data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); } if (master->ats_enabled) - dst->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS, + target.data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS)); - arm_smmu_sync_ste_for_sid(smmu, sid); - /* See comment in arm_smmu_write_ctx_desc() */ - WRITE_ONCE(dst->data[0], cpu_to_le64(val)); - arm_smmu_sync_ste_for_sid(smmu, sid); - - /* It's likely that we'll want to use the new STE soon */ - if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) - arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + target.data[0] = cpu_to_le64(val); + arm_smmu_write_ste(master, sid, dst, &target); } static void arm_smmu_init_bypass_stes(struct arm_smmu_ste *strtab, From 7686aa5f8d61388eeaa64730363ffc8df20a481d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:13 -0400 Subject: [PATCH 0860/1038] iommu/arm-smmu-v3: Consolidate the STE generation for abort/bypass This allows writing the flow of arm_smmu_write_strtab_ent() around abort and bypass domains more naturally. Note that the core code no longer supplies NULL domains, though there is still a flow in the driver that end up in arm_smmu_write_strtab_ent() with NULL. A later patch will remove it. Remove the duplicate calculation of the STE in arm_smmu_init_bypass_stes() and remove the force parameter. arm_smmu_rmr_install_bypass_ste() can now simply invoke arm_smmu_make_bypass_ste() directly. Rename arm_smmu_init_bypass_stes() to arm_smmu_init_initial_stes() to better reflect its purpose. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/2-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 99 ++++++++++++--------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 9805d989dafd..12ba1b97d696 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1447,6 +1447,24 @@ static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); } +static void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) +{ + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT)); +} + +static void arm_smmu_make_bypass_ste(struct arm_smmu_ste *target) +{ + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS)); + target->data[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); +} + static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, struct arm_smmu_ste *dst) { @@ -1457,37 +1475,31 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, struct arm_smmu_domain *smmu_domain = master->domain; struct arm_smmu_ste target = {}; - if (smmu_domain) { - switch (smmu_domain->stage) { - case ARM_SMMU_DOMAIN_S1: - cd_table = &master->cd_table; - break; - case ARM_SMMU_DOMAIN_S2: - s2_cfg = &smmu_domain->s2_cfg; - break; - default: - break; - } + if (!smmu_domain) { + if (disable_bypass) + arm_smmu_make_abort_ste(&target); + else + arm_smmu_make_bypass_ste(&target); + arm_smmu_write_ste(master, sid, dst, &target); + return; + } + + switch (smmu_domain->stage) { + case ARM_SMMU_DOMAIN_S1: + cd_table = &master->cd_table; + break; + case ARM_SMMU_DOMAIN_S2: + s2_cfg = &smmu_domain->s2_cfg; + break; + case ARM_SMMU_DOMAIN_BYPASS: + arm_smmu_make_bypass_ste(&target); + arm_smmu_write_ste(master, sid, dst, &target); + return; } /* Nuke the existing STE_0 value, as we're going to rewrite it */ val = STRTAB_STE_0_V; - /* Bypass/fault */ - if (!smmu_domain || !(cd_table || s2_cfg)) { - if (!smmu_domain && disable_bypass) - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); - else - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); - - target.data[0] = cpu_to_le64(val); - target.data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, - STRTAB_STE_1_SHCFG_INCOMING)); - target.data[2] = 0; /* Nuke the VMID */ - arm_smmu_write_ste(master, sid, dst, &target); - return; - } - if (cd_table) { u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; @@ -1534,22 +1546,20 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, arm_smmu_write_ste(master, sid, dst, &target); } -static void arm_smmu_init_bypass_stes(struct arm_smmu_ste *strtab, - unsigned int nent, bool force) +/* + * This can safely directly manipulate the STE memory without a sync sequence + * because the STE table has not been installed in the SMMU yet. + */ +static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, + unsigned int nent) { unsigned int i; - u64 val = STRTAB_STE_0_V; - - if (disable_bypass && !force) - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); - else - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); for (i = 0; i < nent; ++i) { - strtab->data[0] = cpu_to_le64(val); - strtab->data[1] = cpu_to_le64(FIELD_PREP( - STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); - strtab->data[2] = 0; + if (disable_bypass) + arm_smmu_make_abort_ste(strtab); + else + arm_smmu_make_bypass_ste(strtab); strtab++; } } @@ -1577,7 +1587,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return -ENOMEM; } - arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false); + arm_smmu_init_initial_stes(desc->l2ptr, 1 << STRTAB_SPLIT); arm_smmu_write_strtab_l1_desc(strtab, desc); return 0; } @@ -3196,7 +3206,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); cfg->strtab_base_cfg = reg; - arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false); + arm_smmu_init_initial_stes(strtab, cfg->num_l1_ents); return 0; } @@ -3907,7 +3917,6 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); list_for_each_entry(e, &rmr_list, list) { - struct arm_smmu_ste *step; struct iommu_iort_rmr_data *rmr; int ret, i; @@ -3920,8 +3929,12 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) continue; } - step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]); - arm_smmu_init_bypass_stes(step, 1, true); + /* + * STE table is not programmed to HW, see + * arm_smmu_initial_bypass_stes() + */ + arm_smmu_make_bypass_ste( + arm_smmu_get_step_for_sid(smmu, rmr->sids[i])); } } From efe15df08727d483bd247ff905a828f0de955de6 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:14 -0400 Subject: [PATCH 0861/1038] iommu/arm-smmu-v3: Move the STE generation for S1 and S2 domains into functions This is preparation to move the STE calculation higher up in to the call chain and remove arm_smmu_write_strtab_ent(). These new functions will be called directly from attach_dev. Reviewed-by: Moritz Fischer Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/3-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 138 ++++++++++++-------- 1 file changed, 83 insertions(+), 55 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 12ba1b97d696..e34c31819669 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1465,13 +1465,89 @@ static void arm_smmu_make_bypass_ste(struct arm_smmu_ste *target) FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); } +static void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master) +{ + struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; + struct arm_smmu_device *smmu = master->smmu; + + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | + FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) | + (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | + FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax)); + + target->data[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | + FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | + ((smmu->features & ARM_SMMU_FEAT_STALLS && + !master->stall_enabled) ? + STRTAB_STE_1_S1STALLD : + 0) | + FIELD_PREP(STRTAB_STE_1_EATS, + master->ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); + + if (smmu->features & ARM_SMMU_FEAT_E2H) { + /* + * To support BTM the streamworld needs to match the + * configuration of the CPU so that the ASID broadcasts are + * properly matched. This means either S/NS-EL2-E2H (hypervisor) + * or NS-EL1 (guest). Since an SVA domain can be installed in a + * PASID this should always use a BTM compatible configuration + * if the HW supports it. + */ + target->data[1] |= cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2)); + } else { + target->data[1] |= cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1)); + + /* + * VMID 0 is reserved for stage-2 bypass EL1 STEs, see + * arm_smmu_domain_alloc_id() + */ + target->data[2] = + cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0)); + } +} + +static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg; + + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS)); + + target->data[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_EATS, + master->ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) | + FIELD_PREP(STRTAB_STE_1_SHCFG, + STRTAB_STE_1_SHCFG_INCOMING)); + + target->data[2] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | + FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | + STRTAB_STE_2_S2AA64 | +#ifdef __BIG_ENDIAN + STRTAB_STE_2_S2ENDI | +#endif + STRTAB_STE_2_S2PTW | + STRTAB_STE_2_S2R); + + target->data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); +} + static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, struct arm_smmu_ste *dst) { - u64 val; - struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_ctx_desc_cfg *cd_table = NULL; - struct arm_smmu_s2_cfg *s2_cfg = NULL; struct arm_smmu_domain *smmu_domain = master->domain; struct arm_smmu_ste target = {}; @@ -1486,63 +1562,15 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: - cd_table = &master->cd_table; + arm_smmu_make_cdtable_ste(&target, master); break; case ARM_SMMU_DOMAIN_S2: - s2_cfg = &smmu_domain->s2_cfg; + arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); break; case ARM_SMMU_DOMAIN_BYPASS: arm_smmu_make_bypass_ste(&target); - arm_smmu_write_ste(master, sid, dst, &target); - return; + break; } - - /* Nuke the existing STE_0 value, as we're going to rewrite it */ - val = STRTAB_STE_0_V; - - if (cd_table) { - u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? - STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; - - target.data[1] = cpu_to_le64( - FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | - FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | - FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | - FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | - FIELD_PREP(STRTAB_STE_1_STRW, strw)); - - if (smmu->features & ARM_SMMU_FEAT_STALLS && - !master->stall_enabled) - target.data[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); - - val |= (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | - FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | - FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax) | - FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt); - } - - if (s2_cfg) { - target.data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, - STRTAB_STE_1_SHCFG_INCOMING)); - target.data[2] = cpu_to_le64( - FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | - FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | -#ifdef __BIG_ENDIAN - STRTAB_STE_2_S2ENDI | -#endif - STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | - STRTAB_STE_2_S2R); - - target.data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); - - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); - } - - if (master->ats_enabled) - target.data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS, - STRTAB_STE_1_EATS_TRANS)); - - target.data[0] = cpu_to_le64(val); arm_smmu_write_ste(master, sid, dst, &target); } From 71b0aa10b18dd589a899449457e5ab7c1da00d01 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:15 -0400 Subject: [PATCH 0862/1038] iommu/arm-smmu-v3: Build the whole STE in arm_smmu_make_s2_domain_ste() Half the code was living in arm_smmu_domain_finalise_s2(), just move it here and take the values directly from the pgtbl_ops instead of storing copies. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/4-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 27 ++++++++++++--------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 -- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index e34c31819669..b81e621a8e59 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1520,6 +1520,11 @@ static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, struct arm_smmu_domain *smmu_domain) { struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg; + const struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg; + typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr = + &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; + u64 vtcr_val; memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( @@ -1532,9 +1537,16 @@ static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); + vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); target->data[2] = cpu_to_le64( FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | - FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | + FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) | STRTAB_STE_2_S2AA64 | #ifdef __BIG_ENDIAN STRTAB_STE_2_S2ENDI | @@ -1542,7 +1554,8 @@ static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2R); - target->data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); + target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr & + STRTAB_STE_3_S2TTB_MASK); } static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, @@ -2302,7 +2315,6 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, int vmid; struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; - typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr; /* Reserve VMID 0 for stage-2 bypass STEs */ vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1, @@ -2310,16 +2322,7 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, if (vmid < 0) return vmid; - vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; cfg->vmid = (u16)vmid; - cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; - cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); return 0; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 65fb388d5173..eb669121f195 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -609,8 +609,6 @@ struct arm_smmu_ctx_desc_cfg { struct arm_smmu_s2_cfg { u16 vmid; - u64 vttbr; - u64 vtcr; }; struct arm_smmu_strtab_cfg { From 9f7c68911579bc15c57d227d021ccd253da2b635 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:16 -0400 Subject: [PATCH 0863/1038] iommu/arm-smmu-v3: Hold arm_smmu_asid_lock during all of attach_dev The BTM support wants to be able to change the ASID of any smmu_domain. When it goes to do this it holds the arm_smmu_asid_lock and iterates over the target domain's devices list. During attach of a S1 domain we must ensure that the devices list and CD are in sync, otherwise we could miss CD updates or a parallel CD update could push an out of date CD. This is pretty complicated, and almost works today because arm_smmu_detach_dev() removes the master from the linked list before working on the CD entries, preventing parallel update of the CD. However, it does have an issue where the CD can remain programed while the domain appears to be unattached. arm_smmu_share_asid() will then not clear any CD entriess and install its own CD entry with the same ASID concurrently. This creates a small race window where the IOMMU can see two ASIDs pointing to different translations. CPU0 CPU1 arm_smmu_attach_dev() arm_smmu_detach_dev() spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_del(&master->domain_head); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); arm_smmu_mmu_notifier_get() arm_smmu_alloc_shared_cd() arm_smmu_share_asid(): // Does nothing due to list_del above arm_smmu_update_ctx_desc_devices() arm_smmu_tlb_inv_asid() arm_smmu_write_ctx_desc() ** Now the ASID is in two CDs with different translation arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); Solve this by wrapping most of the attach flow in the arm_smmu_asid_lock. This locks more than strictly needed to prepare for the next patch which will reorganize the order of the linked list, STE and CD changes. Move arm_smmu_detach_dev() till after we have initialized the domain so the lock can be held for less time. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/5-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 22 ++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index b81e621a8e59..d2fc609fab60 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2586,8 +2586,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -EBUSY; } - arm_smmu_detach_dev(master); - mutex_lock(&smmu_domain->init_mutex); if (!smmu_domain->smmu) { @@ -2602,6 +2600,16 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (ret) return ret; + /* + * Prevent arm_smmu_share_asid() from trying to change the ASID + * of either the old or new domain while we are working on it. + * This allows the STE and the smmu_domain->devices list to + * be inconsistent during this routine. + */ + mutex_lock(&arm_smmu_asid_lock); + + arm_smmu_detach_dev(master); + master->domain = smmu_domain; /* @@ -2627,13 +2635,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } } - /* - * Prevent SVA from concurrently modifying the CD or writing to - * the CD entry - */ - mutex_lock(&arm_smmu_asid_lock); ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd); - mutex_unlock(&arm_smmu_asid_lock); if (ret) { master->domain = NULL; goto out_list_del; @@ -2643,13 +2645,15 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_install_ste_for_dev(master); arm_smmu_enable_ats(master); - return 0; + goto out_unlock; out_list_del: spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_del(&master->domain_head); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); +out_unlock: + mutex_unlock(&arm_smmu_asid_lock); return ret; } From 65547275d76965c3106fbcd4a9244242eb88224c Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:17 -0400 Subject: [PATCH 0864/1038] iommu/arm-smmu-v3: Compute the STE only once for each master Currently arm_smmu_install_ste_for_dev() iterates over every SID and computes from scratch an identical STE. Every SID should have the same STE contents. Turn this inside out so that the STE is supplied by the caller and arm_smmu_install_ste_for_dev() simply installs it to every SID. This is possible now that the STE generation does not inform what sequence should be used to program it. This allows splitting the STE calculation up according to the call site, which following patches will make use of, and removes the confusing NULL domain special case that only supported arm_smmu_detach_dev(). Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/6-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 57 ++++++++------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d2fc609fab60..6cdf075e9a7e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1558,35 +1558,6 @@ static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, STRTAB_STE_3_S2TTB_MASK); } -static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, - struct arm_smmu_ste *dst) -{ - struct arm_smmu_domain *smmu_domain = master->domain; - struct arm_smmu_ste target = {}; - - if (!smmu_domain) { - if (disable_bypass) - arm_smmu_make_abort_ste(&target); - else - arm_smmu_make_bypass_ste(&target); - arm_smmu_write_ste(master, sid, dst, &target); - return; - } - - switch (smmu_domain->stage) { - case ARM_SMMU_DOMAIN_S1: - arm_smmu_make_cdtable_ste(&target, master); - break; - case ARM_SMMU_DOMAIN_S2: - arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); - break; - case ARM_SMMU_DOMAIN_BYPASS: - arm_smmu_make_bypass_ste(&target); - break; - } - arm_smmu_write_ste(master, sid, dst, &target); -} - /* * This can safely directly manipulate the STE memory without a sync sequence * because the STE table has not been installed in the SMMU yet. @@ -2413,7 +2384,8 @@ arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) } } -static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) +static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master, + const struct arm_smmu_ste *target) { int i, j; struct arm_smmu_device *smmu = master->smmu; @@ -2430,7 +2402,7 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) if (j < i) continue; - arm_smmu_write_strtab_ent(master, sid, step); + arm_smmu_write_ste(master, sid, step, target); } } @@ -2537,6 +2509,7 @@ static void arm_smmu_disable_pasid(struct arm_smmu_master *master) static void arm_smmu_detach_dev(struct arm_smmu_master *master) { unsigned long flags; + struct arm_smmu_ste target; struct arm_smmu_domain *smmu_domain = master->domain; if (!smmu_domain) @@ -2550,7 +2523,11 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) master->domain = NULL; master->ats_enabled = false; - arm_smmu_install_ste_for_dev(master); + if (disable_bypass) + arm_smmu_make_abort_ste(&target); + else + arm_smmu_make_bypass_ste(&target); + arm_smmu_install_ste_for_dev(master, &target); /* * Clearing the CD entry isn't strictly required to detach the domain * since the table is uninstalled anyway, but it helps avoid confusion @@ -2565,6 +2542,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; unsigned long flags; + struct arm_smmu_ste target; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -2626,7 +2604,8 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) list_add(&master->domain_head, &smmu_domain->devices); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + switch (smmu_domain->stage) { + case ARM_SMMU_DOMAIN_S1: if (!master->cd_table.cdtab) { ret = arm_smmu_alloc_cd_tables(master); if (ret) { @@ -2640,9 +2619,17 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) master->domain = NULL; goto out_list_del; } - } - arm_smmu_install_ste_for_dev(master); + arm_smmu_make_cdtable_ste(&target, master); + break; + case ARM_SMMU_DOMAIN_S2: + arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); + break; + case ARM_SMMU_DOMAIN_BYPASS: + arm_smmu_make_bypass_ste(&target); + break; + } + arm_smmu_install_ste_for_dev(master, &target); arm_smmu_enable_ats(master); goto out_unlock; From 8c73c32c83ce7f3c31864cb044abd3daefacd996 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:18 -0400 Subject: [PATCH 0865/1038] iommu/arm-smmu-v3: Do not change the STE twice during arm_smmu_attach_dev() This was needed because the STE code required the STE to be in ABORT/BYPASS inorder to program a cdtable or S2 STE. Now that the STE code can automatically handle all transitions we can remove this step from the attach_dev flow. A few small bugs exist because of this: 1) If the core code does BLOCKED -> UNMANAGED with disable_bypass=false then there will be a moment where the STE points at BYPASS. Since this can be done by VFIO/IOMMUFD it is a small security race. 2) If the core code does IDENTITY -> DMA then any IOMMU_RESV_DIRECT regions will temporarily become BLOCKED. We'd like drivers to work in a way that allows IOMMU_RESV_DIRECT to be continuously functional during these transitions. Make arm_smmu_release_device() put the STE back to the correct ABORT/BYPASS setting. Fix a bug where a IOMMU_RESV_DIRECT was ignored on this path. As noted before the reordering of the linked list/STE/CD changes is OK against concurrent arm_smmu_share_asid() because of the arm_smmu_asid_lock. Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Reviewed-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/7-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 6cdf075e9a7e..597a8c5f9658 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2509,7 +2509,6 @@ static void arm_smmu_disable_pasid(struct arm_smmu_master *master) static void arm_smmu_detach_dev(struct arm_smmu_master *master) { unsigned long flags; - struct arm_smmu_ste target; struct arm_smmu_domain *smmu_domain = master->domain; if (!smmu_domain) @@ -2523,11 +2522,6 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) master->domain = NULL; master->ats_enabled = false; - if (disable_bypass) - arm_smmu_make_abort_ste(&target); - else - arm_smmu_make_bypass_ste(&target); - arm_smmu_install_ste_for_dev(master, &target); /* * Clearing the CD entry isn't strictly required to detach the domain * since the table is uninstalled anyway, but it helps avoid confusion @@ -2875,9 +2869,18 @@ err_free_master: static void arm_smmu_release_device(struct device *dev) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); + struct arm_smmu_ste target; if (WARN_ON(arm_smmu_master_sva_enabled(master))) iopf_queue_remove_device(master->smmu->evtq.iopf, dev); + + /* Put the STE back to what arm_smmu_init_strtab() sets */ + if (disable_bypass && !dev->iommu->require_direct) + arm_smmu_make_abort_ste(&target); + else + arm_smmu_make_bypass_ste(&target); + arm_smmu_install_ste_for_dev(master, &target); + arm_smmu_detach_dev(master); arm_smmu_disable_pasid(master); arm_smmu_remove_master(master); From d2e053d73247b68144c7f44d002ebf56acaf2d48 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:19 -0400 Subject: [PATCH 0866/1038] iommu/arm-smmu-v3: Put writing the context descriptor in the right order Get closer to the IOMMU API ideal that changes between domains can be hitless. The ordering for the CD table entry is not entirely clean from this perspective. When switching away from a STE with a CD table programmed in it we should write the new STE first, then clear any old data in the CD entry. If we are programming a CD table for the first time to a STE then the CD entry should be programmed before the STE is loaded. If we are replacing a CD table entry when the STE already points at the CD entry then we just need to do the make/break sequence. Lift this code out of arm_smmu_detach_dev() so it can all be sequenced properly. The only other caller is arm_smmu_release_device() and it is going to free the cdtable anyhow, so it doesn't matter what is in it. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/8-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 597a8c5f9658..ec05743ee208 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2522,14 +2522,6 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) master->domain = NULL; master->ats_enabled = false; - /* - * Clearing the CD entry isn't strictly required to detach the domain - * since the table is uninstalled anyway, but it helps avoid confusion - * in the call to arm_smmu_write_ctx_desc on the next attach (which - * expects the entry to be empty). - */ - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) @@ -2606,6 +2598,17 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) master->domain = NULL; goto out_list_del; } + } else { + /* + * arm_smmu_write_ctx_desc() relies on the entry being + * invalid to work, clear any existing entry. + */ + ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, + NULL); + if (ret) { + master->domain = NULL; + goto out_list_del; + } } ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd); @@ -2615,15 +2618,23 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } arm_smmu_make_cdtable_ste(&target, master); + arm_smmu_install_ste_for_dev(master, &target); break; case ARM_SMMU_DOMAIN_S2: arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); + arm_smmu_install_ste_for_dev(master, &target); + if (master->cd_table.cdtab) + arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, + NULL); break; case ARM_SMMU_DOMAIN_BYPASS: arm_smmu_make_bypass_ste(&target); + arm_smmu_install_ste_for_dev(master, &target); + if (master->cd_table.cdtab) + arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, + NULL); break; } - arm_smmu_install_ste_for_dev(master, &target); arm_smmu_enable_ats(master); goto out_unlock; From d550ddc5b789f258cb5ce3bfe74af6d5383589b5 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:20 -0400 Subject: [PATCH 0867/1038] iommu/arm-smmu-v3: Pass smmu_domain to arm_enable/disable_ats() The caller already has the domain, just pass it in. A following patch will remove master->domain. Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/9-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index ec05743ee208..9d36ddecf2ad 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2421,12 +2421,12 @@ static bool arm_smmu_ats_supported(struct arm_smmu_master *master) return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)); } -static void arm_smmu_enable_ats(struct arm_smmu_master *master) +static void arm_smmu_enable_ats(struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) { size_t stu; struct pci_dev *pdev; struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_domain *smmu_domain = master->domain; /* Don't enable ATS at the endpoint if it's not enabled in the STE */ if (!master->ats_enabled) @@ -2442,10 +2442,9 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master) dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); } -static void arm_smmu_disable_ats(struct arm_smmu_master *master) +static void arm_smmu_disable_ats(struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) { - struct arm_smmu_domain *smmu_domain = master->domain; - if (!master->ats_enabled) return; @@ -2514,7 +2513,7 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) if (!smmu_domain) return; - arm_smmu_disable_ats(master); + arm_smmu_disable_ats(master, smmu_domain); spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_del(&master->domain_head); @@ -2636,7 +2635,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) break; } - arm_smmu_enable_ats(master); + arm_smmu_enable_ats(master, smmu_domain); goto out_unlock; out_list_del: From 1b50017d39f650d78a0066734d6fe05920a8c9e8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:21 -0400 Subject: [PATCH 0868/1038] iommu/arm-smmu-v3: Remove arm_smmu_master->domain Introducing global statics which are of type struct iommu_domain, not struct arm_smmu_domain makes it difficult to retain arm_smmu_master->domain, as it can no longer point to an IDENTITY or BLOCKED domain. The only place that uses the value is arm_smmu_detach_dev(). Change things to work like other drivers and call iommu_get_domain_for_dev() to obtain the current domain. The master->domain is subtly protecting the master->domain_head against being unused as only PAGING domains will set master->domain and only paging domains use the master->domain_head. To make it simple keep the master->domain_head initialized so that the list_del() logic just does nothing for attached non-PAGING domains. Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/10-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 26 ++++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 - 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 9d36ddecf2ad..19a7f0468149 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2507,19 +2507,20 @@ static void arm_smmu_disable_pasid(struct arm_smmu_master *master) static void arm_smmu_detach_dev(struct arm_smmu_master *master) { + struct iommu_domain *domain = iommu_get_domain_for_dev(master->dev); + struct arm_smmu_domain *smmu_domain; unsigned long flags; - struct arm_smmu_domain *smmu_domain = master->domain; - if (!smmu_domain) + if (!domain) return; + smmu_domain = to_smmu_domain(domain); arm_smmu_disable_ats(master, smmu_domain); spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_del(&master->domain_head); + list_del_init(&master->domain_head); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - master->domain = NULL; master->ats_enabled = false; } @@ -2573,8 +2574,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_detach_dev(master); - master->domain = smmu_domain; - /* * The SMMU does not support enabling ATS with bypass. When the STE is * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and @@ -2593,10 +2592,8 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) case ARM_SMMU_DOMAIN_S1: if (!master->cd_table.cdtab) { ret = arm_smmu_alloc_cd_tables(master); - if (ret) { - master->domain = NULL; + if (ret) goto out_list_del; - } } else { /* * arm_smmu_write_ctx_desc() relies on the entry being @@ -2604,17 +2601,13 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) */ ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); - if (ret) { - master->domain = NULL; + if (ret) goto out_list_del; - } } ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd); - if (ret) { - master->domain = NULL; + if (ret) goto out_list_del; - } arm_smmu_make_cdtable_ste(&target, master); arm_smmu_install_ste_for_dev(master, &target); @@ -2640,7 +2633,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) out_list_del: spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_del(&master->domain_head); + list_del_init(&master->domain_head); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); out_unlock: @@ -2841,6 +2834,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) master->dev = dev; master->smmu = smmu; INIT_LIST_HEAD(&master->bonds); + INIT_LIST_HEAD(&master->domain_head); dev_iommu_priv_set(dev, master); ret = arm_smmu_insert_master(smmu, master); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index eb669121f195..6b63ea7dae72 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -695,7 +695,6 @@ struct arm_smmu_stream { struct arm_smmu_master { struct arm_smmu_device *smmu; struct device *dev; - struct arm_smmu_domain *domain; struct list_head domain_head; struct arm_smmu_stream *streams; /* Locked by the iommu core using the group mutex */ From ae91f6552c301e5e8569667e9d5440d5f75a90c4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:22 -0400 Subject: [PATCH 0869/1038] iommu/arm-smmu-v3: Check that the RID domain is S1 in SVA The SVA code only works if the RID domain is a S1 domain and has already installed the cdtable. Originally the check for this was in arm_smmu_sva_bind() but when the op was removed the test didn't get copied over to the new arm_smmu_sva_set_dev_pasid(). Without the test wrong usage usually will hit a WARN_ON() in arm_smmu_write_ctx_desc() due to a missing ctx table. However, the next patches wil change things so that an IDENTITY domain is not a struct arm_smmu_domain and this will get into memory corruption if the struct is wrongly casted. Fail in arm_smmu_sva_set_dev_pasid() if the STE does not have a S1, which is a proxy for the STE having a pointer to the CD table. Write it in a way that will be compatible with the next patches. Fixes: 386fa64fd52b ("arm-smmu-v3/sva: Add SVA domain support") Reported-by: Shameerali Kolothum Thodi Closes: https://lore.kernel.org/linux-iommu/2a828e481416405fb3a4cceb9e075a59@huawei.com/ Tested-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/11-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 4a27fbdb2d84..2610e82c0ecd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -364,7 +364,13 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, struct arm_smmu_bond *bond; struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_domain *smmu_domain; + + if (!(domain->type & __IOMMU_DOMAIN_PAGING)) + return -ENODEV; + smmu_domain = to_smmu_domain(domain); + if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) + return -ENODEV; if (!master || !master->sva_enabled) return -ENODEV; From 12dacfb5b938cdd90ced0109165eee9cb27061d9 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:23 -0400 Subject: [PATCH 0870/1038] iommu/arm-smmu-v3: Add a global static IDENTITY domain Move to the new static global for identity domains. Move all the logic out of arm_smmu_attach_dev into an identity only function. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/12-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 82 +++++++++++++++------ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 - 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 19a7f0468149..842ff8a95baa 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2200,8 +2200,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) return arm_smmu_sva_domain_alloc(); if (type != IOMMU_DOMAIN_UNMANAGED && - type != IOMMU_DOMAIN_DMA && - type != IOMMU_DOMAIN_IDENTITY) + type != IOMMU_DOMAIN_DMA) return NULL; /* @@ -2309,11 +2308,6 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; - if (domain->type == IOMMU_DOMAIN_IDENTITY) { - smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; - return 0; - } - /* Restrict the stage to what we can actually support */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) smmu_domain->stage = ARM_SMMU_DOMAIN_S2; @@ -2511,7 +2505,7 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) struct arm_smmu_domain *smmu_domain; unsigned long flags; - if (!domain) + if (!domain || !(domain->type & __IOMMU_DOMAIN_PAGING)) return; smmu_domain = to_smmu_domain(domain); @@ -2574,15 +2568,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_detach_dev(master); - /* - * The SMMU does not support enabling ATS with bypass. When the STE is - * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and - * Translated transactions are denied as though ATS is disabled for the - * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and - * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). - */ - if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) - master->ats_enabled = arm_smmu_ats_supported(master); + master->ats_enabled = arm_smmu_ats_supported(master); spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_add(&master->domain_head, &smmu_domain->devices); @@ -2619,13 +2605,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); break; - case ARM_SMMU_DOMAIN_BYPASS: - arm_smmu_make_bypass_ste(&target); - arm_smmu_install_ste_for_dev(master, &target); - if (master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, - NULL); - break; } arm_smmu_enable_ats(master, smmu_domain); @@ -2641,6 +2620,60 @@ out_unlock: return ret; } +static int arm_smmu_attach_dev_ste(struct device *dev, + struct arm_smmu_ste *ste) +{ + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + + if (arm_smmu_master_sva_enabled(master)) + return -EBUSY; + + /* + * Do not allow any ASID to be changed while are working on the STE, + * otherwise we could miss invalidations. + */ + mutex_lock(&arm_smmu_asid_lock); + + /* + * The SMMU does not support enabling ATS with bypass/abort. When the + * STE is in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests + * and Translated transactions are denied as though ATS is disabled for + * the stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and + * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). + */ + arm_smmu_detach_dev(master); + + arm_smmu_install_ste_for_dev(master, ste); + mutex_unlock(&arm_smmu_asid_lock); + + /* + * This has to be done after removing the master from the + * arm_smmu_domain->devices to avoid races updating the same context + * descriptor from arm_smmu_share_asid(). + */ + if (master->cd_table.cdtab) + arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); + return 0; +} + +static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, + struct device *dev) +{ + struct arm_smmu_ste ste; + + arm_smmu_make_bypass_ste(&ste); + return arm_smmu_attach_dev_ste(dev, &ste); +} + +static const struct iommu_domain_ops arm_smmu_identity_ops = { + .attach_dev = arm_smmu_attach_dev_identity, +}; + +static struct iommu_domain arm_smmu_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &arm_smmu_identity_ops, +}; + static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) @@ -3030,6 +3063,7 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid) } static struct iommu_ops arm_smmu_ops = { + .identity_domain = &arm_smmu_identity_domain, .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, .probe_device = arm_smmu_probe_device, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 6b63ea7dae72..23baf117e7e4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -712,7 +712,6 @@ struct arm_smmu_master { enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, ARM_SMMU_DOMAIN_S2, - ARM_SMMU_DOMAIN_BYPASS, }; struct arm_smmu_domain { From 352bd64cd8288c5c6808735d52a75809dfef8635 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:24 -0400 Subject: [PATCH 0871/1038] iommu/arm-smmu-v3: Add a global static BLOCKED domain Using the same design as the IDENTITY domain install an STRTAB_STE_0_CFG_ABORT STE. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/13-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 842ff8a95baa..baec827e6ae4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2674,6 +2674,24 @@ static struct iommu_domain arm_smmu_identity_domain = { .ops = &arm_smmu_identity_ops, }; +static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, + struct device *dev) +{ + struct arm_smmu_ste ste; + + arm_smmu_make_abort_ste(&ste); + return arm_smmu_attach_dev_ste(dev, &ste); +} + +static const struct iommu_domain_ops arm_smmu_blocked_ops = { + .attach_dev = arm_smmu_attach_dev_blocked, +}; + +static struct iommu_domain arm_smmu_blocked_domain = { + .type = IOMMU_DOMAIN_BLOCKED, + .ops = &arm_smmu_blocked_ops, +}; + static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) @@ -3064,6 +3082,7 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid) static struct iommu_ops arm_smmu_ops = { .identity_domain = &arm_smmu_identity_domain, + .blocked_domain = &arm_smmu_blocked_domain, .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, .probe_device = arm_smmu_probe_device, From d36464f40f29c984984168ea89e08629bdba41df Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:25 -0400 Subject: [PATCH 0872/1038] iommu/arm-smmu-v3: Use the identity/blocked domain during release Consolidate some more code by having release call arm_smmu_attach_dev_identity/blocked() instead of open coding this. Reviewed-by: Nicolin Chen Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/14-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index baec827e6ae4..1303e9c603fc 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2924,19 +2924,16 @@ err_free_master: static void arm_smmu_release_device(struct device *dev) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); - struct arm_smmu_ste target; if (WARN_ON(arm_smmu_master_sva_enabled(master))) iopf_queue_remove_device(master->smmu->evtq.iopf, dev); /* Put the STE back to what arm_smmu_init_strtab() sets */ if (disable_bypass && !dev->iommu->require_direct) - arm_smmu_make_abort_ste(&target); + arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev); else - arm_smmu_make_bypass_ste(&target); - arm_smmu_install_ste_for_dev(master, &target); + arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev); - arm_smmu_detach_dev(master); arm_smmu_disable_pasid(master); arm_smmu_remove_master(master); if (master->cd_table.cdtab) From d8cd200609cf6a404cda73794f0c8c4fd74c568c Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:26 -0400 Subject: [PATCH 0873/1038] iommu/arm-smmu-v3: Pass arm_smmu_domain and arm_smmu_device to finalize Instead of putting container_of() casts in the internals, use the proper type in this call chain. This makes it easier to check that the two global static domains are not leaking into call chains they should not. Passing the smmu avoids the only caller from having to set it and unset it in the error path. Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/15-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 35 +++++++++++---------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1303e9c603fc..ebd8362c8aa3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -89,6 +89,9 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { { 0, NULL}, }; +static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu); + static void parse_driver_options(struct arm_smmu_device *smmu) { int i = 0; @@ -2242,12 +2245,12 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) kfree(smmu_domain); } -static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, +static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) { int ret; u32 asid; - struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; @@ -2279,11 +2282,11 @@ out_unlock: return ret; } -static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, +static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) { int vmid; - struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; /* Reserve VMID 0 for stage-2 bypass STEs */ @@ -2296,17 +2299,17 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, return 0; } -static int arm_smmu_domain_finalise(struct iommu_domain *domain) +static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu) { int ret; unsigned long ias, oas; enum io_pgtable_fmt fmt; struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable_ops *pgtbl_ops; - int (*finalise_stage_fn)(struct arm_smmu_domain *, - struct io_pgtable_cfg *); - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; + int (*finalise_stage_fn)(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, + struct io_pgtable_cfg *pgtbl_cfg); /* Restrict the stage to what we can actually support */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) @@ -2345,17 +2348,18 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!pgtbl_ops) return -ENOMEM; - domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; - domain->geometry.force_aperture = true; + smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; + smmu_domain->domain.geometry.force_aperture = true; - ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); + ret = finalise_stage_fn(smmu, smmu_domain, &pgtbl_cfg); if (ret < 0) { free_io_pgtable_ops(pgtbl_ops); return ret; } smmu_domain->pgtbl_ops = pgtbl_ops; + smmu_domain->smmu = smmu; return 0; } @@ -2547,10 +2551,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) mutex_lock(&smmu_domain->init_mutex); if (!smmu_domain->smmu) { - smmu_domain->smmu = smmu; - ret = arm_smmu_domain_finalise(domain); - if (ret) - smmu_domain->smmu = NULL; + ret = arm_smmu_domain_finalise(smmu_domain, smmu); } else if (smmu_domain->smmu != smmu) ret = -EINVAL; From 327e10b47ae99f76ac53f0b8b73a0539f390d2d2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 26 Feb 2024 13:07:27 -0400 Subject: [PATCH 0874/1038] iommu/arm-smmu-v3: Convert to domain_alloc_paging() Now that the BLOCKED and IDENTITY behaviors are managed with their own domains change to the domain_alloc_paging() op. For now SVA remains using the old interface, eventually it will get its own op that can pass in the device and mm_struct which will let us have a sane lifetime for the mmu_notifier. Call arm_smmu_domain_finalise() early if dev is available. Tested-by: Shameer Kolothum Tested-by: Nicolin Chen Tested-by: Moritz Fischer Reviewed-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/16-v6-96275f25c39d+2d4-smmuv3_newapi_p1_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 22 ++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index ebd8362c8aa3..b7938f17222b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2197,14 +2197,15 @@ static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { - struct arm_smmu_domain *smmu_domain; if (type == IOMMU_DOMAIN_SVA) return arm_smmu_sva_domain_alloc(); + return ERR_PTR(-EOPNOTSUPP); +} - if (type != IOMMU_DOMAIN_UNMANAGED && - type != IOMMU_DOMAIN_DMA) - return NULL; +static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) +{ + struct arm_smmu_domain *smmu_domain; /* * Allocate the domain and initialise some of its data structures. @@ -2213,13 +2214,23 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) */ smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); if (!smmu_domain) - return NULL; + return ERR_PTR(-ENOMEM); mutex_init(&smmu_domain->init_mutex); INIT_LIST_HEAD(&smmu_domain->devices); spin_lock_init(&smmu_domain->devices_lock); INIT_LIST_HEAD(&smmu_domain->mmu_notifiers); + if (dev) { + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + int ret; + + ret = arm_smmu_domain_finalise(smmu_domain, master->smmu); + if (ret) { + kfree(smmu_domain); + return ERR_PTR(ret); + } + } return &smmu_domain->domain; } @@ -3083,6 +3094,7 @@ static struct iommu_ops arm_smmu_ops = { .blocked_domain = &arm_smmu_blocked_domain, .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, + .domain_alloc_paging = arm_smmu_domain_alloc_paging, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, From 8ec7071385508822892cabc2c0cb74333859407a Mon Sep 17 00:00:00 2001 From: Avadhut Naik Date: Thu, 29 Feb 2024 00:22:45 -0600 Subject: [PATCH 0875/1038] ACPI: APEI: Skip initialization of GHES_ASSIST structures for Machine Check Architecture To support GHES_ASSIST on Machine Check Architecture (MCA) error sources, a set of GHES structures is provided by the system firmware for each MCA error source. Each of these sets consists of a GHES structure for each MCA bank on each logical CPU, with all structures of a set sharing a common Related Source ID, equal to the Source ID of one of the MCA error source structures.[1] On SOCs with large core counts, this typically equates to tens of thousands of GHES_ASSIST structures for MCA under "/sys/bus/platform/drivers/GHES". Support for GHES_ASSIST however, hasn't been implemented in the kernel. As such, the information provided through these structures is not consumed by Linux. Moreover, these GHES_ASSIST structures for MCA, which are supposed to provide supplemental information in context of an error reported by hardware, are setup as independent error sources by the kernel during HEST initialization. Additionally, if the Type field of the Notification structure, associated with these GHES_ASSIST structures for MCA, is set to Polled, the kernel sets up a timer for each individual structure. The duration of the timer is derived from the Poll Interval field of the Notification structure. On SOCs with high core counts, this will result in tens of thousands of timers expiring periodically causing unnecessary preemptions and wastage of CPU cycles. The problem will particularly intensify if Poll Interval duration is not sufficiently high. Since GHES_ASSIST support is not present in kernel, skip initialization of GHES_ASSIST structures for MCA to eliminate their performance impact. [1] ACPI specification 6.5, section 18.7 Signed-off-by: Avadhut Naik Reviewed-by: Yazen Ghannam Reviewed-by: Tony Luck Signed-off-by: Rafael J. Wysocki --- drivers/acpi/apei/hest.c | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/drivers/acpi/apei/hest.c b/drivers/acpi/apei/hest.c index 6aef1ee5e1bd..20d757687e3d 100644 --- a/drivers/acpi/apei/hest.c +++ b/drivers/acpi/apei/hest.c @@ -37,6 +37,20 @@ EXPORT_SYMBOL_GPL(hest_disable); static struct acpi_table_hest *__read_mostly hest_tab; +/* + * Since GHES_ASSIST is not supported, skip initialization of GHES_ASSIST + * structures for MCA. + * During HEST parsing, detected MCA error sources are cached from early + * table entries so that the Flags and Source Id fields from these cached + * values are then referred to in later table entries to determine if the + * encountered GHES_ASSIST structure should be initialized. + */ +static struct { + struct acpi_hest_ia_corrected *cmc; + struct acpi_hest_ia_machine_check *mc; + struct acpi_hest_ia_deferred_check *dmc; +} mces; + static const int hest_esrc_len_tab[ACPI_HEST_TYPE_RESERVED] = { [ACPI_HEST_TYPE_IA32_CHECK] = -1, /* need further calculation */ [ACPI_HEST_TYPE_IA32_CORRECTED_CHECK] = -1, @@ -70,22 +84,54 @@ static int hest_esrc_len(struct acpi_hest_header *hest_hdr) cmc = (struct acpi_hest_ia_corrected *)hest_hdr; len = sizeof(*cmc) + cmc->num_hardware_banks * sizeof(struct acpi_hest_ia_error_bank); + mces.cmc = cmc; } else if (hest_type == ACPI_HEST_TYPE_IA32_CHECK) { struct acpi_hest_ia_machine_check *mc; mc = (struct acpi_hest_ia_machine_check *)hest_hdr; len = sizeof(*mc) + mc->num_hardware_banks * sizeof(struct acpi_hest_ia_error_bank); + mces.mc = mc; } else if (hest_type == ACPI_HEST_TYPE_IA32_DEFERRED_CHECK) { struct acpi_hest_ia_deferred_check *mc; mc = (struct acpi_hest_ia_deferred_check *)hest_hdr; len = sizeof(*mc) + mc->num_hardware_banks * sizeof(struct acpi_hest_ia_error_bank); + mces.dmc = mc; } BUG_ON(len == -1); return len; }; +/* + * GHES and GHESv2 structures share the same format, starting from + * Source Id and ending in Error Status Block Length (inclusive). + */ +static bool is_ghes_assist_struct(struct acpi_hest_header *hest_hdr) +{ + struct acpi_hest_generic *ghes; + u16 related_source_id; + + if (hest_hdr->type != ACPI_HEST_TYPE_GENERIC_ERROR && + hest_hdr->type != ACPI_HEST_TYPE_GENERIC_ERROR_V2) + return false; + + ghes = (struct acpi_hest_generic *)hest_hdr; + related_source_id = ghes->related_source_id; + + if (mces.cmc && mces.cmc->flags & ACPI_HEST_GHES_ASSIST && + related_source_id == mces.cmc->header.source_id) + return true; + if (mces.mc && mces.mc->flags & ACPI_HEST_GHES_ASSIST && + related_source_id == mces.mc->header.source_id) + return true; + if (mces.dmc && mces.dmc->flags & ACPI_HEST_GHES_ASSIST && + related_source_id == mces.dmc->header.source_id) + return true; + + return false; +} + typedef int (*apei_hest_func_t)(struct acpi_hest_header *hest_hdr, void *data); static int apei_hest_parse(apei_hest_func_t func, void *data) @@ -114,6 +160,11 @@ static int apei_hest_parse(apei_hest_func_t func, void *data) return -EINVAL; } + if (is_ghes_assist_struct(hest_hdr)) { + hest_hdr = (void *)hest_hdr + len; + continue; + } + rc = func(hest_hdr, data); if (rc) return rc; From 8164f743326404fbe00a721a12efd86b2a8d74d2 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Tue, 27 Feb 2024 15:39:24 +0800 Subject: [PATCH 0876/1038] cpufreq: amd-pstate: adjust min/max limit perf The min/max limit perf values calculated based on frequency may exceed the reasonable range of perf(highest perf, lowest perf). Signed-off-by: Meng Li Reviewed-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/amd-pstate.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index aa5e57e27d2b..2015c9fcc3c9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -484,12 +484,19 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u32 max_limit_perf, min_limit_perf; + u32 max_limit_perf, min_limit_perf, lowest_perf; struct amd_cpudata *cpudata = policy->driver_data; max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); + lowest_perf = READ_ONCE(cpudata->lowest_perf); + if (min_limit_perf < lowest_perf) + min_limit_perf = lowest_perf; + + if (max_limit_perf < min_limit_perf) + max_limit_perf = min_limit_perf; + WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); WRITE_ONCE(cpudata->max_limit_freq, policy->max); @@ -1387,6 +1394,12 @@ static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); + if (min_limit_perf < min_perf) + min_limit_perf = min_perf; + + if (max_limit_perf < min_limit_perf) + max_limit_perf = min_limit_perf; + WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); From e65095686441e724e5c64bf065d66e189048b4d5 Mon Sep 17 00:00:00 2001 From: Yiwei Lin Date: Sun, 25 Feb 2024 21:00:57 +0800 Subject: [PATCH 0877/1038] Documentation: PM: Fix runtime_pm.rst markdown syntax The '7. Generic subsystem callbacks' should be a section title, but the markdown syntax is not adequate now. Fix it so the chapter can be linked at docs.kernel.org correctly. Signed-off-by: Yiwei Lin [ rjw: Changelog edits ] Signed-off-by: Rafael J. Wysocki --- Documentation/power/runtime_pm.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index 6fa50e4f87ce..5c4e730f38d0 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -734,6 +734,7 @@ out the following operations: for it, respectively. 7. Generic subsystem callbacks +============================== Subsystems may wish to conserve code space by using the set of generic power management callbacks provided by the PM core, defined in From d394abcb12bb1a6f309c1221fdb8e73594ecf1b4 Mon Sep 17 00:00:00 2001 From: Shivnandan Kumar Date: Tue, 27 Feb 2024 14:43:51 +0530 Subject: [PATCH 0878/1038] cpufreq: Limit resolving a frequency to policy min/max Resolving a frequency to an efficient one should not transgress policy->max (which can be set for thermal reason) and policy->min. Currently, there is possibility where scaling_cur_freq can exceed scaling_max_freq when scaling_max_freq is an inefficient frequency. Add a check to ensure that resolving a frequency will respect policy->min/max. Cc: All applicable Fixes: 1f39fa0dccff ("cpufreq: Introducing CPUFREQ_RELATION_E") Signed-off-by: Shivnandan Kumar [ rjw: Whitespace adjustment, changelog edits ] Signed-off-by: Rafael J. Wysocki --- include/linux/cpufreq.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 85908b3a2f24..692ea6e55129 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -1020,6 +1020,18 @@ static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy, efficiencies); } +static inline bool cpufreq_is_in_limits(struct cpufreq_policy *policy, int idx) +{ + unsigned int freq; + + if (idx < 0) + return false; + + freq = policy->freq_table[idx].frequency; + + return freq == clamp_val(freq, policy->min, policy->max); +} + static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) @@ -1053,7 +1065,8 @@ retry: return 0; } - if (idx < 0 && efficiencies) { + /* Limit frequency index to honor policy->min/max */ + if (!cpufreq_is_in_limits(policy, idx) && efficiencies) { efficiencies = false; goto retry; } From a755d0e2d41b9b0c7b4e0bc387c6225e3edba128 Mon Sep 17 00:00:00 2001 From: Qais Yousef Date: Tue, 27 Feb 2024 23:34:52 +0000 Subject: [PATCH 0879/1038] cpufreq: Honour transition_latency over transition_delay_us Some platforms like Arm's Juno can have a high transition latency that can be larger than the 2ms cap introduced. If a driver reports a transition_latency that is higher than the cap, then use it as-is. Update comment s/10/2/ to reflect the new cap of 2ms. Reported-by: Pierre Gondois Signed-off-by: Qais Yousef Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 66cef33c4ec7..926a51cb7e52 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -576,8 +576,17 @@ unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; if (latency) { + unsigned int max_delay_us = 2 * MSEC_PER_SEC; + /* - * For platforms that can change the frequency very fast (< 10 + * If the platform already has high transition_latency, use it + * as-is. + */ + if (latency > max_delay_us) + return latency; + + /* + * For platforms that can change the frequency very fast (< 2 * us), the above formula gives a decent transition delay. But * for platforms where transition_latency is in milliseconds, it * ends up giving unrealistic values. @@ -586,7 +595,7 @@ unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) * a reasonable amount of time after which we should reevaluate * the frequency. */ - return min(latency * LATENCY_MULTIPLIER, (unsigned int)(2 * MSEC_PER_SEC)); + return min(latency * LATENCY_MULTIPLIER, max_delay_us); } return LATENCY_MULTIPLIER; From 9bc4ffd32ef8943f5c5a42c9637cfd04771d021b Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Thu, 29 Feb 2024 12:14:59 +0530 Subject: [PATCH 0880/1038] PM: suspend: Set mem_sleep_current during kernel command line setup psci_init_system_suspend() invokes suspend_set_ops() very early during bootup even before kernel command line for mem_sleep_default is setup. This leads to kernel command line mem_sleep_default=s2idle not working as mem_sleep_current gets changed to deep via suspend_set_ops() and never changes back to s2idle. Set mem_sleep_current along with mem_sleep_default during kernel command line setup as default suspend mode. Fixes: faf7ec4a92c0 ("drivers: firmware: psci: add system suspend support") CC: stable@vger.kernel.org # 5.4+ Signed-off-by: Maulik Shah Signed-off-by: Rafael J. Wysocki --- kernel/power/suspend.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 742eb26618cc..e3ae93bbcb9b 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -192,6 +192,7 @@ static int __init mem_sleep_default_setup(char *str) if (mem_sleep_labels[state] && !strcmp(str, mem_sleep_labels[state])) { mem_sleep_default = state; + mem_sleep_current = state; break; } From adcad5364a692dac85e6e741bdce2a7609449ab8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 29 Feb 2024 16:51:38 +0200 Subject: [PATCH 0881/1038] gpio: of: Make of_gpio_get_count() take firmware node as a parameter Make of_gpio_get_count() take firmware node as a parameter in order to be aligned with other functions and decouple from unused device pointer. The latter helps to create a common fwnode_gpio_count() in the future. While at it, rename to be of_gpio_count() to be aligned with the others. Signed-off-by: Andy Shevchenko Reviewed-by: Linus Walleij Reviewed-by: Mika Westerberg Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-of.c | 14 +++++++------- drivers/gpio/gpiolib-of.h | 6 ++++-- drivers/gpio/gpiolib.c | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index e35a9c7da4ee..cb0cefaec37e 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -68,7 +68,7 @@ static int of_gpio_named_count(const struct device_node *np, /** * of_gpio_spi_cs_get_count() - special GPIO counting for SPI - * @dev: Consuming device + * @np: Consuming device node * @con_id: Function within the GPIO consumer * * Some elder GPIO controllers need special quirks. Currently we handle @@ -78,10 +78,9 @@ static int of_gpio_named_count(const struct device_node *np, * the counting of "cs-gpios" to count "gpios" transparent to the * driver. */ -static int of_gpio_spi_cs_get_count(struct device *dev, const char *con_id) +static int of_gpio_spi_cs_get_count(const struct device_node *np, + const char *con_id) { - struct device_node *np = dev->of_node; - if (!IS_ENABLED(CONFIG_SPI_MASTER)) return 0; if (!con_id || strcmp(con_id, "cs")) @@ -93,13 +92,14 @@ static int of_gpio_spi_cs_get_count(struct device *dev, const char *con_id) return of_gpio_named_count(np, "gpios"); } -int of_gpio_get_count(struct device *dev, const char *con_id) +int of_gpio_count(const struct fwnode_handle *fwnode, const char *con_id) { + const struct device_node *np = to_of_node(fwnode); int ret; char propname[32]; unsigned int i; - ret = of_gpio_spi_cs_get_count(dev, con_id); + ret = of_gpio_spi_cs_get_count(np, con_id); if (ret > 0) return ret; @@ -111,7 +111,7 @@ int of_gpio_get_count(struct device *dev, const char *con_id) snprintf(propname, sizeof(propname), "%s", gpio_suffixes[i]); - ret = of_gpio_named_count(dev->of_node, propname); + ret = of_gpio_named_count(np, propname); if (ret > 0) break; } diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h index 6b3a5347c5d9..16d6ac8cb156 100644 --- a/drivers/gpio/gpiolib-of.h +++ b/drivers/gpio/gpiolib-of.h @@ -9,6 +9,7 @@ #include struct device; +struct fwnode_handle; struct gpio_chip; struct gpio_desc; @@ -21,7 +22,7 @@ struct gpio_desc *of_find_gpio(struct device_node *np, unsigned long *lookupflags); int of_gpiochip_add(struct gpio_chip *gc); void of_gpiochip_remove(struct gpio_chip *gc); -int of_gpio_get_count(struct device *dev, const char *con_id); +int of_gpio_count(const struct fwnode_handle *fwnode, const char *con_id); #else static inline struct gpio_desc *of_find_gpio(struct device_node *np, const char *con_id, @@ -32,7 +33,8 @@ static inline struct gpio_desc *of_find_gpio(struct device_node *np, } static inline int of_gpiochip_add(struct gpio_chip *gc) { return 0; } static inline void of_gpiochip_remove(struct gpio_chip *gc) { } -static inline int of_gpio_get_count(struct device *dev, const char *con_id) +static inline int of_gpio_count(const struct fwnode_handle *fwnode, + const char *con_id) { return 0; } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 3e2cd85673b6..01016b4c4674 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4285,7 +4285,7 @@ int gpiod_count(struct device *dev, const char *con_id) int count = -ENOENT; if (is_of_node(fwnode)) - count = of_gpio_get_count(dev, con_id); + count = of_gpio_count(fwnode, con_id); else if (is_acpi_node(fwnode)) count = acpi_gpio_count(dev, con_id); else if (is_software_node(fwnode)) From 8122c7c625fca4da427e85eac5f3261b6f6daeaa Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 29 Feb 2024 16:51:39 +0200 Subject: [PATCH 0882/1038] gpio: acpi: Make acpi_gpio_count() take firmware node as a parameter Make acpi_gpio_count() take firmware node as a parameter in order to be aligned with other functions and decouple from unused device pointer. The latter helps to create a common fwnode_gpio_count() in the future. Signed-off-by: Andy Shevchenko Reviewed-by: Linus Walleij Reviewed-by: Mika Westerberg Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-acpi.c | 13 ++++++------- drivers/gpio/gpiolib-acpi.h | 5 +++-- drivers/gpio/gpiolib.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 899cd505073e..7f140df40f35 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -1402,17 +1402,17 @@ static int acpi_find_gpio_count(struct acpi_resource *ares, void *data) } /** - * acpi_gpio_count - count the GPIOs associated with a device / function - * @dev: GPIO consumer, can be %NULL for system-global GPIOs + * acpi_gpio_count - count the GPIOs associated with a firmware node / function + * @fwnode: firmware node of the GPIO consumer * @con_id: function within the GPIO consumer * * Return: - * The number of GPIOs associated with a device / function or %-ENOENT, + * The number of GPIOs associated with a firmware node / function or %-ENOENT, * if no GPIO has been assigned to the requested function. */ -int acpi_gpio_count(struct device *dev, const char *con_id) +int acpi_gpio_count(const struct fwnode_handle *fwnode, const char *con_id) { - struct acpi_device *adev = ACPI_COMPANION(dev); + struct acpi_device *adev = to_acpi_device_node(fwnode); const union acpi_object *obj; const struct acpi_gpio_mapping *gm; int count = -ENOENT; @@ -1429,8 +1429,7 @@ int acpi_gpio_count(struct device *dev, const char *con_id) snprintf(propname, sizeof(propname), "%s", gpio_suffixes[i]); - ret = acpi_dev_get_property(adev, propname, ACPI_TYPE_ANY, - &obj); + ret = acpi_dev_get_property(adev, propname, ACPI_TYPE_ANY, &obj); if (ret == 0) { if (obj->type == ACPI_TYPE_LOCAL_REFERENCE) count = 1; diff --git a/drivers/gpio/gpiolib-acpi.h b/drivers/gpio/gpiolib-acpi.h index 0fcd7e14d7f9..7e1c51d04040 100644 --- a/drivers/gpio/gpiolib-acpi.h +++ b/drivers/gpio/gpiolib-acpi.h @@ -33,7 +33,7 @@ struct gpio_desc *acpi_find_gpio(struct fwnode_handle *fwnode, enum gpiod_flags *dflags, unsigned long *lookupflags); -int acpi_gpio_count(struct device *dev, const char *con_id); +int acpi_gpio_count(const struct fwnode_handle *fwnode, const char *con_id); #else static inline void acpi_gpiochip_add(struct gpio_chip *chip) { } static inline void acpi_gpiochip_remove(struct gpio_chip *chip) { } @@ -51,7 +51,8 @@ acpi_find_gpio(struct fwnode_handle *fwnode, const char *con_id, { return ERR_PTR(-ENOENT); } -static inline int acpi_gpio_count(struct device *dev, const char *con_id) +static inline int acpi_gpio_count(const struct fwnode_handle *fwnode, + const char *con_id) { return -ENODEV; } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 01016b4c4674..6ea71b72c19a 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4287,7 +4287,7 @@ int gpiod_count(struct device *dev, const char *con_id) if (is_of_node(fwnode)) count = of_gpio_count(fwnode, con_id); else if (is_acpi_node(fwnode)) - count = acpi_gpio_count(dev, con_id); + count = acpi_gpio_count(fwnode, con_id); else if (is_software_node(fwnode)) count = swnode_gpio_count(fwnode, con_id); From 177f954f56bdfeef8a104a356a566389fefd2f95 Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Wed, 28 Feb 2024 11:00:43 +1030 Subject: [PATCH 0883/1038] dt-bindings: gpio: aspeed,ast2400-gpio: Convert to DT schema Squash warnings such as: ``` arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-galaxy100.dtb: /ahb/apb@1e600000/gpio@1e780000: failed to match any schema with compatible: ['aspeed,ast2400-gpio'] ``` Signed-off-by: Andrew Jeffery Reviewed-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- .../bindings/gpio/aspeed,ast2400-gpio.yaml | 148 ++++++++++++++++++ .../devicetree/bindings/gpio/gpio-aspeed.txt | 39 ----- 2 files changed, 148 insertions(+), 39 deletions(-) create mode 100644 Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml delete mode 100644 Documentation/devicetree/bindings/gpio/gpio-aspeed.txt diff --git a/Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml b/Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml new file mode 100644 index 000000000000..cf11aa7ec8c7 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/aspeed,ast2400-gpio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Aspeed GPIO controller + +maintainers: + - Andrew Jeffery + +properties: + compatible: + enum: + - aspeed,ast2400-gpio + - aspeed,ast2500-gpio + - aspeed,ast2600-gpio + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + description: The clock to use for debounce timings + + gpio-controller: true + gpio-line-names: + minItems: 36 + maxItems: 232 + + gpio-ranges: true + + "#gpio-cells": + const: 2 + + interrupts: + maxItems: 1 + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + ngpios: + minimum: 36 + maximum: 232 + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - "#interrupt-cells" + - gpio-controller + - "#gpio-cells" + +allOf: + - if: + properties: + compatible: + contains: + const: aspeed,ast2400-gpio + then: + properties: + gpio-line-names: + minItems: 220 + maxItems: 220 + ngpios: + const: 220 + - if: + properties: + compatible: + contains: + const: aspeed,ast2500-gpio + then: + properties: + gpio-line-names: + minItems: 232 + maxItems: 232 + ngpios: + const: 232 + - if: + properties: + compatible: + contains: + const: aspeed,ast2600-gpio + then: + properties: + gpio-line-names: + minItems: 36 + maxItems: 208 + ngpios: + enum: [ 36, 208 ] + required: + - ngpios + +additionalProperties: false + +examples: + - | + gpio@1e780000 { + compatible = "aspeed,ast2400-gpio"; + reg = <0x1e780000 0x1000>; + interrupts = <20>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + }; + - | + gpio: gpio@1e780000 { + compatible = "aspeed,ast2500-gpio"; + reg = <0x1e780000 0x200>; + interrupts = <20>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl 0 0 232>; + }; + - | + #include + #include + #include + gpio0: gpio@1e780000 { + compatible = "aspeed,ast2600-gpio"; + reg = <0x1e780000 0x400>; + clocks = <&syscon ASPEED_CLK_APB2>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <2>; + #gpio-cells = <2>; + gpio-controller; + gpio-ranges = <&pinctrl 0 0 208>; + ngpios = <208>; + }; + gpio1: gpio@1e780800 { + compatible = "aspeed,ast2600-gpio"; + reg = <0x1e780800 0x800>; + clocks = <&syscon ASPEED_CLK_APB1>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl 0 208 36>; + ngpios = <36>; + }; diff --git a/Documentation/devicetree/bindings/gpio/gpio-aspeed.txt b/Documentation/devicetree/bindings/gpio/gpio-aspeed.txt deleted file mode 100644 index b2033fc3a71a..000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-aspeed.txt +++ /dev/null @@ -1,39 +0,0 @@ -Aspeed GPIO controller Device Tree Bindings -------------------------------------------- - -Required properties: -- compatible : Either "aspeed,ast2400-gpio", "aspeed,ast2500-gpio", - or "aspeed,ast2600-gpio". - -- #gpio-cells : Should be two - - First cell is the GPIO line number - - Second cell is used to specify optional - parameters (unused) - -- reg : Address and length of the register set for the device -- gpio-controller : Marks the device node as a GPIO controller. -- interrupts : Interrupt specifier (see interrupt bindings for - details) -- interrupt-controller : Mark the GPIO controller as an interrupt-controller - -Optional properties: - -- clocks : A phandle to the clock to use for debounce timings -- ngpios : Number of GPIOs controlled by this controller. Should be set - when there are multiple GPIO controllers on a SoC (ast2600). - -The gpio and interrupt properties are further described in their respective -bindings documentation: - -- Documentation/devicetree/bindings/gpio/gpio.txt -- Documentation/devicetree/bindings/interrupt-controller/interrupts.txt - - Example: - gpio@1e780000 { - #gpio-cells = <2>; - compatible = "aspeed,ast2400-gpio"; - gpio-controller; - interrupts = <20>; - reg = <0x1e780000 0x1000>; - interrupt-controller; - }; From e70e9ecd7cb349d00736c594b8e9bada0a762238 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 15:40:24 +0100 Subject: [PATCH 0884/1038] iommu: constify pointer to bus_type Make pointer to bus_type a pointer to const for code safety. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20240216144027.185959-1-krzysztof.kozlowski@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-priv.h | 5 +++-- drivers/iommu/iommu.c | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h index 2024a2313348..5f731d994803 100644 --- a/drivers/iommu/iommu-priv.h +++ b/drivers/iommu/iommu-priv.h @@ -21,10 +21,11 @@ int iommu_group_replace_domain(struct iommu_group *group, struct iommu_domain *new_domain); int iommu_device_register_bus(struct iommu_device *iommu, - const struct iommu_ops *ops, struct bus_type *bus, + const struct iommu_ops *ops, + const struct bus_type *bus, struct notifier_block *nb); void iommu_device_unregister_bus(struct iommu_device *iommu, - struct bus_type *bus, + const struct bus_type *bus, struct notifier_block *nb); #endif /* __LINUX_IOMMU_PRIV_H */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 35dac3e4629a..fa329bcd60e6 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -289,7 +289,7 @@ EXPORT_SYMBOL_GPL(iommu_device_unregister); #if IS_ENABLED(CONFIG_IOMMUFD_TEST) void iommu_device_unregister_bus(struct iommu_device *iommu, - struct bus_type *bus, + const struct bus_type *bus, struct notifier_block *nb) { bus_unregister_notifier(bus, nb); @@ -303,7 +303,8 @@ EXPORT_SYMBOL_GPL(iommu_device_unregister_bus); * some memory to hold a notifier_block. */ int iommu_device_register_bus(struct iommu_device *iommu, - const struct iommu_ops *ops, struct bus_type *bus, + const struct iommu_ops *ops, + const struct bus_type *bus, struct notifier_block *nb) { int err; From b42a905b6aad40c092cf17f4b295a4c389bc7206 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 15:40:25 +0100 Subject: [PATCH 0885/1038] iommu: constify of_phandle_args in xlate The xlate callbacks are supposed to translate of_phandle_args to proper provider without modifying the of_phandle_args. Make the argument pointer to const for code safety and readability. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216144027.185959-2-krzysztof.kozlowski@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/apple-dart.c | 3 ++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 ++- drivers/iommu/arm/arm-smmu/arm-smmu.c | 3 ++- drivers/iommu/arm/arm-smmu/qcom_iommu.c | 3 ++- drivers/iommu/exynos-iommu.c | 2 +- drivers/iommu/iommu.c | 2 +- drivers/iommu/ipmmu-vmsa.c | 4 ++-- drivers/iommu/msm_iommu.c | 4 ++-- drivers/iommu/mtk_iommu.c | 3 ++- drivers/iommu/mtk_iommu_v1.c | 3 ++- drivers/iommu/rockchip-iommu.c | 2 +- drivers/iommu/sprd-iommu.c | 3 ++- drivers/iommu/sun50i-iommu.c | 2 +- drivers/iommu/tegra-smmu.c | 4 ++-- drivers/iommu/virtio-iommu.c | 3 ++- include/linux/iommu.h | 4 ++-- 16 files changed, 28 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ef3ee95706da..eb1e62cd499a 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -779,7 +779,8 @@ static void apple_dart_domain_free(struct iommu_domain *domain) kfree(dart_domain); } -static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args) +static int apple_dart_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct platform_device *iommu_pdev = of_find_device_by_node(args->np); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 02580364acda..0234ddb39c22 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2704,7 +2704,8 @@ static int arm_smmu_enable_nesting(struct iommu_domain *domain) return ret; } -static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int arm_smmu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); } diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 68b6bc5e7c71..8e5e4ab5fad3 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1551,7 +1551,8 @@ static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, return ret; } -static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int arm_smmu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { u32 mask, fwid = 0; diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index 17a1c163fef6..e079bb7a993e 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -546,7 +546,8 @@ static struct iommu_device *qcom_iommu_probe_device(struct device *dev) return &qcom_iommu->iommu; } -static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int qcom_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct qcom_iommu_dev *qcom_iommu; struct platform_device *iommu_pdev; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 2c6e9094f1e9..d98c9161948a 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1431,7 +1431,7 @@ static void exynos_iommu_release_device(struct device *dev) } static int exynos_iommu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct platform_device *sysmmu = of_find_device_by_node(spec->np); struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index fa329bcd60e6..b0a41f598dcf 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2825,7 +2825,7 @@ void iommu_fwspec_free(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_fwspec_free); -int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) +int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i, new_num; diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index ace1fc4bd34b..cd7219319c8b 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -709,7 +709,7 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, } static int ipmmu_init_platform_device(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *ipmmu_pdev; @@ -773,7 +773,7 @@ static bool ipmmu_device_is_allowed(struct device *dev) } static int ipmmu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { if (!ipmmu_device_is_allowed(dev)) return -ENODEV; diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index f86af9815d6f..989e0869d805 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -598,7 +598,7 @@ static void print_ctx_regs(void __iomem *base, int ctx) static int insert_iommu_master(struct device *dev, struct msm_iommu_dev **iommu, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct msm_iommu_ctx_dev *master = dev_iommu_priv_get(dev); int sid; @@ -626,7 +626,7 @@ static int insert_iommu_master(struct device *dev, } static int qcom_iommu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct msm_iommu_dev *iommu = NULL, *iter; unsigned long flags; diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7abe9e85a570..955de8fb7732 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -957,7 +957,8 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) return group; } -static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *m4updev; diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 25b41222abae..ae16d8238d84 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -398,7 +398,8 @@ static const struct iommu_ops mtk_iommu_v1_ops; * MTK generation one iommu HW only support one iommu domain, and all the client * sharing the same iova address space. */ -static int mtk_iommu_v1_create_mapping(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_v1_create_mapping(struct device *dev, + const struct of_phandle_args *args) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_v1_data *data; diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 2685861c0a12..da79d9f4cf63 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1140,7 +1140,7 @@ static void rk_iommu_release_device(struct device *dev) } static int rk_iommu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_dev; struct rk_iommudata *data; diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 537359f10997..ba53571a8239 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -390,7 +390,8 @@ static struct iommu_device *sprd_iommu_probe_device(struct device *dev) return &sdev->iommu; } -static int sprd_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int sprd_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *pdev; diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index 41484a5a399b..decd52cba998 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -819,7 +819,7 @@ static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) } static int sun50i_iommu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); unsigned id = args->args[0]; diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 310871728ab4..14e525bd0d9b 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -830,7 +830,7 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np) } static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { const struct iommu_ops *ops = smmu->iommu.ops; int err; @@ -959,7 +959,7 @@ static struct iommu_group *tegra_smmu_device_group(struct device *dev) } static int tegra_smmu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); struct tegra_mc *mc = platform_get_drvdata(iommu_pdev); diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 34db37fd9675..04048f64a2c0 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1051,7 +1051,8 @@ static struct iommu_group *viommu_device_group(struct device *dev) return generic_device_group(dev); } -static int viommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int viommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 7cc56cfe98dd..98a958621dcb 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -567,7 +567,7 @@ struct iommu_ops { /* Request/Free a list of reserved regions for a device */ void (*get_resv_regions)(struct device *dev, struct list_head *list); - int (*of_xlate)(struct device *dev, struct of_phandle_args *args); + int (*of_xlate)(struct device *dev, const struct of_phandle_args *args); bool (*is_attach_deferred)(struct device *dev); /* Per device IOMMU features */ @@ -985,7 +985,7 @@ struct iommu_mm_data { int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops); void iommu_fwspec_free(struct device *dev); -int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); +int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids); const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) From 5896e6e39b86c1d820b3ccf5caea9aef40c2eacd Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 15:40:26 +0100 Subject: [PATCH 0886/1038] iommu: constify fwnode in iommu_ops_from_fwnode() Make pointer to fwnode_handle a pointer to const for code safety. Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216144027.185959-3-krzysztof.kozlowski@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 2 +- include/linux/iommu.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index b0a41f598dcf..28b18ed83273 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2774,7 +2774,7 @@ bool iommu_default_passthrough(void) } EXPORT_SYMBOL_GPL(iommu_default_passthrough); -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) { const struct iommu_ops *ops = NULL; struct iommu_device *iommu; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 98a958621dcb..7c8032202457 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -986,7 +986,7 @@ int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops); void iommu_fwspec_free(struct device *dev); int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids); -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode); static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) { @@ -1309,7 +1309,7 @@ static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, } static inline -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) { return NULL; } From b5a1f7513a2fcb1b9e646128ff0cb0dbba5ed6c1 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 16 Feb 2024 15:40:27 +0100 Subject: [PATCH 0887/1038] iommu: re-use local fwnode variable in iommu_ops_from_fwnode() iommu_ops_from_fwnode() stores &iommu_spec->np->fwnode in local variable, so use it to simplify the code (iommu_spec is not changed between these dereferences). Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240216144027.185959-4-krzysztof.kozlowski@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/of_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 719652b60840..3afe0b48a48d 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -29,7 +29,7 @@ static int of_iommu_xlate(struct device *dev, !of_device_is_available(iommu_spec->np)) return -ENODEV; - ret = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops); + ret = iommu_fwspec_init(dev, fwnode, ops); if (ret) return ret; /* From 4b8d18c0c986877fe89ade2214e1e81ad817cef1 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 27 Feb 2024 10:14:34 +0800 Subject: [PATCH 0888/1038] iommu/vt-d: Remove INTEL_IOMMU_BROKEN_GFX_WA Commit 62edf5dc4a524 ("intel-iommu: Restore DMAR_BROKEN_GFX_WA option for broken graphics drivers") was introduced 24 years ago as a temporary workaround for graphics drivers that used physical addresses for DMA and avoided DMA APIs. This workaround was disabled by default. As 24 years have passed, it is expected that graphics driver developers have migrated their drivers to use kernel DMA APIs. Therefore, this workaround is no longer required and could been removed. The Intel iommu driver also provides a "igfx_off" option to turn off the DMA translation for the graphic dedicated IOMMU. Hence, there is really no good reason to keep this config option. Suggested-by: Kevin Tian Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240130060823.57990-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/Kconfig | 11 ----------- drivers/iommu/intel/iommu.c | 4 ---- 2 files changed, 15 deletions(-) diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 012cd2541a68..d2d34eb28d94 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -64,17 +64,6 @@ config INTEL_IOMMU_DEFAULT_ON one is found. If this option is not selected, DMAR support can be enabled by passing intel_iommu=on to the kernel. -config INTEL_IOMMU_BROKEN_GFX_WA - bool "Workaround broken graphics drivers (going away soon)" - depends on BROKEN && X86 - help - Current Graphics drivers tend to use physical address - for DMA and avoid using DMA APIs. Setting this config - option permits the IOMMU driver to set a unity map for - all the OS-visible memory. Hence the driver can continue - to use physical addresses for DMA, at least until this - option is removed in the 2.6.32 kernel. - config INTEL_IOMMU_FLOPPY_WA def_bool y depends on X86 diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 6fb5f6fceea1..fc52fcd786aa 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2660,10 +2660,6 @@ static int __init init_dmars(void) iommu_set_root_entry(iommu); } -#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA - dmar_map_gfx = 0; -#endif - if (!dmar_map_gfx) iommu_identity_mapping |= IDENTMAP_GFX; From 8379054869a0e8d0ebf8f2160ccc778ed6268b96 Mon Sep 17 00:00:00 2001 From: Erick Archer Date: Tue, 27 Feb 2024 10:14:35 +0800 Subject: [PATCH 0889/1038] iommu/vt-d: Use kcalloc() instead of kzalloc() This is an effort to get rid of all multiplications from allocation functions in order to prevent integer overflows [1]. Here the multiplication is obviously safe because DMAR_LATENCY_NUM is the number of latency types defined in the "latency_type" enum. enum latency_type { DMAR_LATENCY_INV_IOTLB = 0, DMAR_LATENCY_INV_DEVTLB, DMAR_LATENCY_INV_IEC, DMAR_LATENCY_PRQ, DMAR_LATENCY_NUM }; However, using kcalloc() is more appropriate [2] and improves readability. This patch has no effect on runtime behavior. Link: https://github.com/KSPP/linux/issues/162 [1] Link: https://www.kernel.org/doc/html/next/process/deprecated.html#open-coded-arithmetic-in-allocator-arguments [2] Signed-off-by: Erick Archer Reviewed-by: Kees Cook Reviewed-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20240211175143.9229-1-erick.archer@gmx.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/perf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c index 94ee70ac38e3..adc4de6bbd88 100644 --- a/drivers/iommu/intel/perf.c +++ b/drivers/iommu/intel/perf.c @@ -33,7 +33,7 @@ int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type) spin_lock_irqsave(&latency_lock, flags); if (!iommu->perf_statistic) { - iommu->perf_statistic = kzalloc(sizeof(*lstat) * DMAR_LATENCY_NUM, + iommu->perf_statistic = kcalloc(DMAR_LATENCY_NUM, sizeof(*lstat), GFP_ATOMIC); if (!iommu->perf_statistic) { ret = -ENOMEM; From 967912a3a51e2378ebfd20c2f480eb974acd0fc1 Mon Sep 17 00:00:00 2001 From: Jingqi Liu Date: Tue, 27 Feb 2024 10:14:36 +0800 Subject: [PATCH 0890/1038] iommu/vt-d: Add the document for Intel IOMMU debugfs This document guides users to dump the Intel IOMMU internals by debugfs. Signed-off-by: Jingqi Liu Link: https://lore.kernel.org/r/20240207090742.23857-1-Jingqi.liu@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- Documentation/ABI/testing/debugfs-intel-iommu | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 Documentation/ABI/testing/debugfs-intel-iommu diff --git a/Documentation/ABI/testing/debugfs-intel-iommu b/Documentation/ABI/testing/debugfs-intel-iommu new file mode 100644 index 000000000000..2ab8464504a9 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-intel-iommu @@ -0,0 +1,276 @@ +What: /sys/kernel/debug/iommu/intel/iommu_regset +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps all the register contents for each IOMMU device. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/iommu_regset + + IOMMU: dmar0 Register Base Address: 26be37000 + + Name Offset Contents + VER 0x00 0x0000000000000010 + GCMD 0x18 0x0000000000000000 + GSTS 0x1c 0x00000000c7000000 + FSTS 0x34 0x0000000000000000 + FECTL 0x38 0x0000000000000000 + + [...] + + IOMMU: dmar1 Register Base Address: fed90000 + + Name Offset Contents + VER 0x00 0x0000000000000010 + GCMD 0x18 0x0000000000000000 + GSTS 0x1c 0x00000000c7000000 + FSTS 0x34 0x0000000000000000 + FECTL 0x38 0x0000000000000000 + + [...] + + IOMMU: dmar2 Register Base Address: fed91000 + + Name Offset Contents + VER 0x00 0x0000000000000010 + GCMD 0x18 0x0000000000000000 + GSTS 0x1c 0x00000000c7000000 + FSTS 0x34 0x0000000000000000 + FECTL 0x38 0x0000000000000000 + + [...] + +What: /sys/kernel/debug/iommu/intel/ir_translation_struct +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps the table entries for Interrupt + remapping and Interrupt posting. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/ir_translation_struct + + Remapped Interrupt supported on IOMMU: dmar0 + IR table address:100900000 + + Entry SrcID DstID Vct IRTE_high IRTE_low + 0 00:0a.0 00000080 24 0000000000040050 000000800024000d + 1 00:0a.0 00000001 ef 0000000000040050 0000000100ef000d + + Remapped Interrupt supported on IOMMU: dmar1 + IR table address:100300000 + Entry SrcID DstID Vct IRTE_high IRTE_low + 0 00:02.0 00000002 26 0000000000040010 000000020026000d + + [...] + + **** + + Posted Interrupt supported on IOMMU: dmar0 + IR table address:100900000 + Entry SrcID PDA_high PDA_low Vct IRTE_high IRTE_low + +What: /sys/kernel/debug/iommu/intel/dmar_translation_struct +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps Intel IOMMU DMA remapping tables, such + as root table, context table, PASID directory and PASID + table entries in debugfs. For legacy mode, it doesn't + support PASID, and hence PASID field is defaulted to + '-1' and other PASID related fields are invalid. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/dmar_translation_struct + + IOMMU dmar1: Root Table Address: 0x103027000 + B.D.F Root_entry + 00:02.0 0x0000000000000000:0x000000010303e001 + + Context_entry + 0x0000000000000102:0x000000010303f005 + + PASID PASID_table_entry + -1 0x0000000000000000:0x0000000000000000:0x0000000000000000 + + IOMMU dmar0: Root Table Address: 0x103028000 + B.D.F Root_entry + 00:0a.0 0x0000000000000000:0x00000001038a7001 + + Context_entry + 0x0000000000000000:0x0000000103220e7d + + PASID PASID_table_entry + 0 0x0000000000000000:0x0000000000800002:0x00000001038a5089 + + [...] + +What: /sys/kernel/debug/iommu/intel/invalidation_queue +Date: December 2023 +Contact: Jingqi Liu +Description: + This file exports invalidation queue internals of each + IOMMU device. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/invalidation_queue + + Invalidation queue on IOMMU: dmar0 + Base: 0x10022e000 Head: 20 Tail: 20 + Index qw0 qw1 qw2 + 0 0000000000000014 0000000000000000 0000000000000000 + 1 0000000200000025 0000000100059c04 0000000000000000 + 2 0000000000000014 0000000000000000 0000000000000000 + + qw3 status + 0000000000000000 0000000000000000 + 0000000000000000 0000000000000000 + 0000000000000000 0000000000000000 + + [...] + + Invalidation queue on IOMMU: dmar1 + Base: 0x10026e000 Head: 32 Tail: 32 + Index qw0 qw1 status + 0 0000000000000004 0000000000000000 0000000000000000 + 1 0000000200000025 0000000100059804 0000000000000000 + 2 0000000000000011 0000000000000000 0000000000000000 + + [...] + +What: /sys/kernel/debug/iommu/intel/dmar_perf_latency +Date: December 2023 +Contact: Jingqi Liu +Description: + This file is used to control and show counts of + execution time ranges for various types per DMAR. + + Firstly, write a value to + /sys/kernel/debug/iommu/intel/dmar_perf_latency + to enable sampling. + + The possible values are as follows: + + * 0 - disable sampling all latency data + + * 1 - enable sampling IOTLB invalidation latency data + + * 2 - enable sampling devTLB invalidation latency data + + * 3 - enable sampling intr entry cache invalidation latency data + + Next, read /sys/kernel/debug/iommu/intel/dmar_perf_latency gives + a snapshot of sampling result of all enabled monitors. + + Examples in Kabylake: + + :: + + 1) Disable sampling all latency data: + + $ sudo echo 0 > /sys/kernel/debug/iommu/intel/dmar_perf_latency + + 2) Enable sampling IOTLB invalidation latency data + + $ sudo echo 1 > /sys/kernel/debug/iommu/intel/dmar_perf_latency + + $ sudo cat /sys/kernel/debug/iommu/intel/dmar_perf_latency + + IOMMU: dmar0 Register Base Address: 26be37000 + <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms + inv_iotlb 0 0 0 0 0 + + 1ms-10ms >=10ms min(us) max(us) average(us) + inv_iotlb 0 0 0 0 0 + + [...] + + IOMMU: dmar2 Register Base Address: fed91000 + <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms + inv_iotlb 0 0 18 0 0 + + 1ms-10ms >=10ms min(us) max(us) average(us) + inv_iotlb 0 0 2 2 2 + + 3) Enable sampling devTLB invalidation latency data + + $ sudo echo 2 > /sys/kernel/debug/iommu/intel/dmar_perf_latency + + $ sudo cat /sys/kernel/debug/iommu/intel/dmar_perf_latency + + IOMMU: dmar0 Register Base Address: 26be37000 + <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms + inv_devtlb 0 0 0 0 0 + + >=10ms min(us) max(us) average(us) + inv_devtlb 0 0 0 0 + + [...] + +What: /sys/kernel/debug/iommu/intel//domain_translation_struct +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps a specified page table of Intel IOMMU + in legacy mode or scalable mode. + + For a device that only supports legacy mode, dump its + page table by the debugfs file in the debugfs device + directory. e.g. + /sys/kernel/debug/iommu/intel/0000:00:02.0/domain_translation_struct. + + For a device that supports scalable mode, dump the + page table of specified pasid by the debugfs file in + the debugfs pasid directory. e.g. + /sys/kernel/debug/iommu/intel/0000:00:02.0/1/domain_translation_struct. + + Examples in Kabylake: + + :: + + 1) Dump the page table of device "0000:00:02.0" that only supports legacy mode. + + $ sudo cat /sys/kernel/debug/iommu/intel/0000:00:02.0/domain_translation_struct + + Device 0000:00:02.0 @0x1017f8000 + IOVA_PFN PML5E PML4E + 0x000000008d800 | 0x0000000000000000 0x00000001017f9003 + 0x000000008d801 | 0x0000000000000000 0x00000001017f9003 + 0x000000008d802 | 0x0000000000000000 0x00000001017f9003 + + PDPE PDE PTE + 0x00000001017fa003 0x00000001017fb003 0x000000008d800003 + 0x00000001017fa003 0x00000001017fb003 0x000000008d801003 + 0x00000001017fa003 0x00000001017fb003 0x000000008d802003 + + [...] + + 2) Dump the page table of device "0000:00:0a.0" with PASID "1" that + supports scalable mode. + + $ sudo cat /sys/kernel/debug/iommu/intel/0000:00:0a.0/1/domain_translation_struct + + Device 0000:00:0a.0 with pasid 1 @0x10c112000 + IOVA_PFN PML5E PML4E + 0x0000000000000 | 0x0000000000000000 0x000000010df93003 + 0x0000000000001 | 0x0000000000000000 0x000000010df93003 + 0x0000000000002 | 0x0000000000000000 0x000000010df93003 + + PDPE PDE PTE + 0x0000000106ae6003 0x0000000104b38003 0x0000000147c00803 + 0x0000000106ae6003 0x0000000104b38003 0x0000000147c01803 + 0x0000000106ae6003 0x0000000104b38003 0x0000000147c02803 + + [...] From b4b1054f6cdd08b040fc5132936a5700c1d2d05b Mon Sep 17 00:00:00 2001 From: Tina Zhang Date: Tue, 27 Feb 2024 10:14:37 +0800 Subject: [PATCH 0891/1038] iommu/vt-d: Remove treatment for revoking PASIDs with pending page faults Commit 2f26e0a9c986 ("iommu/vt-d: Add basic SVM PASID support") added a special treatment to mandate that no page faults may be outstanding for the PASID after intel_svm_unbind_mm() is called, as the PASID will be released and reused after unbind. This is unnecessary anymore as no outstanding page faults have been ensured in the driver's remove_dev_pasid path: - Tear down the pasid entry, which guarantees that new page faults for the PASID will be rejected by the iommu hardware. - All outstanding page faults have been responded to. - All hardware pending faults are drained in intel_drain_pasid_prq(). Remove this unnecessary code. Signed-off-by: Tina Zhang Link: https://lore.kernel.org/r/20240219125723.1645703-2-tina.zhang@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 40edd282903f..a815362c8e60 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -408,13 +408,6 @@ void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid) if (svm->notifier.ops) mmu_notifier_unregister(&svm->notifier, mm); pasid_private_remove(svm->pasid); - /* - * We mandate that no page faults may be outstanding - * for the PASID when intel_svm_unbind_mm() is called. - * If that is not obeyed, subtle errors will happen. - * Let's make them less subtle... - */ - memset(svm, 0x6b, sizeof(*svm)); kfree(svm); } } From cb0b95e56269bb6ca996373a02ff0b6edfa09d32 Mon Sep 17 00:00:00 2001 From: Tina Zhang Date: Tue, 27 Feb 2024 10:14:38 +0800 Subject: [PATCH 0892/1038] iommu/vt-d: Remove initialization for dynamically heap-allocated rcu_head The rcu_head structures allocated dynamically in the heap don't need any initialization. Therefore, remove the init_rcu_head(). Signed-off-by: Tina Zhang Link: https://lore.kernel.org/r/20240219125723.1645703-3-tina.zhang@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index a815362c8e60..a92a9e2239e2 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -360,7 +360,6 @@ static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, sdev->iommu = iommu; sdev->did = FLPT_DEFAULT_DID; sdev->sid = PCI_DEVID(info->bus, info->devfn); - init_rcu_head(&sdev->rcu); if (info->ats_enabled) { sdev->qdep = info->ats_qdep; if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) From 8ca918cbc2522e72faa5c102d02059e53d5128d0 Mon Sep 17 00:00:00 2001 From: Tina Zhang Date: Tue, 27 Feb 2024 10:14:39 +0800 Subject: [PATCH 0893/1038] iommu/vt-d: Merge intel_svm_bind_mm() into its caller intel_svm_set_dev_pasid() is the only caller of intel_svm_bind_mm(). Merge them and remove intel_svm_bind_mm(). No functional change intended. Signed-off-by: Tina Zhang Link: https://lore.kernel.org/r/20240219125723.1645703-4-tina.zhang@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index a92a9e2239e2..1dd56d4eb88c 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -315,10 +315,11 @@ out: return 0; } -static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, - struct iommu_domain *domain, ioasid_t pasid) +static int intel_svm_set_dev_pasid(struct iommu_domain *domain, + struct device *dev, ioasid_t pasid) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; struct mm_struct *mm = domain->mm; struct intel_svm_dev *sdev; struct intel_svm *svm; @@ -796,15 +797,6 @@ out: return ret; } -static int intel_svm_set_dev_pasid(struct iommu_domain *domain, - struct device *dev, ioasid_t pasid) -{ - struct device_domain_info *info = dev_iommu_priv_get(dev); - struct intel_iommu *iommu = info->iommu; - - return intel_svm_bind_mm(iommu, dev, domain, pasid); -} - static void intel_svm_domain_free(struct iommu_domain *domain) { kfree(to_dmar_domain(domain)); From 1a75cc710b956010137b4fe1d1fa3282bfd8f86c Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 27 Feb 2024 10:14:40 +0800 Subject: [PATCH 0894/1038] iommu/vt-d: Use rbtree to track iommu probed devices Use a red-black tree(rbtree) to track devices probed by the driver's probe_device callback. These devices need to be looked up quickly by a source ID when the hardware reports a fault, either recoverable or unrecoverable. Fault reporting paths are critical. Searching a list in this scenario is inefficient, with an algorithm complexity of O(n). An rbtree is a self-balancing binary search tree, offering an average search time complexity of O(log(n)). This significant performance improvement makes rbtrees a better choice. Furthermore, rbtrees are implemented on a per-iommu basis, eliminating the need for global searches and further enhancing efficiency in critical fault paths. The rbtree is protected by a spin lock with interrupts disabled to ensure thread-safe access even within interrupt contexts. Co-developed-by: Huang Jiaqing Signed-off-by: Huang Jiaqing Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240220065939.121116-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 3 +- drivers/iommu/intel/iommu.c | 88 ++++++++++++++++++++++++++++++++++++- drivers/iommu/intel/iommu.h | 8 ++++ 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 23cb80d62a9a..f9b63c2875f7 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1095,7 +1095,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->agaw = agaw; iommu->msagaw = msagaw; iommu->segment = drhd->segment; - + iommu->device_rbtree = RB_ROOT; + spin_lock_init(&iommu->device_rbtree_lock); iommu->node = NUMA_NO_NODE; ver = readl(iommu->reg + DMAR_VER_REG); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index fc52fcd786aa..025d7385cf58 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -97,6 +97,81 @@ static phys_addr_t root_entry_uctp(struct root_entry *re) return re->hi & VTD_PAGE_MASK; } +static int device_rid_cmp_key(const void *key, const struct rb_node *node) +{ + struct device_domain_info *info = + rb_entry(node, struct device_domain_info, node); + const u16 *rid_lhs = key; + + if (*rid_lhs < PCI_DEVID(info->bus, info->devfn)) + return -1; + + if (*rid_lhs > PCI_DEVID(info->bus, info->devfn)) + return 1; + + return 0; +} + +static int device_rid_cmp(struct rb_node *lhs, const struct rb_node *rhs) +{ + struct device_domain_info *info = + rb_entry(lhs, struct device_domain_info, node); + u16 key = PCI_DEVID(info->bus, info->devfn); + + return device_rid_cmp_key(&key, rhs); +} + +/* + * Looks up an IOMMU-probed device using its source ID. + * + * Returns the pointer to the device if there is a match. Otherwise, + * returns NULL. + * + * Note that this helper doesn't guarantee that the device won't be + * released by the iommu subsystem after being returned. The caller + * should use its own synchronization mechanism to avoid the device + * being released during its use if its possibly the case. + */ +struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid) +{ + struct device_domain_info *info = NULL; + struct rb_node *node; + unsigned long flags; + + spin_lock_irqsave(&iommu->device_rbtree_lock, flags); + node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key); + if (node) + info = rb_entry(node, struct device_domain_info, node); + spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); + + return info ? info->dev : NULL; +} + +static int device_rbtree_insert(struct intel_iommu *iommu, + struct device_domain_info *info) +{ + struct rb_node *curr; + unsigned long flags; + + spin_lock_irqsave(&iommu->device_rbtree_lock, flags); + curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp); + spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); + if (WARN_ON(curr)) + return -EEXIST; + + return 0; +} + +static void device_rbtree_remove(struct device_domain_info *info) +{ + struct intel_iommu *iommu = info->iommu; + unsigned long flags; + + spin_lock_irqsave(&iommu->device_rbtree_lock, flags); + rb_erase(&info->node, &iommu->device_rbtree); + spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); +} + /* * This domain is a statically identity mapping domain. * 1. This domain creats a static 1:1 mapping to all usable memory. @@ -4261,25 +4336,34 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) } dev_iommu_priv_set(dev, info); + ret = device_rbtree_insert(iommu, info); + if (ret) + goto free; if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { ret = intel_pasid_alloc_table(dev); if (ret) { dev_err(dev, "PASID table allocation failed\n"); - kfree(info); - return ERR_PTR(ret); + goto clear_rbtree; } } intel_iommu_debugfs_create_dev(info); return &iommu->iommu; +clear_rbtree: + device_rbtree_remove(info); +free: + kfree(info); + + return ERR_PTR(ret); } static void intel_iommu_release_device(struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); + device_rbtree_remove(info); dmar_remove_one_dev_info(dev); intel_pasid_free_table(dev); intel_iommu_debugfs_remove_dev(info); diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index d02f916d8e59..50d1e196db52 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -716,6 +716,11 @@ struct intel_iommu { struct q_inval *qi; /* Queued invalidation info */ u32 iommu_state[MAX_SR_DMAR_REGS]; /* Store iommu states between suspend and resume.*/ + /* rb tree for all probed devices */ + struct rb_root device_rbtree; + /* protect the device_rbtree */ + spinlock_t device_rbtree_lock; + #ifdef CONFIG_IRQ_REMAP struct ir_table *ir_table; /* Interrupt remapping info */ struct irq_domain *ir_domain; @@ -749,6 +754,8 @@ struct device_domain_info { struct intel_iommu *iommu; /* IOMMU used by this device */ struct dmar_domain *domain; /* pointer to domain */ struct pasid_table *pasid_table; /* pasid table */ + /* device tracking node(lookup by PCI RID) */ + struct rb_node node; #ifdef CONFIG_INTEL_IOMMU_DEBUGFS struct dentry *debugfs_dentry; /* pointer to device directory dentry */ #endif @@ -1074,6 +1081,7 @@ void free_pgtable_page(void *vaddr); void iommu_flush_write_buffer(struct intel_iommu *iommu); struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, const struct iommu_user_data *user_data); +struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid); #ifdef CONFIG_INTEL_IOMMU_SVM void intel_svm_check(struct intel_iommu *iommu); From def054b01a867822254e1dda13d587f5c7a99e2a Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 27 Feb 2024 10:14:41 +0800 Subject: [PATCH 0895/1038] iommu/vt-d: Use device rbtree in iopf reporting path The existing I/O page fault handler currently locates the PCI device by calling pci_get_domain_bus_and_slot(). This function searches the list of all PCI devices until the desired device is found. To improve lookup efficiency, replace it with device_rbtree_find() to search the device within the probed device rbtree. The I/O page fault is initiated by the device, which does not have any synchronization mechanism with the software to ensure that the device stays in the probed device tree. Theoretically, a device could be released by the IOMMU subsystem after device_rbtree_find() and before iopf_get_dev_fault_param(), which would cause a use-after-free problem. Add a mutex to synchronize the I/O page fault reporting path and the IOMMU release device path. This lock doesn't introduce any performance overhead, as the conflict between I/O page fault reporting and device releasing is very rare. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240220065939.121116-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 1 + drivers/iommu/intel/iommu.c | 3 +++ drivers/iommu/intel/iommu.h | 2 ++ drivers/iommu/intel/svm.c | 17 +++++++++-------- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index f9b63c2875f7..d14797aabb7a 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1097,6 +1097,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->segment = drhd->segment; iommu->device_rbtree = RB_ROOT; spin_lock_init(&iommu->device_rbtree_lock); + mutex_init(&iommu->iopf_lock); iommu->node = NUMA_NO_NODE; ver = readl(iommu->reg + DMAR_VER_REG); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 025d7385cf58..60aa2dce32ef 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4362,8 +4362,11 @@ free: static void intel_iommu_release_device(struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + mutex_lock(&iommu->iopf_lock); device_rbtree_remove(info); + mutex_unlock(&iommu->iopf_lock); dmar_remove_one_dev_info(dev); intel_pasid_free_table(dev); intel_iommu_debugfs_remove_dev(info); diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 50d1e196db52..a0feab099f12 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -713,6 +713,8 @@ struct intel_iommu { #endif struct iopf_queue *iopf_queue; unsigned char iopfq_name[16]; + /* Synchronization between fault report and iommu device release. */ + struct mutex iopf_lock; struct q_inval *qi; /* Queued invalidation info */ u32 iommu_state[MAX_SR_DMAR_REGS]; /* Store iommu states between suspend and resume.*/ diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 1dd56d4eb88c..bdf3584ca0af 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -643,7 +643,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) struct intel_iommu *iommu = d; struct page_req_dsc *req; int head, tail, handled; - struct pci_dev *pdev; + struct device *dev; u64 address; /* @@ -689,23 +689,24 @@ bad_req: if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) goto prq_advance; - pdev = pci_get_domain_bus_and_slot(iommu->segment, - PCI_BUS_NUM(req->rid), - req->rid & 0xff); /* * If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ - if (!pdev) + mutex_lock(&iommu->iopf_lock); + dev = device_rbtree_find(iommu, req->rid); + if (!dev) { + mutex_unlock(&iommu->iopf_lock); goto bad_req; + } - if (intel_svm_prq_report(iommu, &pdev->dev, req)) + if (intel_svm_prq_report(iommu, dev, req)) handle_bad_prq_event(iommu, req, QI_RESP_INVALID); else - trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, + trace_prq_report(iommu, dev, req->qw_0, req->qw_1, req->priv_data[0], req->priv_data[1], iommu->prq_seq_number++); - pci_dev_put(pdev); + mutex_unlock(&iommu->iopf_lock); prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; } From f304f6b443a7d5910ac1e29094d9eee5fd767868 Mon Sep 17 00:00:00 2001 From: Harshit Mogalapalli Date: Mon, 12 Feb 2024 02:25:04 -0800 Subject: [PATCH 0896/1038] dm vdo volume-index: fix an assert statement in start_restoring_volume_sub_index() Use "==" instead of "=" in ASSERT() statement. Fixes: ef074a31e88e ("dm vdo: implement the volume index") Signed-off-by: Harshit Mogalapalli Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/volume-index.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c index 8731ea1662b1..eebc19fe7d6f 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/volume-index.c @@ -834,7 +834,7 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, decode_u32_le(buffer, &offset, &header.first_list); decode_u32_le(buffer, &offset, &header.list_count); - result = ASSERT(offset = sizeof(buffer), + result = ASSERT(offset == sizeof(buffer), "%zu bytes decoded of %zu expected", offset, sizeof(buffer)); if (result != UDS_SUCCESS) From 50944062f7d2a61c7dc8787563f233823115c249 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 11:04:11 -0600 Subject: [PATCH 0897/1038] dm vdo block-map: rename page state name from "UDS_FREE" to "FREE" Only used for log message, but no need for "UDS_" prefix. Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index cc87e1ac72b6..e3fadb5f2c2d 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -284,7 +284,7 @@ static const char * __must_check get_page_state_name(enum vdo_page_buffer_state { int result; static const char * const state_names[] = { - "UDS_FREE", "INCOMING", "FAILED", "RESIDENT", "DIRTY", "OUTGOING" + "FREE", "INCOMING", "FAILED", "RESIDENT", "DIRTY", "OUTGOING" }; BUILD_BUG_ON(ARRAY_SIZE(state_names) != PAGE_STATE_COUNT); From 9d8741894520ceeac3f45e88e9ef2dacc084041d Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 09:54:53 -0600 Subject: [PATCH 0898/1038] dm vdo: make uds_*_semaphore interface private to uds-threads.c Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/uds-threads.c | 39 +++++++++++++++++++++++++++++++++ drivers/md/dm-vdo/uds-threads.h | 37 ------------------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/drivers/md/dm-vdo/uds-threads.c b/drivers/md/dm-vdo/uds-threads.c index 769c783e342a..33117f68cf36 100644 --- a/drivers/md/dm-vdo/uds-threads.c +++ b/drivers/md/dm-vdo/uds-threads.c @@ -136,10 +136,49 @@ int uds_join_threads(struct thread *thread) return UDS_SUCCESS; } +static inline int __must_check uds_initialize_semaphore(struct semaphore *semaphore, + unsigned int value) +{ + sema_init(semaphore, value); + return UDS_SUCCESS; +} + +static inline int uds_destroy_semaphore(struct semaphore *semaphore) +{ + return UDS_SUCCESS; +} + +static inline void uds_acquire_semaphore(struct semaphore *semaphore) +{ + /* + * Do not use down(semaphore). Instead use down_interruptible so that + * we do not get 120 second stall messages in kern.log. + */ + while (down_interruptible(semaphore) != 0) { + /* + * If we're called from a user-mode process (e.g., "dmsetup + * remove") while waiting for an operation that may take a + * while (e.g., UDS index save), and a signal is sent (SIGINT, + * SIGUSR2), then down_interruptible will not block. If that + * happens, sleep briefly to avoid keeping the CPU locked up in + * this loop. We could just call cond_resched, but then we'd + * still keep consuming CPU time slices and swamp other threads + * trying to do computational work. [VDO-4980] + */ + fsleep(1000); + } +} + +static inline void uds_release_semaphore(struct semaphore *semaphore) +{ + up(semaphore); +} + int uds_initialize_barrier(struct barrier *barrier, unsigned int thread_count) { int result; + /* FIXME: must cleanup, uds_initialize_semaphore never fails! */ result = uds_initialize_semaphore(&barrier->mutex, 1); if (result != UDS_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/uds-threads.h b/drivers/md/dm-vdo/uds-threads.h index 9f3bf7991383..b77a2d46da80 100644 --- a/drivers/md/dm-vdo/uds-threads.h +++ b/drivers/md/dm-vdo/uds-threads.h @@ -74,42 +74,5 @@ static inline void uds_unlock_mutex(struct mutex *mutex) mutex_unlock(mutex); } -static inline int __must_check uds_initialize_semaphore(struct semaphore *semaphore, - unsigned int value) -{ - sema_init(semaphore, value); - return UDS_SUCCESS; -} - -static inline int uds_destroy_semaphore(struct semaphore *semaphore) -{ - return UDS_SUCCESS; -} - -static inline void uds_acquire_semaphore(struct semaphore *semaphore) -{ - /* - * Do not use down(semaphore). Instead use down_interruptible so that - * we do not get 120 second stall messages in kern.log. - */ - while (down_interruptible(semaphore) != 0) { - /* - * If we're called from a user-mode process (e.g., "dmsetup - * remove") while waiting for an operation that may take a - * while (e.g., UDS index save), and a signal is sent (SIGINT, - * SIGUSR2), then down_interruptible will not block. If that - * happens, sleep briefly to avoid keeping the CPU locked up in - * this loop. We could just call cond_resched, but then we'd - * still keep consuming CPU time slices and swamp other threads - * trying to do computational work. [VDO-4980] - */ - fsleep(1000); - } -} - -static inline void uds_release_semaphore(struct semaphore *semaphore) -{ - up(semaphore); -} #endif /* UDS_THREADS_H */ From 2d98aa17806f2a76c1f0dc8cf9f7d40571c80159 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 10:46:04 -0600 Subject: [PATCH 0899/1038] dm vdo uds-threads: eliminate uds_*_semaphore interfaces The implementation of thread 'barrier' data structure does not require overdone private semaphore wrappers. Also rename the barrier structure's 'mutex' member (a semaphore) to 'lock'. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/uds-threads.c | 55 ++++++++------------------------- drivers/md/dm-vdo/uds-threads.h | 4 +-- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/drivers/md/dm-vdo/uds-threads.c b/drivers/md/dm-vdo/uds-threads.c index 33117f68cf36..af6c58eaf449 100644 --- a/drivers/md/dm-vdo/uds-threads.c +++ b/drivers/md/dm-vdo/uds-threads.c @@ -136,19 +136,7 @@ int uds_join_threads(struct thread *thread) return UDS_SUCCESS; } -static inline int __must_check uds_initialize_semaphore(struct semaphore *semaphore, - unsigned int value) -{ - sema_init(semaphore, value); - return UDS_SUCCESS; -} - -static inline int uds_destroy_semaphore(struct semaphore *semaphore) -{ - return UDS_SUCCESS; -} - -static inline void uds_acquire_semaphore(struct semaphore *semaphore) +static inline void __down(struct semaphore *semaphore) { /* * Do not use down(semaphore). Instead use down_interruptible so that @@ -169,53 +157,36 @@ static inline void uds_acquire_semaphore(struct semaphore *semaphore) } } -static inline void uds_release_semaphore(struct semaphore *semaphore) -{ - up(semaphore); -} - int uds_initialize_barrier(struct barrier *barrier, unsigned int thread_count) { - int result; - - /* FIXME: must cleanup, uds_initialize_semaphore never fails! */ - result = uds_initialize_semaphore(&barrier->mutex, 1); - if (result != UDS_SUCCESS) - return result; - + sema_init(&barrier->lock, 1); barrier->arrived = 0; barrier->thread_count = thread_count; - return uds_initialize_semaphore(&barrier->wait, 0); + sema_init(&barrier->wait, 0); + + return UDS_SUCCESS; } int uds_destroy_barrier(struct barrier *barrier) { - int result; - - result = uds_destroy_semaphore(&barrier->mutex); - if (result != UDS_SUCCESS) - return result; - - return uds_destroy_semaphore(&barrier->wait); + return UDS_SUCCESS; } int uds_enter_barrier(struct barrier *barrier) { - bool last_thread; - - uds_acquire_semaphore(&barrier->mutex); - last_thread = (++barrier->arrived == barrier->thread_count); - if (last_thread) { + __down(&barrier->lock); + if (++barrier->arrived == barrier->thread_count) { + /* last thread */ int i; for (i = 1; i < barrier->thread_count; i++) - uds_release_semaphore(&barrier->wait); + up(&barrier->wait); barrier->arrived = 0; - uds_release_semaphore(&barrier->mutex); + up(&barrier->lock); } else { - uds_release_semaphore(&barrier->mutex); - uds_acquire_semaphore(&barrier->wait); + up(&barrier->lock); + __down(&barrier->wait); } return UDS_SUCCESS; diff --git a/drivers/md/dm-vdo/uds-threads.h b/drivers/md/dm-vdo/uds-threads.h index b77a2d46da80..e6fa32af1feb 100644 --- a/drivers/md/dm-vdo/uds-threads.h +++ b/drivers/md/dm-vdo/uds-threads.h @@ -25,8 +25,8 @@ struct cond_var { struct thread; struct barrier { - /* Mutex for this barrier object */ - struct semaphore mutex; + /* Lock for this barrier object */ + struct semaphore lock; /* Semaphore for threads waiting at the barrier */ struct semaphore wait; /* Number of threads which have arrived */ From 0593855a8320e97c3855e57196af24c83e97ef31 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 11:09:14 -0600 Subject: [PATCH 0900/1038] dm vdo uds-threads: push 'barrier' down to sparse-cache The sparse-cache is the only user of the 'barrier' data structure, so just move it private to it. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/sparse-cache.c | 69 +++++++++++++++++++++++++++++++- drivers/md/dm-vdo/uds-threads.c | 56 -------------------------- drivers/md/dm-vdo/uds-threads.h | 15 ------- 3 files changed, 68 insertions(+), 72 deletions(-) diff --git a/drivers/md/dm-vdo/sparse-cache.c b/drivers/md/dm-vdo/sparse-cache.c index 5b41c94f53fa..dcd5ef25b360 100644 --- a/drivers/md/dm-vdo/sparse-cache.c +++ b/drivers/md/dm-vdo/sparse-cache.c @@ -6,6 +6,7 @@ #include "sparse-cache.h" #include +#include #include #include "chapter-index.h" @@ -14,7 +15,6 @@ #include "logger.h" #include "memory-alloc.h" #include "permassert.h" -#include "uds-threads.h" /* * Since the cache is small, it is implemented as a simple array of cache entries. Searching for a @@ -141,6 +141,17 @@ struct search_list { struct cached_chapter_index *entries[]; }; +struct barrier { + /* Lock for this barrier object */ + struct semaphore lock; + /* Semaphore for threads waiting at this barrier */ + struct semaphore wait; + /* Number of threads which have arrived */ + int arrived; + /* Total number of threads using this barrier */ + int thread_count; +}; + struct sparse_cache { const struct index_geometry *geometry; unsigned int capacity; @@ -156,6 +167,62 @@ struct sparse_cache { struct cached_chapter_index chapters[]; }; +static int uds_initialize_barrier(struct barrier *barrier, unsigned int thread_count) +{ + sema_init(&barrier->lock, 1); + barrier->arrived = 0; + barrier->thread_count = thread_count; + sema_init(&barrier->wait, 0); + + return UDS_SUCCESS; +} + +static int uds_destroy_barrier(struct barrier *barrier) +{ + return UDS_SUCCESS; +} + +static inline void __down(struct semaphore *semaphore) +{ + /* + * Do not use down(semaphore). Instead use down_interruptible so that + * we do not get 120 second stall messages in kern.log. + */ + while (down_interruptible(semaphore) != 0) { + /* + * If we're called from a user-mode process (e.g., "dmsetup + * remove") while waiting for an operation that may take a + * while (e.g., UDS index save), and a signal is sent (SIGINT, + * SIGUSR2), then down_interruptible will not block. If that + * happens, sleep briefly to avoid keeping the CPU locked up in + * this loop. We could just call cond_resched, but then we'd + * still keep consuming CPU time slices and swamp other threads + * trying to do computational work. [VDO-4980] + */ + fsleep(1000); + } +} + +static int uds_enter_barrier(struct barrier *barrier) +{ + __down(&barrier->lock); + if (++barrier->arrived == barrier->thread_count) { + /* last thread */ + int i; + + for (i = 1; i < barrier->thread_count; i++) + up(&barrier->wait); + + barrier->arrived = 0; + up(&barrier->lock); + } else { + up(&barrier->lock); + __down(&barrier->wait); + } + + return UDS_SUCCESS; +} + static int __must_check initialize_cached_chapter_index(struct cached_chapter_index *chapter, const struct index_geometry *geometry) { diff --git a/drivers/md/dm-vdo/uds-threads.c b/drivers/md/dm-vdo/uds-threads.c index af6c58eaf449..3bfd22bae8b1 100644 --- a/drivers/md/dm-vdo/uds-threads.c +++ b/drivers/md/dm-vdo/uds-threads.c @@ -135,59 +135,3 @@ int uds_join_threads(struct thread *thread) uds_free(thread); return UDS_SUCCESS; } - -static inline void __down(struct semaphore *semaphore) -{ - /* - * Do not use down(semaphore). Instead use down_interruptible so that - * we do not get 120 second stall messages in kern.log. - */ - while (down_interruptible(semaphore) != 0) { - /* - * If we're called from a user-mode process (e.g., "dmsetup - * remove") while waiting for an operation that may take a - * while (e.g., UDS index save), and a signal is sent (SIGINT, - * SIGUSR2), then down_interruptible will not block. If that - * happens, sleep briefly to avoid keeping the CPU locked up in - * this loop. We could just call cond_resched, but then we'd - * still keep consuming CPU time slices and swamp other threads - * trying to do computational work. [VDO-4980] - */ - fsleep(1000); - } -} - -int uds_initialize_barrier(struct barrier *barrier, unsigned int thread_count) -{ - sema_init(&barrier->lock, 1); - barrier->arrived = 0; - barrier->thread_count = thread_count; - sema_init(&barrier->wait, 0); - - return UDS_SUCCESS; -} - -int uds_destroy_barrier(struct barrier *barrier) -{ - return UDS_SUCCESS; -} - -int uds_enter_barrier(struct barrier *barrier) -{ - __down(&barrier->lock); - if (++barrier->arrived == barrier->thread_count) { - /* last thread */ - int i; - - for (i = 1; i < barrier->thread_count; i++) - up(&barrier->wait); - - barrier->arrived = 0; - up(&barrier->lock); - } else { - up(&barrier->lock); - __down(&barrier->wait); - } - - return UDS_SUCCESS; -} diff --git a/drivers/md/dm-vdo/uds-threads.h b/drivers/md/dm-vdo/uds-threads.h index e6fa32af1feb..2a0580e4482b 100644 --- a/drivers/md/dm-vdo/uds-threads.h +++ b/drivers/md/dm-vdo/uds-threads.h @@ -24,16 +24,6 @@ struct cond_var { struct thread; -struct barrier { - /* Lock for this barrier object */ - struct semaphore lock; - /* Semaphore for threads waiting at the barrier */ - struct semaphore wait; - /* Number of threads which have arrived */ - int arrived; - /* Total number of threads using this barrier */ - int thread_count; -}; int __must_check uds_create_thread(void (*thread_function)(void *), void *thread_data, const char *name, struct thread **new_thread); @@ -42,11 +32,6 @@ void uds_perform_once(atomic_t *once_state, void (*function) (void)); int uds_join_threads(struct thread *thread); -int __must_check uds_initialize_barrier(struct barrier *barrier, - unsigned int thread_count); -int uds_destroy_barrier(struct barrier *barrier); -int uds_enter_barrier(struct barrier *barrier); - int __must_check uds_init_cond(struct cond_var *cond); int uds_signal_cond(struct cond_var *cond); int uds_broadcast_cond(struct cond_var *cond); From eef7cf5e22d93277cf174ae843de37fd7fb082d2 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 11:30:00 -0600 Subject: [PATCH 0901/1038] dm vdo indexer sparse-cache: cleanup threads_barrier code Rename 'barrier' to 'threads_barrier', remove useless uds_destroy_barrier(), return void from remaining methods and clean up uds_make_sparse_cache() accordingly. Also remove uds_ prefix from the 2 remaining threads_barrier functions. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/sparse-cache.c | 60 ++++++++++---------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/drivers/md/dm-vdo/sparse-cache.c b/drivers/md/dm-vdo/sparse-cache.c index dcd5ef25b360..216c8d6256a9 100644 --- a/drivers/md/dm-vdo/sparse-cache.c +++ b/drivers/md/dm-vdo/sparse-cache.c @@ -141,7 +141,7 @@ struct search_list { struct cached_chapter_index *entries[]; }; -struct barrier { +struct threads_barrier { /* Lock for this barrier object */ struct semaphore lock; /* Semaphore for threads waiting at this barrier */ @@ -161,25 +161,19 @@ struct sparse_cache { struct search_list *search_lists[MAX_ZONES]; struct cached_chapter_index **scratch_entries; - struct barrier begin_update_barrier; - struct barrier end_update_barrier; + struct threads_barrier begin_update_barrier; + struct threads_barrier end_update_barrier; struct cached_chapter_index chapters[]; }; -static int uds_initialize_barrier(struct barrier *barrier, unsigned int thread_count) +static void initialize_threads_barrier(struct threads_barrier *barrier, + unsigned int thread_count) { sema_init(&barrier->lock, 1); barrier->arrived = 0; barrier->thread_count = thread_count; sema_init(&barrier->wait, 0); - - return UDS_SUCCESS; -} - -static int uds_destroy_barrier(struct barrier *barrier) -{ - return UDS_SUCCESS; } static inline void __down(struct semaphore *semaphore) @@ -203,7 +197,7 @@ static inline void __down(struct semaphore *semaphore) } } -static int uds_enter_barrier(struct barrier *barrier) +static void enter_threads_barrier(struct threads_barrier *barrier) { __down(&barrier->lock); if (++barrier->arrived == barrier->thread_count) { @@ -219,8 +213,6 @@ static int uds_enter_barrier(struct barrier *barrier) up(&barrier->lock); __down(&barrier->wait); } - - return UDS_SUCCESS; } static int __must_check initialize_cached_chapter_index(struct cached_chapter_index *chapter, @@ -287,44 +279,32 @@ int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int ca */ cache->skip_threshold = (SKIP_SEARCH_THRESHOLD / zone_count); - result = uds_initialize_barrier(&cache->begin_update_barrier, zone_count); - if (result != UDS_SUCCESS) { - uds_free_sparse_cache(cache); - return result; - } - - result = uds_initialize_barrier(&cache->end_update_barrier, zone_count); - if (result != UDS_SUCCESS) { - uds_free_sparse_cache(cache); - return result; - } + initialize_threads_barrier(&cache->begin_update_barrier, zone_count); + initialize_threads_barrier(&cache->end_update_barrier, zone_count); for (i = 0; i < capacity; i++) { result = initialize_cached_chapter_index(&cache->chapters[i], geometry); - if (result != UDS_SUCCESS) { - uds_free_sparse_cache(cache); - return result; - } + if (result != UDS_SUCCESS) + goto out; } for (i = 0; i < zone_count; i++) { result = make_search_list(cache, &cache->search_lists[i]); - if (result != UDS_SUCCESS) { - uds_free_sparse_cache(cache); - return result; - } + if (result != UDS_SUCCESS) + goto out; } /* purge_search_list() needs some temporary lists for sorting. */ result = uds_allocate(capacity * 2, struct cached_chapter_index *, "scratch entries", &cache->scratch_entries); - if (result != UDS_SUCCESS) { - uds_free_sparse_cache(cache); - return result; - } + if (result != UDS_SUCCESS) + goto out; *cache_ptr = cache; return UDS_SUCCESS; +out: + uds_free_sparse_cache(cache); + return result; } static inline void set_skip_search(struct cached_chapter_index *chapter, @@ -381,8 +361,6 @@ void uds_free_sparse_cache(struct sparse_cache *cache) uds_free(cache->chapters[i].page_buffers); } - uds_destroy_barrier(&cache->begin_update_barrier); - uds_destroy_barrier(&cache->end_update_barrier); uds_free(cache); } @@ -525,7 +503,7 @@ int uds_update_sparse_cache(struct index_zone *zone, u64 virtual_chapter) * Wait for every zone thread to reach its corresponding barrier request and invoke this * function before starting to modify the cache. */ - uds_enter_barrier(&cache->begin_update_barrier); + enter_threads_barrier(&cache->begin_update_barrier); /* * This is the start of the critical section: the zone zero thread is captain, effectively @@ -553,7 +531,7 @@ int uds_update_sparse_cache(struct index_zone *zone, u64 virtual_chapter) /* * This is the end of the critical section. All cache invariants must have been restored. */ - uds_enter_barrier(&cache->end_update_barrier); + enter_threads_barrier(&cache->end_update_barrier); return result; } From c2f54aa2b2707d835c00a2a43933dd3882c44580 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 10:10:03 -0600 Subject: [PATCH 0902/1038] dm vdo: rename uds-threads.[ch] to thread-utils.[ch] Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/Makefile | 2 +- drivers/md/dm-vdo/config.c | 2 +- drivers/md/dm-vdo/funnel-requestqueue.c | 2 +- drivers/md/dm-vdo/index-page-map.c | 2 +- drivers/md/dm-vdo/index-session.h | 2 +- drivers/md/dm-vdo/logger.c | 2 +- drivers/md/dm-vdo/status-codes.c | 2 +- drivers/md/dm-vdo/thread-cond-var.c | 2 +- drivers/md/dm-vdo/{uds-threads.c => thread-utils.c} | 2 +- drivers/md/dm-vdo/{uds-threads.h => thread-utils.h} | 5 ++--- drivers/md/dm-vdo/volume-index.c | 2 +- drivers/md/dm-vdo/volume-index.h | 2 +- drivers/md/dm-vdo/volume.c | 2 +- drivers/md/dm-vdo/volume.h | 2 +- 14 files changed, 15 insertions(+), 16 deletions(-) rename drivers/md/dm-vdo/{uds-threads.c => thread-utils.c} (99%) rename drivers/md/dm-vdo/{uds-threads.h => thread-utils.h} (94%) diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 8c06c3b969e3..be5020b81c47 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -51,8 +51,8 @@ dm-vdo-objs := \ thread-cond-var.o \ thread-device.o \ thread-registry.o \ + thread-utils.o \ uds-sysfs.o \ - uds-threads.o \ vdo.o \ vio.o \ volume-index.o \ diff --git a/drivers/md/dm-vdo/config.c b/drivers/md/dm-vdo/config.c index e9c7e9bdbce0..0bf315e7b5d1 100644 --- a/drivers/md/dm-vdo/config.c +++ b/drivers/md/dm-vdo/config.c @@ -9,7 +9,7 @@ #include "memory-alloc.h" #include "numeric.h" #include "string-utils.h" -#include "uds-threads.h" +#include "thread-utils.h" static const u8 INDEX_CONFIG_MAGIC[] = "ALBIC"; static const u8 INDEX_CONFIG_VERSION_6_02[] = "06.02"; diff --git a/drivers/md/dm-vdo/funnel-requestqueue.c b/drivers/md/dm-vdo/funnel-requestqueue.c index c8ba04c1089c..e7a3a4962295 100644 --- a/drivers/md/dm-vdo/funnel-requestqueue.c +++ b/drivers/md/dm-vdo/funnel-requestqueue.c @@ -12,7 +12,7 @@ #include "funnel-queue.h" #include "logger.h" #include "memory-alloc.h" -#include "uds-threads.h" +#include "thread-utils.h" /* * This queue will attempt to handle requests in reasonably sized batches instead of reacting diff --git a/drivers/md/dm-vdo/index-page-map.c b/drivers/md/dm-vdo/index-page-map.c index f3748a915c03..8441f86ef3a4 100644 --- a/drivers/md/dm-vdo/index-page-map.c +++ b/drivers/md/dm-vdo/index-page-map.c @@ -12,7 +12,7 @@ #include "numeric.h" #include "permassert.h" #include "string-utils.h" -#include "uds-threads.h" +#include "thread-utils.h" #include "uds.h" /* diff --git a/drivers/md/dm-vdo/index-session.h b/drivers/md/dm-vdo/index-session.h index c77ee021d510..62a9020dd9fa 100644 --- a/drivers/md/dm-vdo/index-session.h +++ b/drivers/md/dm-vdo/index-session.h @@ -10,7 +10,7 @@ #include #include "config.h" -#include "uds-threads.h" +#include "thread-utils.h" #include "uds.h" /* diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 1efbf8d52f2c..ff1c570f81bf 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -12,7 +12,7 @@ #include #include "thread-device.h" -#include "uds-threads.h" +#include "thread-utils.h" struct priority_name { const char *name; diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index b4d7eb7f94ff..d77bc5e4a99a 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -8,7 +8,7 @@ #include "errors.h" #include "logger.h" #include "permassert.h" -#include "uds-threads.h" +#include "thread-utils.h" const struct error_info vdo_status_list[] = { { "VDO_NOT_IMPLEMENTED", "Not implemented" }, diff --git a/drivers/md/dm-vdo/thread-cond-var.c b/drivers/md/dm-vdo/thread-cond-var.c index ed7f0b79ca0a..82b80338b448 100644 --- a/drivers/md/dm-vdo/thread-cond-var.c +++ b/drivers/md/dm-vdo/thread-cond-var.c @@ -7,8 +7,8 @@ #include #include "errors.h" +#include "thread-utils.h" #include "time-utils.h" -#include "uds-threads.h" int uds_init_cond(struct cond_var *cv) { diff --git a/drivers/md/dm-vdo/uds-threads.c b/drivers/md/dm-vdo/thread-utils.c similarity index 99% rename from drivers/md/dm-vdo/uds-threads.c rename to drivers/md/dm-vdo/thread-utils.c index 3bfd22bae8b1..1a1eb9ae9e33 100644 --- a/drivers/md/dm-vdo/uds-threads.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -3,7 +3,7 @@ * Copyright 2023 Red Hat */ -#include "uds-threads.h" +#include "thread-utils.h" #include #include diff --git a/drivers/md/dm-vdo/uds-threads.h b/drivers/md/dm-vdo/thread-utils.h similarity index 94% rename from drivers/md/dm-vdo/uds-threads.h rename to drivers/md/dm-vdo/thread-utils.h index 2a0580e4482b..30637dd264cc 100644 --- a/drivers/md/dm-vdo/uds-threads.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -3,8 +3,8 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_THREADS_H -#define UDS_THREADS_H +#ifndef THREAD_UTILS_H +#define THREAD_UTILS_H #include #include @@ -14,7 +14,6 @@ #include #include "errors.h" -#include "time-utils.h" /* Thread and synchronization utilities for UDS */ diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c index eebc19fe7d6f..daeafe7691ea 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/volume-index.c @@ -18,8 +18,8 @@ #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" +#include "thread-utils.h" #include "uds.h" -#include "uds-threads.h" /* * The volume index is a combination of two separate subindexes, one containing sparse hook entries diff --git a/drivers/md/dm-vdo/volume-index.h b/drivers/md/dm-vdo/volume-index.h index 537e9947cf4a..2eb2cee7ee58 100644 --- a/drivers/md/dm-vdo/volume-index.h +++ b/drivers/md/dm-vdo/volume-index.h @@ -10,8 +10,8 @@ #include "config.h" #include "delta-index.h" +#include "thread-utils.h" #include "uds.h" -#include "uds-threads.h" /* * The volume index is the primary top-level index for UDS. It contains records which map a record diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 8bd64057c2ca..5b3cb5d89e47 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -20,7 +20,7 @@ #include "permassert.h" #include "sparse-cache.h" #include "string-utils.h" -#include "uds-threads.h" +#include "thread-utils.h" /* * The first block of the volume layout is reserved for the volume header, which is no longer used. diff --git a/drivers/md/dm-vdo/volume.h b/drivers/md/dm-vdo/volume.h index 066680282340..7ef9945d8403 100644 --- a/drivers/md/dm-vdo/volume.h +++ b/drivers/md/dm-vdo/volume.h @@ -19,8 +19,8 @@ #include "permassert.h" #include "radix-sort.h" #include "sparse-cache.h" +#include "thread-utils.h" #include "uds.h" -#include "uds-threads.h" /* * The volume manages deduplication records on permanent storage. The term "volume" can also refer From 8e6333af19830efdd5adbc994f256fcd5f31e7e7 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 12:16:13 -0600 Subject: [PATCH 0903/1038] dm vdo indexer: rename uds.h to indexer.h Also remove unnecessary include from funnel-queue.c. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/chapter-index.c | 2 +- drivers/md/dm-vdo/config.h | 2 +- drivers/md/dm-vdo/data-vio.h | 2 +- drivers/md/dm-vdo/dedupe.c | 2 +- drivers/md/dm-vdo/dedupe.h | 2 +- drivers/md/dm-vdo/delta-index.c | 2 +- drivers/md/dm-vdo/encodings.h | 2 +- drivers/md/dm-vdo/funnel-queue.c | 1 - drivers/md/dm-vdo/funnel-requestqueue.h | 2 +- drivers/md/dm-vdo/geometry.c | 2 +- drivers/md/dm-vdo/geometry.h | 2 +- drivers/md/dm-vdo/hash-utils.h | 2 +- drivers/md/dm-vdo/index-layout.h | 2 +- drivers/md/dm-vdo/index-page-map.c | 2 +- drivers/md/dm-vdo/index-session.h | 2 +- drivers/md/dm-vdo/{uds.h => indexer.h} | 6 +++--- drivers/md/dm-vdo/sparse-cache.h | 2 +- drivers/md/dm-vdo/uds-sysfs.c | 2 +- drivers/md/dm-vdo/vdo.h | 2 +- drivers/md/dm-vdo/volume-index.c | 2 +- drivers/md/dm-vdo/volume-index.h | 2 +- drivers/md/dm-vdo/volume.h | 2 +- 22 files changed, 23 insertions(+), 24 deletions(-) rename drivers/md/dm-vdo/{uds.h => indexer.h} (99%) diff --git a/drivers/md/dm-vdo/chapter-index.c b/drivers/md/dm-vdo/chapter-index.c index 363991d56218..9b9185c2c237 100644 --- a/drivers/md/dm-vdo/chapter-index.c +++ b/drivers/md/dm-vdo/chapter-index.c @@ -7,10 +7,10 @@ #include "errors.h" #include "hash-utils.h" +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "permassert.h" -#include "uds.h" int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, const struct index_geometry *geometry, u64 volume_nonce) diff --git a/drivers/md/dm-vdo/config.h b/drivers/md/dm-vdo/config.h index 7d19863800d6..08507dc2f7a1 100644 --- a/drivers/md/dm-vdo/config.h +++ b/drivers/md/dm-vdo/config.h @@ -7,8 +7,8 @@ #define UDS_CONFIG_H #include "geometry.h" +#include "indexer.h" #include "io-factory.h" -#include "uds.h" /* * The uds_configuration records a variety of parameters used to configure a new UDS index. Some diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h index 78744d064e96..e7729623a6bb 100644 --- a/drivers/md/dm-vdo/data-vio.h +++ b/drivers/md/dm-vdo/data-vio.h @@ -10,8 +10,8 @@ #include #include +#include "indexer.h" #include "permassert.h" -#include "uds.h" #include "block-map.h" #include "completion.h" diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 2a1902c4423c..942a50ef8b0d 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -126,12 +126,12 @@ #include #include +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "string-utils.h" -#include "uds.h" #include "action-manager.h" #include "admin-state.h" diff --git a/drivers/md/dm-vdo/dedupe.h b/drivers/md/dm-vdo/dedupe.h index 773dde5f9365..1566fc972ea7 100644 --- a/drivers/md/dm-vdo/dedupe.h +++ b/drivers/md/dm-vdo/dedupe.h @@ -9,7 +9,7 @@ #include #include -#include "uds.h" +#include "indexer.h" #include "admin-state.h" #include "constants.h" diff --git a/drivers/md/dm-vdo/delta-index.c b/drivers/md/dm-vdo/delta-index.c index 6306777bb202..66f51b5f8fd2 100644 --- a/drivers/md/dm-vdo/delta-index.c +++ b/drivers/md/dm-vdo/delta-index.c @@ -13,13 +13,13 @@ #include "config.h" #include "cpu.h" #include "errors.h" +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "string-utils.h" #include "time-utils.h" -#include "uds.h" /* * The entries in a delta index could be stored in a single delta list, but to reduce search times diff --git a/drivers/md/dm-vdo/encodings.h b/drivers/md/dm-vdo/encodings.h index ba3db9867f4a..18794fd59b0b 100644 --- a/drivers/md/dm-vdo/encodings.h +++ b/drivers/md/dm-vdo/encodings.h @@ -11,8 +11,8 @@ #include #include +#include "indexer.h" #include "numeric.h" -#include "uds.h" #include "constants.h" #include "types.h" diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c index 6940b282086d..d5d96bb38b94 100644 --- a/drivers/md/dm-vdo/funnel-queue.c +++ b/drivers/md/dm-vdo/funnel-queue.c @@ -8,7 +8,6 @@ #include "cpu.h" #include "memory-alloc.h" #include "permassert.h" -#include "uds.h" int uds_make_funnel_queue(struct funnel_queue **queue_ptr) { diff --git a/drivers/md/dm-vdo/funnel-requestqueue.h b/drivers/md/dm-vdo/funnel-requestqueue.h index e74c231fe269..9b0f53939b4d 100644 --- a/drivers/md/dm-vdo/funnel-requestqueue.h +++ b/drivers/md/dm-vdo/funnel-requestqueue.h @@ -6,7 +6,7 @@ #ifndef UDS_REQUEST_QUEUE_H #define UDS_REQUEST_QUEUE_H -#include "uds.h" +#include "indexer.h" /* * A simple request queue which will handle new requests in the order in which they are received, diff --git a/drivers/md/dm-vdo/geometry.c b/drivers/md/dm-vdo/geometry.c index 0e83bba4184a..04c07195a01c 100644 --- a/drivers/md/dm-vdo/geometry.c +++ b/drivers/md/dm-vdo/geometry.c @@ -10,10 +10,10 @@ #include "delta-index.h" #include "errors.h" +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "permassert.h" -#include "uds.h" /* * An index volume is divided into a fixed number of fixed-size chapters, each consisting of a diff --git a/drivers/md/dm-vdo/geometry.h b/drivers/md/dm-vdo/geometry.h index 9a4a66ac2e46..a2ecdb238cf2 100644 --- a/drivers/md/dm-vdo/geometry.h +++ b/drivers/md/dm-vdo/geometry.h @@ -6,7 +6,7 @@ #ifndef UDS_INDEX_GEOMETRY_H #define UDS_INDEX_GEOMETRY_H -#include "uds.h" +#include "indexer.h" /* * The index_geometry records parameters that define the layout of a UDS index volume, and the size and diff --git a/drivers/md/dm-vdo/hash-utils.h b/drivers/md/dm-vdo/hash-utils.h index e22be69695be..e3b865bbe9b2 100644 --- a/drivers/md/dm-vdo/hash-utils.h +++ b/drivers/md/dm-vdo/hash-utils.h @@ -7,8 +7,8 @@ #define UDS_HASH_UTILS_H #include "geometry.h" +#include "indexer.h" #include "numeric.h" -#include "uds.h" /* Utilities for extracting portions of a request name for various uses. */ diff --git a/drivers/md/dm-vdo/index-layout.h b/drivers/md/dm-vdo/index-layout.h index 84a9eb43a49d..e9ac6f4302d6 100644 --- a/drivers/md/dm-vdo/index-layout.h +++ b/drivers/md/dm-vdo/index-layout.h @@ -7,8 +7,8 @@ #define UDS_INDEX_LAYOUT_H #include "config.h" +#include "indexer.h" #include "io-factory.h" -#include "uds.h" /* * The index layout describes the format of the index on the underlying storage, and is responsible diff --git a/drivers/md/dm-vdo/index-page-map.c b/drivers/md/dm-vdo/index-page-map.c index 8441f86ef3a4..1bb12066ad1a 100644 --- a/drivers/md/dm-vdo/index-page-map.c +++ b/drivers/md/dm-vdo/index-page-map.c @@ -7,13 +7,13 @@ #include "errors.h" #include "hash-utils.h" +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "string-utils.h" #include "thread-utils.h" -#include "uds.h" /* * The index page map is conceptually a two-dimensional array indexed by chapter number and index diff --git a/drivers/md/dm-vdo/index-session.h b/drivers/md/dm-vdo/index-session.h index 62a9020dd9fa..733d10f8a56c 100644 --- a/drivers/md/dm-vdo/index-session.h +++ b/drivers/md/dm-vdo/index-session.h @@ -10,8 +10,8 @@ #include #include "config.h" +#include "indexer.h" #include "thread-utils.h" -#include "uds.h" /* * The index session mediates all interactions with a UDS index. Once the index session is created, diff --git a/drivers/md/dm-vdo/uds.h b/drivers/md/dm-vdo/indexer.h similarity index 99% rename from drivers/md/dm-vdo/uds.h rename to drivers/md/dm-vdo/indexer.h index 1264362f8372..59e6a5ca2acb 100644 --- a/drivers/md/dm-vdo/uds.h +++ b/drivers/md/dm-vdo/indexer.h @@ -3,8 +3,8 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_H -#define UDS_H +#ifndef INDEXER_H +#define INDEXER_H #include @@ -326,4 +326,4 @@ int __must_check uds_get_index_session_stats(struct uds_index_session *session, /* This function will fail if any required field of the request is not set. */ int __must_check uds_launch_request(struct uds_request *request); -#endif /* UDS_H */ +#endif /* INDEXER_H */ diff --git a/drivers/md/dm-vdo/sparse-cache.h b/drivers/md/dm-vdo/sparse-cache.h index 90b0be155453..45e2dcf165b5 100644 --- a/drivers/md/dm-vdo/sparse-cache.h +++ b/drivers/md/dm-vdo/sparse-cache.h @@ -7,7 +7,7 @@ #define UDS_SPARSE_CACHE_H #include "geometry.h" -#include "uds.h" +#include "indexer.h" /* * The sparse cache is a cache of entire chapter indexes from sparse chapters used for searching diff --git a/drivers/md/dm-vdo/uds-sysfs.c b/drivers/md/dm-vdo/uds-sysfs.c index eee8a5b7d147..1548092e7de1 100644 --- a/drivers/md/dm-vdo/uds-sysfs.c +++ b/drivers/md/dm-vdo/uds-sysfs.c @@ -9,10 +9,10 @@ #include #include +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "string-utils.h" -#include "uds.h" #define UDS_SYSFS_NAME "uds" diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h index 772317e6db52..3938e519ae6a 100644 --- a/drivers/md/dm-vdo/vdo.h +++ b/drivers/md/dm-vdo/vdo.h @@ -17,12 +17,12 @@ #include "admin-state.h" #include "encodings.h" #include "funnel-workqueue.h" +#include "indexer.h" #include "packer.h" #include "physical-zone.h" #include "statistics.h" #include "thread-registry.h" #include "types.h" -#include "uds.h" enum notifier_state { /* Notifications are allowed but not in progress */ diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c index daeafe7691ea..39c4be06780f 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/volume-index.c @@ -14,12 +14,12 @@ #include "errors.h" #include "geometry.h" #include "hash-utils.h" +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "thread-utils.h" -#include "uds.h" /* * The volume index is a combination of two separate subindexes, one containing sparse hook entries diff --git a/drivers/md/dm-vdo/volume-index.h b/drivers/md/dm-vdo/volume-index.h index 2eb2cee7ee58..66bf14fddc90 100644 --- a/drivers/md/dm-vdo/volume-index.h +++ b/drivers/md/dm-vdo/volume-index.h @@ -10,8 +10,8 @@ #include "config.h" #include "delta-index.h" +#include "indexer.h" #include "thread-utils.h" -#include "uds.h" /* * The volume index is the primary top-level index for UDS. It contains records which map a record diff --git a/drivers/md/dm-vdo/volume.h b/drivers/md/dm-vdo/volume.h index 7ef9945d8403..290de5cbf9ec 100644 --- a/drivers/md/dm-vdo/volume.h +++ b/drivers/md/dm-vdo/volume.h @@ -14,13 +14,13 @@ #include "chapter-index.h" #include "config.h" #include "geometry.h" +#include "indexer.h" #include "index-layout.h" #include "index-page-map.h" #include "permassert.h" #include "radix-sort.h" #include "sparse-cache.h" #include "thread-utils.h" -#include "uds.h" /* * The volume manages deduplication records on permanent storage. The term "volume" can also refer From 877f36b76485d1e0dab5a9de8a7175c66cc791e3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 09:54:53 -0600 Subject: [PATCH 0904/1038] dm vdo: fold thread-cond-var.c into thread-utils Further cleanup is needed for thread-utils interfaces given many functions should return void or be removed entirely because they amount to obfuscation via wrappers. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/Makefile | 1 - drivers/md/dm-vdo/thread-cond-var.c | 46 ----------------------------- drivers/md/dm-vdo/thread-utils.c | 11 +++++++ drivers/md/dm-vdo/thread-utils.h | 28 ++++++++++++++---- 4 files changed, 34 insertions(+), 52 deletions(-) delete mode 100644 drivers/md/dm-vdo/thread-cond-var.c diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index be5020b81c47..32266ab04cc1 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -48,7 +48,6 @@ dm-vdo-objs := \ status-codes.o \ string-utils.o \ sysfs.o \ - thread-cond-var.o \ thread-device.o \ thread-registry.o \ thread-utils.o \ diff --git a/drivers/md/dm-vdo/thread-cond-var.c b/drivers/md/dm-vdo/thread-cond-var.c deleted file mode 100644 index 82b80338b448..000000000000 --- a/drivers/md/dm-vdo/thread-cond-var.c +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -#include -#include - -#include "errors.h" -#include "thread-utils.h" -#include "time-utils.h" - -int uds_init_cond(struct cond_var *cv) -{ - init_waitqueue_head(&cv->wait_queue); - return UDS_SUCCESS; -} - -int uds_signal_cond(struct cond_var *cv) -{ - wake_up(&cv->wait_queue); - return UDS_SUCCESS; -} - -int uds_broadcast_cond(struct cond_var *cv) -{ - wake_up_all(&cv->wait_queue); - return UDS_SUCCESS; -} - -int uds_wait_cond(struct cond_var *cv, struct mutex *mutex) -{ - DEFINE_WAIT(__wait); - - prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); - uds_unlock_mutex(mutex); - schedule(); - finish_wait(&cv->wait_queue, &__wait); - uds_lock_mutex(mutex); - return UDS_SUCCESS; -} - -int uds_destroy_cond(struct cond_var *cv) -{ - return UDS_SUCCESS; -} diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index 1a1eb9ae9e33..5d371bfba8ff 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -135,3 +135,14 @@ int uds_join_threads(struct thread *thread) uds_free(thread); return UDS_SUCCESS; } + +void uds_wait_cond(struct cond_var *cv, struct mutex *mutex) +{ + DEFINE_WAIT(__wait); + + prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); + uds_unlock_mutex(mutex); + schedule(); + finish_wait(&cv->wait_queue, &__wait); + uds_lock_mutex(mutex); +} diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h index 30637dd264cc..c7a5d2d948a4 100644 --- a/drivers/md/dm-vdo/thread-utils.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -31,11 +31,29 @@ void uds_perform_once(atomic_t *once_state, void (*function) (void)); int uds_join_threads(struct thread *thread); -int __must_check uds_init_cond(struct cond_var *cond); -int uds_signal_cond(struct cond_var *cond); -int uds_broadcast_cond(struct cond_var *cond); -int uds_wait_cond(struct cond_var *cond, struct mutex *mutex); -int uds_destroy_cond(struct cond_var *cond); +static inline int __must_check uds_init_cond(struct cond_var *cv) +{ + init_waitqueue_head(&cv->wait_queue); + return UDS_SUCCESS; +} + +static inline void uds_signal_cond(struct cond_var *cv) +{ + wake_up(&cv->wait_queue); +} + +static inline void uds_broadcast_cond(struct cond_var *cv) +{ + wake_up_all(&cv->wait_queue); +} + +void uds_wait_cond(struct cond_var *cv, struct mutex *mutex); + +/* FIXME: all below wrappers should be removed! */ + +static inline void uds_destroy_cond(struct cond_var *cv) +{ +} static inline int __must_check uds_init_mutex(struct mutex *mutex) { From 7f2e494ddda6d66214902c31091cc39b30a9b1f6 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 13:06:00 -0600 Subject: [PATCH 0905/1038] dm vdo thread-utils: push uds_*_cond interface down to indexer Only used by indexer components. Also return void from uds_init_cond(), remove uds_destroy_cond(), and fix up all callers. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/index-session.c | 32 +++++++++++++------------------ drivers/md/dm-vdo/index.c | 8 +------- drivers/md/dm-vdo/indexer.h | 24 +++++++++++++++++++++++ drivers/md/dm-vdo/thread-utils.c | 12 ------------ drivers/md/dm-vdo/thread-utils.h | 28 --------------------------- drivers/md/dm-vdo/volume.c | 15 ++------------- 6 files changed, 40 insertions(+), 79 deletions(-) diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c index 7afc19748712..4837621c16db 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/index-session.c @@ -230,36 +230,21 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde return result; } - result = uds_init_cond(&session->request_cond); - if (result != UDS_SUCCESS) { - uds_destroy_mutex(&session->request_mutex); - uds_free(session); - return result; - } + uds_init_cond(&session->request_cond); result = uds_init_mutex(&session->load_context.mutex); if (result != UDS_SUCCESS) { - uds_destroy_cond(&session->request_cond); uds_destroy_mutex(&session->request_mutex); uds_free(session); return result; } - result = uds_init_cond(&session->load_context.cond); - if (result != UDS_SUCCESS) { - uds_destroy_mutex(&session->load_context.mutex); - uds_destroy_cond(&session->request_cond); - uds_destroy_mutex(&session->request_mutex); - uds_free(session); - return result; - } + uds_init_cond(&session->load_context.cond); result = uds_make_request_queue("callbackW", &handle_callbacks, &session->callback_queue); if (result != UDS_SUCCESS) { - uds_destroy_cond(&session->load_context.cond); uds_destroy_mutex(&session->load_context.mutex); - uds_destroy_cond(&session->request_cond); uds_destroy_mutex(&session->request_mutex); uds_free(session); return result; @@ -700,9 +685,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) result = save_and_free_index(index_session); uds_request_queue_finish(index_session->callback_queue); index_session->callback_queue = NULL; - uds_destroy_cond(&index_session->load_context.cond); uds_destroy_mutex(&index_session->load_context.mutex); - uds_destroy_cond(&index_session->request_cond); uds_destroy_mutex(&index_session->request_mutex); uds_log_debug("Destroyed index session"); uds_free(index_session); @@ -758,3 +741,14 @@ int uds_get_index_session_stats(struct uds_index_session *index_session, return UDS_SUCCESS; } + +void uds_wait_cond(struct cond_var *cv, struct mutex *mutex) +{ + DEFINE_WAIT(__wait); + + prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); + uds_unlock_mutex(mutex); + schedule(); + finish_wait(&cv->wait_queue, &__wait); + uds_lock_mutex(mutex); +} diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index 1596f6ba43a5..edd81f03c2b5 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -754,7 +754,6 @@ static void free_chapter_writer(struct chapter_writer *writer) stop_chapter_writer(writer); uds_destroy_mutex(&writer->mutex); - uds_destroy_cond(&writer->cond); uds_free_open_chapter_index(writer->open_chapter_index); uds_free(writer->collated_records); uds_free(writer); @@ -781,12 +780,7 @@ static int make_chapter_writer(struct uds_index *index, return result; } - result = uds_init_cond(&writer->cond); - if (result != UDS_SUCCESS) { - uds_destroy_mutex(&writer->mutex); - uds_free(writer); - return result; - } + uds_init_cond(&writer->cond); result = uds_allocate_cache_aligned(collated_records_size, "collated records", &writer->collated_records); diff --git a/drivers/md/dm-vdo/indexer.h b/drivers/md/dm-vdo/indexer.h index 59e6a5ca2acb..3744aaf625b0 100644 --- a/drivers/md/dm-vdo/indexer.h +++ b/drivers/md/dm-vdo/indexer.h @@ -6,7 +6,10 @@ #ifndef INDEXER_H #define INDEXER_H +#include +#include #include +#include #include "funnel-queue.h" @@ -326,4 +329,25 @@ int __must_check uds_get_index_session_stats(struct uds_index_session *session, /* This function will fail if any required field of the request is not set. */ int __must_check uds_launch_request(struct uds_request *request); +struct cond_var { + wait_queue_head_t wait_queue; +}; + +static inline void uds_init_cond(struct cond_var *cv) +{ + init_waitqueue_head(&cv->wait_queue); +} + +static inline void uds_signal_cond(struct cond_var *cv) +{ + wake_up(&cv->wait_queue); +} + +static inline void uds_broadcast_cond(struct cond_var *cv) +{ + wake_up_all(&cv->wait_queue); +} + +void uds_wait_cond(struct cond_var *cv, struct mutex *mutex); + #endif /* INDEXER_H */ diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index 5d371bfba8ff..30760b1c4d30 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -9,7 +9,6 @@ #include #include #include -#include #include "errors.h" #include "logger.h" @@ -135,14 +134,3 @@ int uds_join_threads(struct thread *thread) uds_free(thread); return UDS_SUCCESS; } - -void uds_wait_cond(struct cond_var *cv, struct mutex *mutex) -{ - DEFINE_WAIT(__wait); - - prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); - uds_unlock_mutex(mutex); - schedule(); - finish_wait(&cv->wait_queue, &__wait); - uds_lock_mutex(mutex); -} diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h index c7a5d2d948a4..fb71f8f1b46e 100644 --- a/drivers/md/dm-vdo/thread-utils.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -11,16 +11,11 @@ #include #include #include -#include #include "errors.h" /* Thread and synchronization utilities for UDS */ -struct cond_var { - wait_queue_head_t wait_queue; -}; - struct thread; @@ -31,30 +26,8 @@ void uds_perform_once(atomic_t *once_state, void (*function) (void)); int uds_join_threads(struct thread *thread); -static inline int __must_check uds_init_cond(struct cond_var *cv) -{ - init_waitqueue_head(&cv->wait_queue); - return UDS_SUCCESS; -} - -static inline void uds_signal_cond(struct cond_var *cv) -{ - wake_up(&cv->wait_queue); -} - -static inline void uds_broadcast_cond(struct cond_var *cv) -{ - wake_up_all(&cv->wait_queue); -} - -void uds_wait_cond(struct cond_var *cv, struct mutex *mutex); - /* FIXME: all below wrappers should be removed! */ -static inline void uds_destroy_cond(struct cond_var *cv) -{ -} - static inline int __must_check uds_init_mutex(struct mutex *mutex) { mutex_init(mutex); @@ -76,5 +49,4 @@ static inline void uds_unlock_mutex(struct mutex *mutex) mutex_unlock(mutex); } - #endif /* UDS_THREADS_H */ diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 5b3cb5d89e47..3b256a78fb02 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -1627,17 +1627,8 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout return result; } - result = uds_init_cond(&volume->read_threads_read_done_cond); - if (result != UDS_SUCCESS) { - uds_free_volume(volume); - return result; - } - - result = uds_init_cond(&volume->read_threads_cond); - if (result != UDS_SUCCESS) { - uds_free_volume(volume); - return result; - } + uds_init_cond(&volume->read_threads_read_done_cond); + uds_init_cond(&volume->read_threads_cond); result = uds_allocate(config->read_threads, struct thread *, "reader threads", &volume->reader_threads); @@ -1700,8 +1691,6 @@ void uds_free_volume(struct volume *volume) if (volume->client != NULL) dm_bufio_client_destroy(uds_forget(volume->client)); - uds_destroy_cond(&volume->read_threads_cond); - uds_destroy_cond(&volume->read_threads_read_done_cond); uds_destroy_mutex(&volume->read_threads_mutex); uds_free_index_page_map(volume->index_page_map); uds_free_radix_sorter(volume->radix_sorter); From fe6e4ccbe872bfcf2d8d6ce48f4ca7ca260d7102 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 12:35:10 -0600 Subject: [PATCH 0906/1038] dm vdo thread-utils: remove all uds_*_mutex wrappers Just use mutex_init, mutex_lock and mutex_unlock. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/index-session.c | 110 +++++++++++++----------------- drivers/md/dm-vdo/index.c | 42 +++++------- drivers/md/dm-vdo/thread-utils.h | 23 ------- drivers/md/dm-vdo/volume-index.c | 40 ++++------- drivers/md/dm-vdo/volume.c | 38 +++++------ 5 files changed, 96 insertions(+), 157 deletions(-) diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/index-session.c index 4837621c16db..a482ccd3981e 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/index-session.c @@ -61,10 +61,10 @@ enum index_session_flag { /* Release a reference to an index session. */ static void release_index_session(struct uds_index_session *index_session) { - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); if (--index_session->request_count == 0) uds_broadcast_cond(&index_session->request_cond); - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); } /* @@ -76,10 +76,10 @@ static int get_index_session(struct uds_index_session *index_session) unsigned int state; int result = UDS_SUCCESS; - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); index_session->request_count++; state = index_session->state; - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); if (state == IS_FLAG_LOADED) { return UDS_SUCCESS; @@ -141,9 +141,9 @@ static void enter_callback_stage(struct uds_request *request) { if (request->status != UDS_SUCCESS) { /* All request errors are considered unrecoverable */ - uds_lock_mutex(&request->session->request_mutex); + mutex_lock(&request->session->request_mutex); request->session->state |= IS_FLAG_DISABLED; - uds_unlock_mutex(&request->session->request_mutex); + mutex_unlock(&request->session->request_mutex); } uds_request_queue_enqueue(request->session->callback_queue, request); @@ -224,28 +224,14 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde if (result != UDS_SUCCESS) return result; - result = uds_init_mutex(&session->request_mutex); - if (result != UDS_SUCCESS) { - uds_free(session); - return result; - } - + mutex_init(&session->request_mutex); uds_init_cond(&session->request_cond); - - result = uds_init_mutex(&session->load_context.mutex); - if (result != UDS_SUCCESS) { - uds_destroy_mutex(&session->request_mutex); - uds_free(session); - return result; - } - + mutex_init(&session->load_context.mutex); uds_init_cond(&session->load_context.cond); result = uds_make_request_queue("callbackW", &handle_callbacks, &session->callback_queue); if (result != UDS_SUCCESS) { - uds_destroy_mutex(&session->load_context.mutex); - uds_destroy_mutex(&session->request_mutex); uds_free(session); return result; } @@ -268,7 +254,7 @@ static int __must_check start_loading_index_session(struct uds_index_session *in { int result; - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); if (index_session->state & IS_FLAG_SUSPENDED) { uds_log_info("Index session is suspended"); result = -EBUSY; @@ -279,20 +265,20 @@ static int __must_check start_loading_index_session(struct uds_index_session *in index_session->state |= IS_FLAG_LOADING; result = UDS_SUCCESS; } - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); return result; } static void finish_loading_index_session(struct uds_index_session *index_session, int result) { - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); index_session->state &= ~IS_FLAG_LOADING; if (result == UDS_SUCCESS) index_session->state |= IS_FLAG_LOADED; uds_broadcast_cond(&index_session->request_cond); - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); } static int initialize_index_session(struct uds_index_session *index_session, @@ -376,12 +362,12 @@ int uds_open_index(enum uds_open_index_type open_type, static void wait_for_no_requests_in_progress(struct uds_index_session *index_session) { - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); while (index_session->request_count > 0) { uds_wait_cond(&index_session->request_cond, &index_session->request_mutex); } - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); } static int __must_check save_index(struct uds_index_session *index_session) @@ -392,7 +378,7 @@ static int __must_check save_index(struct uds_index_session *index_session) static void suspend_rebuild(struct uds_index_session *session) { - uds_lock_mutex(&session->load_context.mutex); + mutex_lock(&session->load_context.mutex); switch (session->load_context.status) { case INDEX_OPENING: session->load_context.status = INDEX_SUSPENDING; @@ -419,7 +405,7 @@ static void suspend_rebuild(struct uds_index_session *session) session->load_context.status); break; } - uds_unlock_mutex(&session->load_context.mutex); + mutex_unlock(&session->load_context.mutex); } /* @@ -433,7 +419,7 @@ int uds_suspend_index_session(struct uds_index_session *session, bool save) bool rebuilding = false; /* Wait for any current index state change to complete. */ - uds_lock_mutex(&session->request_mutex); + mutex_lock(&session->request_mutex); while (session->state & IS_FLAG_CLOSING) uds_wait_cond(&session->request_cond, &session->request_mutex); @@ -453,7 +439,7 @@ int uds_suspend_index_session(struct uds_index_session *session, bool save) session->state |= IS_FLAG_SUSPENDED; uds_broadcast_cond(&session->request_cond); } - uds_unlock_mutex(&session->request_mutex); + mutex_unlock(&session->request_mutex); if (no_work) return uds_status_to_errno(result); @@ -465,11 +451,11 @@ int uds_suspend_index_session(struct uds_index_session *session, bool save) else result = uds_flush_index_session(session); - uds_lock_mutex(&session->request_mutex); + mutex_lock(&session->request_mutex); session->state &= ~IS_FLAG_WAITING; session->state |= IS_FLAG_SUSPENDED; uds_broadcast_cond(&session->request_cond); - uds_unlock_mutex(&session->request_mutex); + mutex_unlock(&session->request_mutex); return uds_status_to_errno(result); } @@ -496,7 +482,7 @@ int uds_resume_index_session(struct uds_index_session *session, bool no_work = false; bool resume_replay = false; - uds_lock_mutex(&session->request_mutex); + mutex_lock(&session->request_mutex); if (session->state & IS_FLAG_WAITING) { uds_log_info("Index session is already changing state"); no_work = true; @@ -510,7 +496,7 @@ int uds_resume_index_session(struct uds_index_session *session, if (session->state & IS_FLAG_LOADING) resume_replay = true; } - uds_unlock_mutex(&session->request_mutex); + mutex_unlock(&session->request_mutex); if (no_work) return result; @@ -518,16 +504,16 @@ int uds_resume_index_session(struct uds_index_session *session, if ((session->index != NULL) && (bdev != session->parameters.bdev)) { result = replace_device(session, bdev); if (result != UDS_SUCCESS) { - uds_lock_mutex(&session->request_mutex); + mutex_lock(&session->request_mutex); session->state &= ~IS_FLAG_WAITING; uds_broadcast_cond(&session->request_cond); - uds_unlock_mutex(&session->request_mutex); + mutex_unlock(&session->request_mutex); return uds_status_to_errno(result); } } if (resume_replay) { - uds_lock_mutex(&session->load_context.mutex); + mutex_lock(&session->load_context.mutex); switch (session->load_context.status) { case INDEX_SUSPENDED: session->load_context.status = INDEX_OPENING; @@ -548,14 +534,14 @@ int uds_resume_index_session(struct uds_index_session *session, session->load_context.status); break; } - uds_unlock_mutex(&session->load_context.mutex); + mutex_unlock(&session->load_context.mutex); } - uds_lock_mutex(&session->request_mutex); + mutex_lock(&session->request_mutex); session->state &= ~IS_FLAG_WAITING; session->state &= ~IS_FLAG_SUSPENDED; uds_broadcast_cond(&session->request_cond); - uds_unlock_mutex(&session->request_mutex); + mutex_unlock(&session->request_mutex); return UDS_SUCCESS; } @@ -568,9 +554,9 @@ static int save_and_free_index(struct uds_index_session *index_session) if (index == NULL) return UDS_SUCCESS; - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); suspended = (index_session->state & IS_FLAG_SUSPENDED); - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); if (!suspended) { result = uds_save_index(index); @@ -585,14 +571,14 @@ static int save_and_free_index(struct uds_index_session *index_session) * Reset all index state that happens to be in the index * session, so it doesn't affect any future index. */ - uds_lock_mutex(&index_session->load_context.mutex); + mutex_lock(&index_session->load_context.mutex); index_session->load_context.status = INDEX_OPENING; - uds_unlock_mutex(&index_session->load_context.mutex); + mutex_unlock(&index_session->load_context.mutex); - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); /* Only the suspend bit will remain relevant. */ index_session->state &= IS_FLAG_SUSPENDED; - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); return result; } @@ -603,7 +589,7 @@ int uds_close_index(struct uds_index_session *index_session) int result = UDS_SUCCESS; /* Wait for any current index state change to complete. */ - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); while ((index_session->state & IS_FLAG_WAITING) || (index_session->state & IS_FLAG_CLOSING)) { uds_wait_cond(&index_session->request_cond, @@ -620,7 +606,7 @@ int uds_close_index(struct uds_index_session *index_session) } else { index_session->state |= IS_FLAG_CLOSING; } - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); if (result != UDS_SUCCESS) return uds_status_to_errno(result); @@ -629,10 +615,10 @@ int uds_close_index(struct uds_index_session *index_session) result = save_and_free_index(index_session); uds_log_debug("Closed index"); - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); index_session->state &= ~IS_FLAG_CLOSING; uds_broadcast_cond(&index_session->request_cond); - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); return uds_status_to_errno(result); } @@ -645,7 +631,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) uds_log_debug("Destroying index session"); /* Wait for any current index state change to complete. */ - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); while ((index_session->state & IS_FLAG_WAITING) || (index_session->state & IS_FLAG_CLOSING)) { uds_wait_cond(&index_session->request_cond, @@ -653,7 +639,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) } if (index_session->state & IS_FLAG_DESTROYING) { - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); uds_log_info("Index session is already closing"); return -EBUSY; } @@ -661,32 +647,30 @@ int uds_destroy_index_session(struct uds_index_session *index_session) index_session->state |= IS_FLAG_DESTROYING; load_pending = ((index_session->state & IS_FLAG_LOADING) && (index_session->state & IS_FLAG_SUSPENDED)); - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); if (load_pending) { /* Tell the index to terminate the rebuild. */ - uds_lock_mutex(&index_session->load_context.mutex); + mutex_lock(&index_session->load_context.mutex); if (index_session->load_context.status == INDEX_SUSPENDED) { index_session->load_context.status = INDEX_FREEING; uds_broadcast_cond(&index_session->load_context.cond); } - uds_unlock_mutex(&index_session->load_context.mutex); + mutex_unlock(&index_session->load_context.mutex); /* Wait until the load exits before proceeding. */ - uds_lock_mutex(&index_session->request_mutex); + mutex_lock(&index_session->request_mutex); while (index_session->state & IS_FLAG_LOADING) { uds_wait_cond(&index_session->request_cond, &index_session->request_mutex); } - uds_unlock_mutex(&index_session->request_mutex); + mutex_unlock(&index_session->request_mutex); } wait_for_no_requests_in_progress(index_session); result = save_and_free_index(index_session); uds_request_queue_finish(index_session->callback_queue); index_session->callback_queue = NULL; - uds_destroy_mutex(&index_session->load_context.mutex); - uds_destroy_mutex(&index_session->request_mutex); uds_log_debug("Destroyed index session"); uds_free(index_session); return uds_status_to_errno(result); @@ -747,8 +731,8 @@ void uds_wait_cond(struct cond_var *cv, struct mutex *mutex) DEFINE_WAIT(__wait); prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); - uds_unlock_mutex(mutex); + mutex_unlock(mutex); schedule(); finish_wait(&cv->wait_queue, &__wait); - uds_lock_mutex(mutex); + mutex_lock(mutex); } diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index edd81f03c2b5..5c9906e73c84 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -180,11 +180,11 @@ static int finish_previous_chapter(struct uds_index *index, u64 current_chapter_ int result; struct chapter_writer *writer = index->chapter_writer; - uds_lock_mutex(&writer->mutex); + mutex_lock(&writer->mutex); while (index->newest_virtual_chapter < current_chapter_number) uds_wait_cond(&writer->cond, &writer->mutex); result = writer->result; - uds_unlock_mutex(&writer->mutex); + mutex_unlock(&writer->mutex); if (result != UDS_SUCCESS) return uds_log_error_strerror(result, @@ -219,11 +219,11 @@ static unsigned int start_closing_chapter(struct uds_index *index, unsigned int finished_zones; struct chapter_writer *writer = index->chapter_writer; - uds_lock_mutex(&writer->mutex); + mutex_lock(&writer->mutex); finished_zones = ++writer->zones_to_write; writer->chapters[zone_number] = chapter; uds_broadcast_cond(&writer->cond); - uds_unlock_mutex(&writer->mutex); + mutex_unlock(&writer->mutex); return finished_zones; } @@ -678,7 +678,7 @@ static void close_chapters(void *arg) struct uds_index *index = writer->index; uds_log_debug("chapter writer starting"); - uds_lock_mutex(&writer->mutex); + mutex_lock(&writer->mutex); for (;;) { while (writer->zones_to_write < index->zone_count) { if (writer->stop && (writer->zones_to_write == 0)) { @@ -686,7 +686,7 @@ static void close_chapters(void *arg) * We've been told to stop, and all of the zones are in the same * open chapter, so we can exit now. */ - uds_unlock_mutex(&writer->mutex); + mutex_unlock(&writer->mutex); uds_log_debug("chapter writer stopping"); return; } @@ -698,7 +698,7 @@ static void close_chapters(void *arg) * it seems safer in principle. It's OK to access the chapter and chapter_number * fields without the lock since those aren't allowed to change until we're done. */ - uds_unlock_mutex(&writer->mutex); + mutex_unlock(&writer->mutex); if (index->has_saved_open_chapter) { /* @@ -719,7 +719,7 @@ static void close_chapters(void *arg) writer->collated_records, index->newest_virtual_chapter); - uds_lock_mutex(&writer->mutex); + mutex_lock(&writer->mutex); index->newest_virtual_chapter++; index->oldest_virtual_chapter += uds_chapters_to_expire(index->volume->geometry, @@ -734,14 +734,14 @@ static void stop_chapter_writer(struct chapter_writer *writer) { struct thread *writer_thread = NULL; - uds_lock_mutex(&writer->mutex); + mutex_lock(&writer->mutex); if (writer->thread != NULL) { writer_thread = writer->thread; writer->thread = NULL; writer->stop = true; uds_broadcast_cond(&writer->cond); } - uds_unlock_mutex(&writer->mutex); + mutex_unlock(&writer->mutex); if (writer_thread != NULL) uds_join_threads(writer_thread); @@ -753,7 +753,6 @@ static void free_chapter_writer(struct chapter_writer *writer) return; stop_chapter_writer(writer); - uds_destroy_mutex(&writer->mutex); uds_free_open_chapter_index(writer->open_chapter_index); uds_free(writer->collated_records); uds_free(writer); @@ -774,12 +773,7 @@ static int make_chapter_writer(struct uds_index *index, return result; writer->index = index; - result = uds_init_mutex(&writer->mutex); - if (result != UDS_SUCCESS) { - uds_free(writer); - return result; - } - + mutex_init(&writer->mutex); uds_init_cond(&writer->cond); result = uds_allocate_cache_aligned(collated_records_size, "collated records", @@ -957,9 +951,9 @@ static bool check_for_suspend(struct uds_index *index) if (index->load_context == NULL) return false; - uds_lock_mutex(&index->load_context->mutex); + mutex_lock(&index->load_context->mutex); if (index->load_context->status != INDEX_SUSPENDING) { - uds_unlock_mutex(&index->load_context->mutex); + mutex_unlock(&index->load_context->mutex); return false; } @@ -972,7 +966,7 @@ static bool check_for_suspend(struct uds_index *index) uds_wait_cond(&index->load_context->cond, &index->load_context->mutex); closing = (index->load_context->status == INDEX_FREEING); - uds_unlock_mutex(&index->load_context->mutex); + mutex_unlock(&index->load_context->mutex); return closing; } @@ -1261,14 +1255,14 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op } if (index->load_context != NULL) { - uds_lock_mutex(&index->load_context->mutex); + mutex_lock(&index->load_context->mutex); index->load_context->status = INDEX_READY; /* * If we get here, suspend is meaningless, but notify any thread trying to suspend * us so it doesn't hang. */ uds_broadcast_cond(&index->load_context->cond); - uds_unlock_mutex(&index->load_context->mutex); + mutex_unlock(&index->load_context->mutex); } index->has_saved_open_chapter = loaded; @@ -1307,10 +1301,10 @@ void uds_wait_for_idle_index(struct uds_index *index) { struct chapter_writer *writer = index->chapter_writer; - uds_lock_mutex(&writer->mutex); + mutex_lock(&writer->mutex); while (writer->zones_to_write > 0) uds_wait_cond(&writer->cond, &writer->mutex); - uds_unlock_mutex(&writer->mutex); + mutex_unlock(&writer->mutex); } /* This function assumes that all requests have been drained. */ diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h index fb71f8f1b46e..8b55f0d1ab80 100644 --- a/drivers/md/dm-vdo/thread-utils.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -26,27 +26,4 @@ void uds_perform_once(atomic_t *once_state, void (*function) (void)); int uds_join_threads(struct thread *thread); -/* FIXME: all below wrappers should be removed! */ - -static inline int __must_check uds_init_mutex(struct mutex *mutex) -{ - mutex_init(mutex); - return UDS_SUCCESS; -} - -static inline int uds_destroy_mutex(struct mutex *mutex) -{ - return UDS_SUCCESS; -} - -static inline void uds_lock_mutex(struct mutex *mutex) -{ - mutex_lock(mutex); -} - -static inline void uds_unlock_mutex(struct mutex *mutex) -{ - mutex_unlock(mutex); -} - #endif /* UDS_THREADS_H */ diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/volume-index.c index 39c4be06780f..36e3c2e3d799 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/volume-index.c @@ -286,13 +286,8 @@ void uds_free_volume_index(struct volume_index *volume_index) if (volume_index == NULL) return; - if (volume_index->zones != NULL) { - unsigned int zone; - - for (zone = 0; zone < volume_index->zone_count; zone++) - uds_destroy_mutex(&volume_index->zones[zone].hook_mutex); + if (volume_index->zones != NULL) uds_free(uds_forget(volume_index->zones)); - } uninitialize_volume_sub_index(&volume_index->vi_non_hook); uninitialize_volume_sub_index(&volume_index->vi_hook); @@ -546,10 +541,10 @@ int uds_get_volume_index_record(struct volume_index *volume_index, get_volume_sub_index_zone(&volume_index->vi_hook, name); struct mutex *mutex = &volume_index->zones[zone].hook_mutex; - uds_lock_mutex(mutex); + mutex_lock(mutex); result = get_volume_sub_index_record(&volume_index->vi_hook, name, record); - uds_unlock_mutex(mutex); + mutex_unlock(mutex); /* Remember the mutex so that other operations on the index record can use it. */ record->mutex = mutex; } else { @@ -578,13 +573,13 @@ int uds_put_volume_index_record(struct volume_index_record *record, u64 virtual_ } address = extract_address(sub_index, record->name); if (unlikely(record->mutex != NULL)) - uds_lock_mutex(record->mutex); + mutex_lock(record->mutex); result = uds_put_delta_index_entry(&record->delta_entry, address, convert_virtual_to_index(sub_index, virtual_chapter), record->is_found ? record->name->name : NULL); if (unlikely(record->mutex != NULL)) - uds_unlock_mutex(record->mutex); + mutex_unlock(record->mutex); switch (result) { case UDS_SUCCESS: record->virtual_chapter = virtual_chapter; @@ -614,10 +609,10 @@ int uds_remove_volume_index_record(struct volume_index_record *record) /* Mark the record so that it cannot be used again */ record->is_found = false; if (unlikely(record->mutex != NULL)) - uds_lock_mutex(record->mutex); + mutex_lock(record->mutex); result = uds_remove_delta_index_entry(&record->delta_entry); if (unlikely(record->mutex != NULL)) - uds_unlock_mutex(record->mutex); + mutex_unlock(record->mutex); return result; } @@ -688,10 +683,10 @@ void uds_set_volume_index_zone_open_chapter(struct volume_index *volume_index, * chapter number is changing. */ if (has_sparse(volume_index)) { - uds_lock_mutex(mutex); + mutex_lock(mutex); set_volume_sub_index_zone_open_chapter(&volume_index->vi_hook, zone_number, virtual_chapter); - uds_unlock_mutex(mutex); + mutex_unlock(mutex); } } @@ -730,12 +725,12 @@ int uds_set_volume_index_record_chapter(struct volume_index_record *record, } if (unlikely(record->mutex != NULL)) - uds_lock_mutex(record->mutex); + mutex_lock(record->mutex); result = uds_set_delta_entry_value(&record->delta_entry, convert_virtual_to_index(sub_index, virtual_chapter)); if (unlikely(record->mutex != NULL)) - uds_unlock_mutex(record->mutex); + mutex_unlock(record->mutex); if (result != UDS_SUCCESS) return result; @@ -785,9 +780,9 @@ u64 uds_lookup_volume_index_name(const struct volume_index *volume_index, if (!uds_is_volume_index_sample(volume_index, name)) return NO_CHAPTER; - uds_lock_mutex(mutex); + mutex_lock(mutex); virtual_chapter = lookup_volume_sub_index_name(&volume_index->vi_hook, name); - uds_unlock_mutex(mutex); + mutex_unlock(mutex); return virtual_chapter; } @@ -1258,13 +1253,8 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non return result; } - for (zone = 0; zone < config->zone_count; zone++) { - result = uds_init_mutex(&volume_index->zones[zone].hook_mutex); - if (result != UDS_SUCCESS) { - uds_free_volume_index(volume_index); - return result; - } - } + for (zone = 0; zone < config->zone_count; zone++) + mutex_init(&volume_index->zones[zone].hook_mutex); split_configuration(config, &split); result = initialize_volume_sub_index(&split.non_hook_config, volume_nonce, 'd', diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 3b256a78fb02..0fb06fd315ef 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -554,7 +554,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) page = select_victim_in_cache(&volume->page_cache); - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); page_data = dm_bufio_read(volume->client, page_number, &page->buffer); if (IS_ERR(page_data)) { result = -PTR_ERR(page_data); @@ -564,7 +564,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) cancel_page_in_cache(&volume->page_cache, page_number, page); return result; } - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); if (entry->invalid) { uds_log_warning("Page %u invalidated after read", page_number); @@ -626,7 +626,7 @@ static void read_thread_function(void *arg) struct volume *volume = arg; uds_log_debug("reader starting"); - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); while (true) { struct queued_read *queue_entry; int result; @@ -638,7 +638,7 @@ static void read_thread_function(void *arg) result = process_entry(volume, queue_entry); release_queued_requests(volume, queue_entry, result); } - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); uds_log_debug("reader done"); } @@ -769,7 +769,7 @@ static int get_volume_page_protected(struct volume *volume, struct uds_request * /* Prepare to enqueue a read for the page. */ end_pending_search(&volume->page_cache, request->zone_number); - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); /* * Do the lookup again while holding the read mutex (no longer the fast case so this should @@ -787,7 +787,7 @@ static int get_volume_page_protected(struct volume *volume, struct uds_request * * turns out to be significant in some cases. The page is not available yet so * the order does not matter for correctness as it does below. */ - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); begin_pending_search(&volume->page_cache, physical_page, request->zone_number); return UDS_QUEUED; @@ -799,7 +799,7 @@ static int get_volume_page_protected(struct volume *volume, struct uds_request * * the caller gets to look at it. */ begin_pending_search(&volume->page_cache, physical_page, request->zone_number); - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); *page_ptr = page; return UDS_SUCCESS; } @@ -810,9 +810,9 @@ static int get_volume_page(struct volume *volume, u32 chapter, u32 page_number, int result; u32 physical_page = map_to_physical_page(volume->geometry, chapter, page_number); - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); result = get_volume_page_locked(volume, physical_page, page_ptr); - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); return result; } @@ -1053,10 +1053,10 @@ void uds_forget_chapter(struct volume *volume, u64 virtual_chapter) u32 i; uds_log_debug("forgetting chapter %llu", (unsigned long long) virtual_chapter); - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); for (i = 0; i < volume->geometry->pages_per_chapter; i++) invalidate_page(&volume->page_cache, first_page + i); - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); } /* @@ -1141,10 +1141,10 @@ static int write_index_pages(struct volume *volume, u32 physical_chapter_number, physical_chapter_number, index_page_number, delta_list_number - 1); - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); result = donate_index_page_locked(volume, physical_chapter_number, index_page_number, page_buffer); - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); if (result != UDS_SUCCESS) { dm_bufio_release(page_buffer); return result; @@ -1621,12 +1621,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout return result; } - result = uds_init_mutex(&volume->read_threads_mutex); - if (result != UDS_SUCCESS) { - uds_free_volume(volume); - return result; - } - + mutex_init(&volume->read_threads_mutex); uds_init_cond(&volume->read_threads_read_done_cond); uds_init_cond(&volume->read_threads_cond); @@ -1675,10 +1670,10 @@ void uds_free_volume(struct volume *volume) unsigned int i; /* This works even if some threads weren't started. */ - uds_lock_mutex(&volume->read_threads_mutex); + mutex_lock(&volume->read_threads_mutex); volume->read_threads_exiting = true; uds_broadcast_cond(&volume->read_threads_cond); - uds_unlock_mutex(&volume->read_threads_mutex); + mutex_unlock(&volume->read_threads_mutex); for (i = 0; i < volume->read_thread_count; i++) uds_join_threads(volume->reader_threads[i]); uds_free(volume->reader_threads); @@ -1691,7 +1686,6 @@ void uds_free_volume(struct volume *volume) if (volume->client != NULL) dm_bufio_client_destroy(uds_forget(volume->client)); - uds_destroy_mutex(&volume->read_threads_mutex); uds_free_index_page_map(volume->index_page_map); uds_free_radix_sorter(volume->radix_sorter); uds_free(volume->geometry); From 650e3107bc5280855e602e8a24e4f50dd1779ae7 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 12:08:09 -0600 Subject: [PATCH 0907/1038] dm vdo thread-utils: further cleanup of thread functions Change thread function prefix from "uds_" to "vdo_" and fix vdo_join_threads() to return void. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/funnel-requestqueue.c | 8 ++------ drivers/md/dm-vdo/index.c | 4 ++-- drivers/md/dm-vdo/status-codes.c | 2 +- drivers/md/dm-vdo/thread-utils.c | 9 ++++----- drivers/md/dm-vdo/thread-utils.h | 9 ++++----- drivers/md/dm-vdo/volume.c | 4 ++-- 6 files changed, 15 insertions(+), 21 deletions(-) diff --git a/drivers/md/dm-vdo/funnel-requestqueue.c b/drivers/md/dm-vdo/funnel-requestqueue.c index e7a3a4962295..d2b49e39550c 100644 --- a/drivers/md/dm-vdo/funnel-requestqueue.c +++ b/drivers/md/dm-vdo/funnel-requestqueue.c @@ -219,7 +219,7 @@ int uds_make_request_queue(const char *queue_name, return result; } - result = uds_create_thread(request_queue_worker, queue, queue_name, + result = vdo_create_thread(request_queue_worker, queue, queue_name, &queue->thread); if (result != UDS_SUCCESS) { uds_request_queue_finish(queue); @@ -256,8 +256,6 @@ void uds_request_queue_enqueue(struct uds_request_queue *queue, void uds_request_queue_finish(struct uds_request_queue *queue) { - int result; - if (queue == NULL) return; @@ -272,9 +270,7 @@ void uds_request_queue_finish(struct uds_request_queue *queue) if (queue->started) { wake_up_worker(queue); - result = uds_join_threads(queue->thread); - if (result != UDS_SUCCESS) - uds_log_warning_strerror(result, "Failed to join worker thread"); + vdo_join_threads(queue->thread); } uds_free_funnel_queue(queue->main_queue); diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/index.c index 5c9906e73c84..9d4a8e5cbaad 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/index.c @@ -744,7 +744,7 @@ static void stop_chapter_writer(struct chapter_writer *writer) mutex_unlock(&writer->mutex); if (writer_thread != NULL) - uds_join_threads(writer_thread); + vdo_join_threads(writer_thread); } static void free_chapter_writer(struct chapter_writer *writer) @@ -796,7 +796,7 @@ static int make_chapter_writer(struct uds_index *index, collated_records_size + writer->open_chapter_index->memory_size); - result = uds_create_thread(close_chapters, writer, "writer", &writer->thread); + result = vdo_create_thread(close_chapters, writer, "writer", &writer->thread); if (result != UDS_SUCCESS) { free_chapter_writer(writer); return result; diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index d77bc5e4a99a..efba1ead0aca 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -82,7 +82,7 @@ static void do_status_code_registration(void) */ int vdo_register_status_codes(void) { - uds_perform_once(&vdo_status_codes_registered, do_status_code_registration); + vdo_perform_once(&vdo_status_codes_registered, do_status_code_registration); return status_code_registration_result; } diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index 30760b1c4d30..0b80247c7f1b 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -33,7 +33,7 @@ enum { }; /* Run a function once only, and record that fact in the atomic value. */ -void uds_perform_once(atomic_t *once, void (*function)(void)) +void vdo_perform_once(atomic_t *once, void (*function)(void)) { for (;;) { switch (atomic_cmpxchg(once, ONCE_NOT_DONE, ONCE_IN_PROGRESS)) { @@ -63,7 +63,7 @@ static int thread_starter(void *arg) struct thread *thread = arg; thread->thread_task = current; - uds_perform_once(&thread_once, thread_init); + vdo_perform_once(&thread_once, thread_init); mutex_lock(&thread_mutex); hlist_add_head(&thread->thread_links, &thread_list); mutex_unlock(&thread_mutex); @@ -74,7 +74,7 @@ static int thread_starter(void *arg) return 0; } -int uds_create_thread(void (*thread_function)(void *), void *thread_data, +int vdo_create_thread(void (*thread_function)(void *), void *thread_data, const char *name, struct thread **new_thread) { char *name_colon = strchr(name, ':'); @@ -123,7 +123,7 @@ int uds_create_thread(void (*thread_function)(void *), void *thread_data, return UDS_SUCCESS; } -int uds_join_threads(struct thread *thread) +void vdo_join_threads(struct thread *thread) { while (wait_for_completion_interruptible(&thread->thread_done)) fsleep(1000); @@ -132,5 +132,4 @@ int uds_join_threads(struct thread *thread) hlist_del(&thread->thread_links); mutex_unlock(&thread_mutex); uds_free(thread); - return UDS_SUCCESS; } diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h index 8b55f0d1ab80..ebe032e066ff 100644 --- a/drivers/md/dm-vdo/thread-utils.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -14,16 +14,15 @@ #include "errors.h" -/* Thread and synchronization utilities for UDS */ +/* Thread and synchronization utilities */ struct thread; -int __must_check uds_create_thread(void (*thread_function)(void *), void *thread_data, +int __must_check vdo_create_thread(void (*thread_function)(void *), void *thread_data, const char *name, struct thread **new_thread); +void vdo_join_threads(struct thread *thread); -void uds_perform_once(atomic_t *once_state, void (*function) (void)); - -int uds_join_threads(struct thread *thread); +void vdo_perform_once(atomic_t *once_state, void (*function) (void)); #endif /* UDS_THREADS_H */ diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 0fb06fd315ef..37c2ef0777e5 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -1633,7 +1633,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout } for (i = 0; i < config->read_threads; i++) { - result = uds_create_thread(read_thread_function, (void *) volume, + result = vdo_create_thread(read_thread_function, (void *) volume, "reader", &volume->reader_threads[i]); if (result != UDS_SUCCESS) { uds_free_volume(volume); @@ -1675,7 +1675,7 @@ void uds_free_volume(struct volume *volume) uds_broadcast_cond(&volume->read_threads_cond); mutex_unlock(&volume->read_threads_mutex); for (i = 0; i < volume->read_thread_count; i++) - uds_join_threads(volume->reader_threads[i]); + vdo_join_threads(volume->reader_threads[i]); uds_free(volume->reader_threads); volume->reader_threads = NULL; } From cb6f8b75005c385e4b604d6e9e3e94153d58cfaa Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 13:17:05 -0600 Subject: [PATCH 0908/1038] dm vdo thread-utils: cleanup included headers Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/logger.c | 1 + drivers/md/dm-vdo/thread-utils.c | 4 ++-- drivers/md/dm-vdo/thread-utils.h | 6 ------ 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index ff1c570f81bf..969f10771ada 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -11,6 +11,7 @@ #include #include +#include "errors.h" #include "thread-device.h" #include "thread-utils.h" diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index 0b80247c7f1b..160679984d72 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -5,10 +5,10 @@ #include "thread-utils.h" -#include #include -#include #include +#include +#include #include "errors.h" #include "logger.h" diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h index ebe032e066ff..f3619a581c5e 100644 --- a/drivers/md/dm-vdo/thread-utils.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -7,12 +7,6 @@ #define THREAD_UTILS_H #include -#include -#include -#include -#include - -#include "errors.h" /* Thread and synchronization utilities */ From 82b354ffe28f43d6bbaf221bdeeb267bf6ff43d9 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 14:04:34 -0600 Subject: [PATCH 0909/1038] dm vdo thread-registry: rename all methods to reflect vdo-only use Otherwise, uds_ prefix is misleading (vdo_ is the new catch-all for code that is used by vdo-only or _both_ vdo and the indexer code). Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/memory-alloc.c | 12 ++++++------ drivers/md/dm-vdo/thread-device.c | 8 ++++---- drivers/md/dm-vdo/thread-registry.c | 8 ++++---- drivers/md/dm-vdo/thread-registry.h | 14 +++++++------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index 46dd5bda6825..3b2bda9248cb 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -15,14 +15,14 @@ /* * UDS and VDO keep track of which threads are allowed to allocate memory freely, and which threads - * must be careful to not do a memory allocation that does an I/O request. The allocating_threads - * threads_registry and its associated methods implement this tracking. + * must be careful to not do a memory allocation that does an I/O request. The 'allocating_threads' + * thread_registry and its associated methods implement this tracking. */ static struct thread_registry allocating_threads; static bool allocations_allowed(void) { - const bool *pointer = uds_lookup_thread(&allocating_threads); + const bool *pointer = vdo_lookup_thread(&allocating_threads); return (pointer != NULL) ? *pointer : false; } @@ -48,13 +48,13 @@ void uds_register_allocating_thread(struct registered_thread *new_thread, flag_ptr = &allocation_always_allowed; } - uds_register_thread(&allocating_threads, new_thread, flag_ptr); + vdo_register_thread(&allocating_threads, new_thread, flag_ptr); } /* Unregister the current thread as an allocating thread. */ void uds_unregister_allocating_thread(void) { - uds_unregister_thread(&allocating_threads); + vdo_unregister_thread(&allocating_threads); } /* @@ -384,7 +384,7 @@ int uds_duplicate_string(const char *string, const char *what, char **new_string void uds_memory_init(void) { spin_lock_init(&memory_stats.lock); - uds_initialize_thread_registry(&allocating_threads); + vdo_initialize_thread_registry(&allocating_threads); } void uds_memory_exit(void) diff --git a/drivers/md/dm-vdo/thread-device.c b/drivers/md/dm-vdo/thread-device.c index b87de448a83b..2bf14b9f67f8 100644 --- a/drivers/md/dm-vdo/thread-device.c +++ b/drivers/md/dm-vdo/thread-device.c @@ -14,23 +14,23 @@ static struct thread_registry device_id_thread_registry; void uds_register_thread_device_id(struct registered_thread *new_thread, unsigned int *id_ptr) { - uds_register_thread(&device_id_thread_registry, new_thread, id_ptr); + vdo_register_thread(&device_id_thread_registry, new_thread, id_ptr); } void uds_unregister_thread_device_id(void) { - uds_unregister_thread(&device_id_thread_registry); + vdo_unregister_thread(&device_id_thread_registry); } int uds_get_thread_device_id(void) { const unsigned int *pointer; - pointer = uds_lookup_thread(&device_id_thread_registry); + pointer = vdo_lookup_thread(&device_id_thread_registry); return (pointer != NULL) ? *pointer : -1; } void uds_initialize_thread_device_registry(void) { - uds_initialize_thread_registry(&device_id_thread_registry); + vdo_initialize_thread_registry(&device_id_thread_registry); } diff --git a/drivers/md/dm-vdo/thread-registry.c b/drivers/md/dm-vdo/thread-registry.c index 8c887158c224..1314d2b6a26f 100644 --- a/drivers/md/dm-vdo/thread-registry.c +++ b/drivers/md/dm-vdo/thread-registry.c @@ -14,14 +14,14 @@ * their normal operation. For example, we do not want to invoke the logger while holding a lock. */ -void uds_initialize_thread_registry(struct thread_registry *registry) +void vdo_initialize_thread_registry(struct thread_registry *registry) { INIT_LIST_HEAD(®istry->links); spin_lock_init(®istry->lock); } /* Register the current thread and associate it with a data pointer. */ -void uds_register_thread(struct thread_registry *registry, +void vdo_register_thread(struct thread_registry *registry, struct registered_thread *new_thread, const void *pointer) { struct registered_thread *thread; @@ -51,7 +51,7 @@ void uds_register_thread(struct thread_registry *registry, } } -void uds_unregister_thread(struct thread_registry *registry) +void vdo_unregister_thread(struct thread_registry *registry) { struct registered_thread *thread; bool found_it = false; @@ -74,7 +74,7 @@ void uds_unregister_thread(struct thread_registry *registry) } } -const void *uds_lookup_thread(struct thread_registry *registry) +const void *vdo_lookup_thread(struct thread_registry *registry) { struct registered_thread *thread; const void *result = NULL; diff --git a/drivers/md/dm-vdo/thread-registry.h b/drivers/md/dm-vdo/thread-registry.h index f70f755568a1..cc6d78312b9e 100644 --- a/drivers/md/dm-vdo/thread-registry.h +++ b/drivers/md/dm-vdo/thread-registry.h @@ -3,8 +3,8 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_THREAD_REGISTRY_H -#define UDS_THREAD_REGISTRY_H +#ifndef VDO_THREAD_REGISTRY_H +#define VDO_THREAD_REGISTRY_H #include #include @@ -20,13 +20,13 @@ struct registered_thread { struct task_struct *task; }; -void uds_initialize_thread_registry(struct thread_registry *registry); +void vdo_initialize_thread_registry(struct thread_registry *registry); -void uds_register_thread(struct thread_registry *registry, +void vdo_register_thread(struct thread_registry *registry, struct registered_thread *new_thread, const void *pointer); -void uds_unregister_thread(struct thread_registry *registry); +void vdo_unregister_thread(struct thread_registry *registry); -const void *uds_lookup_thread(struct thread_registry *registry); +const void *vdo_lookup_thread(struct thread_registry *registry); -#endif /* UDS_THREAD_REGISTRY_H */ +#endif /* VDO_THREAD_REGISTRY_H */ From 6a87a8a258ed5a6ba8939e88cdca1ed42d4eeebc Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 14:53:05 -0600 Subject: [PATCH 0910/1038] dm vdo thread-device: rename all methods to reflect vdo-only use Also moved vdo_init()'s call to vdo_initialize_thread_device_registry next to other registry initialization. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dm-vdo-target.c | 30 +++++++++++++++--------------- drivers/md/dm-vdo/logger.c | 2 +- drivers/md/dm-vdo/thread-device.c | 10 ++++------ drivers/md/dm-vdo/thread-device.h | 14 +++++++------- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index e754b9e30cab..7afd1dfec649 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -1107,7 +1107,7 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, vdo = get_vdo_for_target(ti); uds_register_allocating_thread(&allocating_thread, NULL); - uds_register_thread_device_id(&instance_thread, &vdo->instance); + vdo_register_thread_device_id(&instance_thread, &vdo->instance); /* * Must be done here so we don't map return codes. The code in dm-ioctl expects a 1 for a @@ -1120,7 +1120,7 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, result = vdo_status_to_errno(process_vdo_message(vdo, argc, argv)); } - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); uds_unregister_allocating_thread(); return result; } @@ -1632,9 +1632,9 @@ static int construct_new_vdo(struct dm_target *ti, unsigned int argc, char **arg if (result != VDO_SUCCESS) return -ENOMEM; - uds_register_thread_device_id(&instance_thread, &instance); + vdo_register_thread_device_id(&instance_thread, &instance); result = construct_new_vdo_registered(ti, argc, argv, instance); - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); return result; } @@ -1913,9 +1913,9 @@ static int vdo_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (vdo == NULL) { result = construct_new_vdo(ti, argc, argv); } else { - uds_register_thread_device_id(&instance_thread, &vdo->instance); + vdo_register_thread_device_id(&instance_thread, &vdo->instance); result = update_existing_vdo(device_name, ti, argc, argv, vdo); - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); } uds_unregister_allocating_thread(); @@ -1935,7 +1935,7 @@ static void vdo_dtr(struct dm_target *ti) unsigned int instance = vdo->instance; struct registered_thread allocating_thread, instance_thread; - uds_register_thread_device_id(&instance_thread, &instance); + vdo_register_thread_device_id(&instance_thread, &instance); uds_register_allocating_thread(&allocating_thread, NULL); device_name = vdo_get_device_name(ti); @@ -1945,7 +1945,7 @@ static void vdo_dtr(struct dm_target *ti) vdo_destroy(uds_forget(vdo)); uds_log_info("device '%s' stopped", device_name); - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); uds_unregister_allocating_thread(); release_instance(instance); } else if (config == vdo->device_config) { @@ -2104,7 +2104,7 @@ static void vdo_postsuspend(struct dm_target *ti) const char *device_name; int result; - uds_register_thread_device_id(&instance_thread, &vdo->instance); + vdo_register_thread_device_id(&instance_thread, &vdo->instance); device_name = vdo_get_device_name(vdo->device_config->owning_target); uds_log_info("suspending device '%s'", device_name); @@ -2129,7 +2129,7 @@ static void vdo_postsuspend(struct dm_target *ti) device_name); } - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); } /** @@ -2846,11 +2846,11 @@ static int vdo_preresume(struct dm_target *ti) struct vdo *vdo = get_vdo_for_target(ti); int result; - uds_register_thread_device_id(&instance_thread, &vdo->instance); + vdo_register_thread_device_id(&instance_thread, &vdo->instance); result = vdo_preresume_registered(ti, vdo); if ((result == VDO_PARAMETER_MISMATCH) || (result == VDO_INVALID_ADMIN_STATE)) result = -EINVAL; - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); return vdo_status_to_errno(result); } @@ -2858,10 +2858,10 @@ static void vdo_resume(struct dm_target *ti) { struct registered_thread instance_thread; - uds_register_thread_device_id(&instance_thread, + vdo_register_thread_device_id(&instance_thread, &get_vdo_for_target(ti)->instance); uds_log_info("device '%s' resumed", vdo_get_device_name(ti)); - uds_unregister_thread_device_id(); + vdo_unregister_thread_device_id(); } /* @@ -2912,10 +2912,10 @@ static int __init vdo_init(void) /* * UDS module level initialization must be done first, as VDO initialization depends on it */ - uds_initialize_thread_device_registry(); uds_memory_init(); uds_init_sysfs(); + vdo_initialize_thread_device_registry(); vdo_initialize_device_registry_once(); uds_log_info("loaded version %s", CURRENT_VERSION); diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 969f10771ada..6ba7e99ee8f9 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -176,7 +176,7 @@ static void emit_log_message(int priority, const char *module, const char *prefi } /* Not at interrupt level; we have a process we can look at, and might have a device ID. */ - device_instance = uds_get_thread_device_id(); + device_instance = vdo_get_thread_device_id(); if (device_instance >= 0) { emit_log_message_to_kernel(priority, "%s%u:%s: %s%pV%pV\n", module, device_instance, current->comm, prefix, vaf1, diff --git a/drivers/md/dm-vdo/thread-device.c b/drivers/md/dm-vdo/thread-device.c index 2bf14b9f67f8..df13ca914db8 100644 --- a/drivers/md/dm-vdo/thread-device.c +++ b/drivers/md/dm-vdo/thread-device.c @@ -5,24 +5,22 @@ #include "thread-device.h" -#include "thread-registry.h" - /* A registry of threads associated with device id numbers. */ static struct thread_registry device_id_thread_registry; /* Any registered thread must be unregistered. */ -void uds_register_thread_device_id(struct registered_thread *new_thread, +void vdo_register_thread_device_id(struct registered_thread *new_thread, unsigned int *id_ptr) { vdo_register_thread(&device_id_thread_registry, new_thread, id_ptr); } -void uds_unregister_thread_device_id(void) +void vdo_unregister_thread_device_id(void) { vdo_unregister_thread(&device_id_thread_registry); } -int uds_get_thread_device_id(void) +int vdo_get_thread_device_id(void) { const unsigned int *pointer; @@ -30,7 +28,7 @@ int uds_get_thread_device_id(void) return (pointer != NULL) ? *pointer : -1; } -void uds_initialize_thread_device_registry(void) +void vdo_initialize_thread_device_registry(void) { vdo_initialize_thread_registry(&device_id_thread_registry); } diff --git a/drivers/md/dm-vdo/thread-device.h b/drivers/md/dm-vdo/thread-device.h index 428b2908541d..494d9c9ef3f6 100644 --- a/drivers/md/dm-vdo/thread-device.h +++ b/drivers/md/dm-vdo/thread-device.h @@ -3,18 +3,18 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_THREAD_DEVICE_H -#define UDS_THREAD_DEVICE_H +#ifndef VDO_THREAD_DEVICE_H +#define VDO_THREAD_DEVICE_H #include "thread-registry.h" -void uds_register_thread_device_id(struct registered_thread *new_thread, +void vdo_register_thread_device_id(struct registered_thread *new_thread, unsigned int *id_ptr); -void uds_unregister_thread_device_id(void); +void vdo_unregister_thread_device_id(void); -int uds_get_thread_device_id(void); +int vdo_get_thread_device_id(void); -void uds_initialize_thread_device_registry(void); +void vdo_initialize_thread_device_registry(void); -#endif /* UDS_THREAD_DEVICE_H */ +#endif /* VDO_THREAD_DEVICE_H */ From 011568eb3117a1b0e1b2e980de37a4ec47952617 Mon Sep 17 00:00:00 2001 From: Xiaolei Wang Date: Wed, 28 Feb 2024 11:04:08 +0800 Subject: [PATCH 0911/1038] mm/slab: Fix a kmemleak in kmem_cache_destroy() For earlier kmem cache creation, slab_sysfs_init() has not been called. Consequently, kmem_cache_destroy() cannot utilize kobj_type::release to release the kmem_cache structure. Therefore, tweak kmem_cache_release() to use slab_kmem_cache_release() for releasing kmem_cache when slab_state isn't FULL. This will fixes the memory leaks like following: unreferenced object 0xffff0000c2d87080 (size 128): comm "swapper/0", pid 1, jiffies 4294893428 hex dump (first 32 bytes): 00 00 00 00 ad 4e ad de ff ff ff ff 6b 6b 6b 6b .....N......kkkk ff ff ff ff ff ff ff ff b8 ab 48 89 00 80 ff ff.....H..... backtrace (crc 8819d0f6): [] kmemleak_alloc+0xb0/0xc4 [] kmem_cache_alloc_node+0x288/0x3a8 [] __kmem_cache_create+0x1e4/0x64c [] kmem_cache_create_usercopy+0x1c4/0x2cc [] kmem_cache_create+0x1c/0x28 [] arm_v7s_alloc_pgtable+0x1c0/0x6d4 [] alloc_io_pgtable_ops+0xe8/0x2d0 [] arm_v7s_do_selftests+0xe0/0x73c [] do_one_initcall+0x11c/0x7ac [] kernel_init_freeable+0x53c/0xbb8 [] kernel_init+0x24/0x144 [] ret_from_fork+0x10/0x20 Signed-off-by: Xiaolei Wang Reviewed-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slab_common.c | 8 ++++++-- mm/slub.c | 6 ++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index e19544043fdf..7d60cfc2b30f 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -404,8 +404,12 @@ EXPORT_SYMBOL(kmem_cache_create); */ static void kmem_cache_release(struct kmem_cache *s) { - sysfs_slab_unlink(s); - sysfs_slab_release(s); + if (slab_state >= FULL) { + sysfs_slab_unlink(s); + sysfs_slab_release(s); + } else { + slab_kmem_cache_release(s); + } } #else static void kmem_cache_release(struct kmem_cache *s) diff --git a/mm/slub.c b/mm/slub.c index e28929e6e252..0e02e072693b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6794,14 +6794,12 @@ out_del_kobj: void sysfs_slab_unlink(struct kmem_cache *s) { - if (slab_state >= FULL) - kobject_del(&s->kobj); + kobject_del(&s->kobj); } void sysfs_slab_release(struct kmem_cache *s) { - if (slab_state >= FULL) - kobject_put(&s->kobj); + kobject_put(&s->kobj); } /* From 3dd549a557f7dc326d59c5fa105e230ebf3d5458 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Thu, 22 Feb 2024 13:02:33 +0000 Subject: [PATCH 0912/1038] mm, slab: remove the corner case of inc_slabs_node() We already have the inc_slabs_node() after kmem_cache_node->node[node] initialized in early_kmem_cache_node_alloc(), this special case of inc_slabs_node() can be removed. Then we don't need to consider the existence of kmem_cache_node in inc_slabs_node() anymore. Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slub.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 0e02e072693b..12066e69688d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1498,16 +1498,8 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) { struct kmem_cache_node *n = get_node(s, node); - /* - * May be called early in order to allocate a slab for the - * kmem_cache_node structure. Solve the chicken-egg - * dilemma by deferring the increment of the count during - * bootstrap (see early_kmem_cache_node_alloc). - */ - if (likely(n)) { - atomic_long_inc(&n->nr_slabs); - atomic_long_add(objects, &n->total_objects); - } + atomic_long_inc(&n->nr_slabs); + atomic_long_add(objects, &n->total_objects); } static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) { @@ -4855,7 +4847,6 @@ static void early_kmem_cache_node_alloc(int node) slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); BUG_ON(!slab); - inc_slabs_node(kmem_cache_node, slab_nid(slab), slab->objects); if (slab_nid(slab) != node) { pr_err("SLUB: Unable to allocate memory from node %d\n", node); pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); From 78cc80d834ed3b792605bc9c2215ed9e63825459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Lebrun?= Date: Thu, 29 Feb 2024 19:10:50 +0100 Subject: [PATCH 0913/1038] dt-bindings: hwmon: lm75: use common hwmon schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reference common hwmon schema which has the generic "label" property, parsed by Linux hwmon subsystem. To: Jean Delvare To: Guenter Roeck Cc: linux-hwmon@vger.kernel.org Signed-off-by: Théo Lebrun Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240229-mbly-i2c-v2-2-b32ed18c098c@bootlin.com Signed-off-by: Guenter Roeck --- Documentation/devicetree/bindings/hwmon/lm75.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/hwmon/lm75.yaml b/Documentation/devicetree/bindings/hwmon/lm75.yaml index ed269e428a3d..29bd7460cc26 100644 --- a/Documentation/devicetree/bindings/hwmon/lm75.yaml +++ b/Documentation/devicetree/bindings/hwmon/lm75.yaml @@ -57,6 +57,7 @@ required: - reg allOf: + - $ref: hwmon-common.yaml# - if: not: properties: @@ -71,7 +72,7 @@ allOf: properties: interrupts: false -additionalProperties: false +unevaluatedProperties: false examples: - | From c4d61a529db788d2e52654f5b02c8d1de4952c5b Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 29 Feb 2024 13:42:07 +0530 Subject: [PATCH 0914/1038] cpufreq: Don't unregister cpufreq cooling on CPU hotplug Offlining a CPU and bringing it back online is a common operation and it happens frequently during system suspend/resume, where the non-boot CPUs are hotplugged out during suspend and brought back at resume. The cpufreq core already tries to make this path as fast as possible as the changes are only temporary in nature and full cleanup of resources isn't required in this case. For example the drivers can implement online()/offline() callbacks to avoid a lot of tear down of resources. On similar lines, there is no need to unregister the cpufreq cooling device during suspend / resume, but only while the policy is getting removed. Moreover, unregistering the cpufreq cooling device is resulting in an unwanted outcome, where the system suspend is eventually aborted in the process. Currently, during system suspend the cpufreq core unregisters the cooling device, which in turn removes a kobject using device_del() and that generates a notification to the userspace via uevent broadcast. This causes system suspend to abort in some setups. This was also earlier reported (indirectly) by Roman [1]. Maybe there is another way around to fixing that problem properly, but this change makes sense anyways. Move the registering and unregistering of the cooling device to policy creation and removal times onlyy. Closes: https://bugzilla.kernel.org/show_bug.cgi?id=218521 Reported-by: Manaf Meethalavalappu Pallikunhi Reported-by: Roman Stratiienko Link: https://patchwork.kernel.org/project/linux-pm/patch/20220710164026.541466-1-r.stratiienko@gmail.com/ [1] Tested-by: Manaf Meethalavalappu Pallikunhi Signed-off-by: Viresh Kumar Reviewed-by: Dhruva Gole Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 926a51cb7e52..f6f8d7f450e7 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1580,7 +1580,8 @@ static int cpufreq_online(unsigned int cpu) if (cpufreq_driver->ready) cpufreq_driver->ready(policy); - if (cpufreq_thermal_control_enabled(cpufreq_driver)) + /* Register cpufreq cooling only for a new policy */ + if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver)) policy->cdev = of_cpufreq_cooling_register(policy); pr_debug("initialization complete\n"); @@ -1664,11 +1665,6 @@ static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) else policy->last_policy = policy->policy; - if (cpufreq_thermal_control_enabled(cpufreq_driver)) { - cpufreq_cooling_unregister(policy->cdev); - policy->cdev = NULL; - } - if (has_target()) cpufreq_exit_governor(policy); @@ -1729,6 +1725,15 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) return; } + /* + * Unregister cpufreq cooling once all the CPUs of the policy are + * removed. + */ + if (cpufreq_thermal_control_enabled(cpufreq_driver)) { + cpufreq_cooling_unregister(policy->cdev); + policy->cdev = NULL; + } + /* We did light-weight exit earlier, do full tear down now */ if (cpufreq_driver->offline) cpufreq_driver->exit(policy); From 44c9cf9aaa48c308b31dba3001d821d74155fc3d Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 1 Mar 2024 16:18:02 +0800 Subject: [PATCH 0915/1038] powercap: dtpm: Fix kernel-doc for dtpm_create_hierarchy() function The existing comment block above the dtpm_create_hierarchy function does not conform to the kernel-doc standard. This patch fixes the documentation to match the expected kernel-doc format, which includes a structured documentation header with param and return value. Signed-off-by: Yang Li Reviewed-by: Randy Dunlap Signed-off-by: Rafael J. Wysocki --- drivers/powercap/dtpm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/powercap/dtpm.c b/drivers/powercap/dtpm.c index ce920f17f45f..f390665743c4 100644 --- a/drivers/powercap/dtpm.c +++ b/drivers/powercap/dtpm.c @@ -522,7 +522,7 @@ static int dtpm_for_each_child(const struct dtpm_node *hierarchy, /** * dtpm_create_hierarchy - Create the dtpm hierarchy - * @hierarchy: An array of struct dtpm_node describing the hierarchy + * @dtpm_match_table: Pointer to the array of device ID structures * * The function is called by the platform specific code with the * description of the different node in the hierarchy. It creates the From c2dd6b93f6df0a115ae449cfb5ec58a938e3e385 Mon Sep 17 00:00:00 2001 From: Jerome Brunet Date: Wed, 21 Feb 2024 16:11:47 +0100 Subject: [PATCH 0916/1038] dt-bindings: pwm: amlogic: fix s4 bindings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit s4 has been added to the compatible list while converting the Amlogic PWM binding documentation from txt to yaml. However, on the s4, the clock bindings have different meaning compared to the previous SoCs. On the previous SoCs the clock bindings used to describe which input the PWM channel multiplexer should pick among its possible parents. This is very much tied to the driver implementation, instead of describing the HW for what it is. When support for the Amlogic PWM was first added, how to deal with clocks through DT was not as clear as it nowadays. The Linux driver now ignores this DT setting, but still relies on the hard-coded list of clock sources. On the s4, the input multiplexer is gone. The clock bindings actually describe the clock as it exists, not a setting. The property has a different meaning, even if it is still 2 clocks and it would pass the check when support is actually added. Also the s4 cannot work if the clocks are not provided, so the property is no longer optional. Finally, for once it makes sense to see the input as being numbered somehow. No need to bother with clock-names on the s4 type of PWM. Fixes: 43a1c4ff3977 ("dt-bindings: pwm: Convert Amlogic Meson PWM binding") Reviewed-by: Rob Herring Signed-off-by: Jerome Brunet Link: https://lore.kernel.org/r/20240221151154.26452-2-jbrunet@baylibre.com Signed-off-by: Uwe Kleine-König --- .../devicetree/bindings/pwm/pwm-amlogic.yaml | 67 ++++++++++++++++--- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml index 527864a4d855..a1d382aacb82 100644 --- a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml +++ b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml @@ -9,9 +9,6 @@ title: Amlogic PWM maintainers: - Heiner Kallweit -allOf: - - $ref: pwm.yaml# - properties: compatible: oneOf: @@ -43,12 +40,8 @@ properties: maxItems: 2 clock-names: - oneOf: - - items: - - enum: [clkin0, clkin1] - - items: - - const: clkin0 - - const: clkin1 + minItems: 1 + maxItems: 2 "#pwm-cells": const: 3 @@ -57,6 +50,55 @@ required: - compatible - reg +allOf: + - $ref: pwm.yaml# + + - if: + properties: + compatible: + contains: + enum: + - amlogic,meson8-pwm + - amlogic,meson8b-pwm + - amlogic,meson-gxbb-pwm + - amlogic,meson-gxbb-ao-pwm + - amlogic,meson-axg-ee-pwm + - amlogic,meson-axg-ao-pwm + - amlogic,meson-g12a-ee-pwm + - amlogic,meson-g12a-ao-pwm-ab + - amlogic,meson-g12a-ao-pwm-cd + then: + # Historic bindings tied to the driver implementation + # The clocks provided here are meant to be matched with the input + # known (hard-coded) in the driver and used to select pwm clock + # source. Currently, the linux driver ignores this. + properties: + clock-names: + oneOf: + - items: + - enum: [clkin0, clkin1] + - items: + - const: clkin0 + - const: clkin1 + + # Newer IP block take a single input per channel, instead of 4 inputs + # for both channels + - if: + properties: + compatible: + contains: + enum: + - amlogic,meson-s4-pwm + then: + properties: + clocks: + items: + - description: input clock of PWM channel A + - description: input clock of PWM channel B + clock-names: false + required: + - clocks + additionalProperties: false examples: @@ -68,3 +110,10 @@ examples: clock-names = "clkin0", "clkin1"; #pwm-cells = <3>; }; + - | + pwm@1000 { + compatible = "amlogic,meson-s4-pwm"; + reg = <0x1000 0x10>; + clocks = <&pwm_src_a>, <&pwm_src_b>; + #pwm-cells = <3>; + }; From 5fd61cc28171e0c0fa2809be04f242c917445e6d Mon Sep 17 00:00:00 2001 From: Jerome Brunet Date: Wed, 21 Feb 2024 16:11:48 +0100 Subject: [PATCH 0917/1038] dt-bindings: pwm: amlogic: Add a new binding for meson8 pwm types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The binding that is used up to now describe which input the PWM channel multiplexer should pick among its possible parents, which are hardcoded in the driver. This isn't a good binding in the sense that it should describe hardware but not usage. Add a new binding deprecating the old one that uses clocks in a better way and how clocks are usually used today: The list of clocks describe the inputs of the PWM block as they are realised in hardware. So deprecate the old bindings and introduce a compatible per SoC family to replace these. Signed-off-by: Jerome Brunet Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240221151154.26452-3-jbrunet@baylibre.com Signed-off-by: Uwe Kleine-König --- .../devicetree/bindings/pwm/pwm-amlogic.yaml | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml index a1d382aacb82..1d71d4f8f328 100644 --- a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml +++ b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml @@ -21,23 +21,36 @@ properties: - amlogic,meson-g12a-ee-pwm - amlogic,meson-g12a-ao-pwm-ab - amlogic,meson-g12a-ao-pwm-cd - - amlogic,meson-s4-pwm + deprecated: true - items: - const: amlogic,meson-gx-pwm - const: amlogic,meson-gxbb-pwm + deprecated: true - items: - const: amlogic,meson-gx-ao-pwm - const: amlogic,meson-gxbb-ao-pwm + deprecated: true - items: - const: amlogic,meson8-pwm - const: amlogic,meson8b-pwm + deprecated: true + - enum: + - amlogic,meson8-pwm-v2 + - amlogic,meson-s4-pwm + - items: + - enum: + - amlogic,meson8b-pwm-v2 + - amlogic,meson-gxbb-pwm-v2 + - amlogic,meson-axg-pwm-v2 + - amlogic,meson-g12-pwm-v2 + - const: amlogic,meson8-pwm-v2 reg: maxItems: 1 clocks: minItems: 1 - maxItems: 2 + maxItems: 4 clock-names: minItems: 1 @@ -68,11 +81,14 @@ allOf: - amlogic,meson-g12a-ao-pwm-ab - amlogic,meson-g12a-ao-pwm-cd then: - # Historic bindings tied to the driver implementation + # Obsolete historic bindings tied to the driver implementation # The clocks provided here are meant to be matched with the input # known (hard-coded) in the driver and used to select pwm clock # source. Currently, the linux driver ignores this. + # This is kept to maintain ABI backward compatibility. properties: + clocks: + maxItems: 2 clock-names: oneOf: - items: @@ -81,6 +97,27 @@ allOf: - const: clkin0 - const: clkin1 + # Newer binding where clock describe the actual clock inputs of the pwm + # block. These are necessary but some inputs may be grounded. + - if: + properties: + compatible: + contains: + enum: + - amlogic,meson8-pwm-v2 + then: + properties: + clocks: + minItems: 1 + items: + - description: input clock 0 of the pwm block + - description: input clock 1 of the pwm block + - description: input clock 2 of the pwm block + - description: input clock 3 of the pwm block + clock-names: false + required: + - clocks + # Newer IP block take a single input per channel, instead of 4 inputs # for both channels - if: @@ -110,6 +147,13 @@ examples: clock-names = "clkin0", "clkin1"; #pwm-cells = <3>; }; + - | + pwm@2000 { + compatible = "amlogic,meson8-pwm-v2"; + reg = <0x1000 0x10>; + clocks = <&xtal>, <0>, <&fdiv4>, <&fdiv5>; + #pwm-cells = <3>; + }; - | pwm@1000 { compatible = "amlogic,meson-s4-pwm"; From f2cea1dc2a9818e65bccfe27c4befaaa5274a700 Mon Sep 17 00:00:00 2001 From: Jerome Brunet Date: Wed, 21 Feb 2024 16:11:49 +0100 Subject: [PATCH 0918/1038] pwm: meson: generalize 4 inputs clock on meson8 pwm type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Meson8 pwm type always has 4 input clocks. Some inputs may be grounded, like in the AO domain of some SoCs. Drop the parent number parameter and make this is constant. This is also done to make the addition of generic meson8 compatible easier. Signed-off-by: Jerome Brunet Link: https://lore.kernel.org/r/20240221151154.26452-4-jbrunet@baylibre.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-meson.c | 53 +++++++++-------------------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c index 40a5b64c26f5..a02fdbc61256 100644 --- a/drivers/pwm/pwm-meson.c +++ b/drivers/pwm/pwm-meson.c @@ -60,7 +60,7 @@ #define MISC_A_EN BIT(0) #define MESON_NUM_PWMS 2 -#define MESON_MAX_MUX_PARENTS 4 +#define MESON_NUM_MUX_PARENTS 4 static struct meson_pwm_channel_data { u8 reg_offset; @@ -97,8 +97,7 @@ struct meson_pwm_channel { }; struct meson_pwm_data { - const char * const *parent_names; - unsigned int num_parents; + const char *const parent_names[MESON_NUM_MUX_PARENTS]; }; struct meson_pwm { @@ -339,62 +338,32 @@ static const struct pwm_ops meson_pwm_ops = { .get_state = meson_pwm_get_state, }; -static const char * const pwm_meson8b_parent_names[] = { - "xtal", NULL, "fclk_div4", "fclk_div3" -}; - static const struct meson_pwm_data pwm_meson8b_data = { - .parent_names = pwm_meson8b_parent_names, - .num_parents = ARRAY_SIZE(pwm_meson8b_parent_names), + .parent_names = { "xtal", NULL, "fclk_div4", "fclk_div3" }, }; /* * Only the 2 first inputs of the GXBB AO PWMs are valid * The last 2 are grounded */ -static const char * const pwm_gxbb_ao_parent_names[] = { - "xtal", "clk81" -}; - static const struct meson_pwm_data pwm_gxbb_ao_data = { - .parent_names = pwm_gxbb_ao_parent_names, - .num_parents = ARRAY_SIZE(pwm_gxbb_ao_parent_names), -}; - -static const char * const pwm_axg_ee_parent_names[] = { - "xtal", "fclk_div5", "fclk_div4", "fclk_div3" + .parent_names = { "xtal", "clk81", NULL, NULL }, }; static const struct meson_pwm_data pwm_axg_ee_data = { - .parent_names = pwm_axg_ee_parent_names, - .num_parents = ARRAY_SIZE(pwm_axg_ee_parent_names), -}; - -static const char * const pwm_axg_ao_parent_names[] = { - "xtal", "axg_ao_clk81", "fclk_div4", "fclk_div5" + .parent_names = { "xtal", "fclk_div5", "fclk_div4", "fclk_div3" }, }; static const struct meson_pwm_data pwm_axg_ao_data = { - .parent_names = pwm_axg_ao_parent_names, - .num_parents = ARRAY_SIZE(pwm_axg_ao_parent_names), -}; - -static const char * const pwm_g12a_ao_ab_parent_names[] = { - "xtal", "g12a_ao_clk81", "fclk_div4", "fclk_div5" + .parent_names = { "xtal", "axg_ao_clk81", "fclk_div4", "fclk_div5" }, }; static const struct meson_pwm_data pwm_g12a_ao_ab_data = { - .parent_names = pwm_g12a_ao_ab_parent_names, - .num_parents = ARRAY_SIZE(pwm_g12a_ao_ab_parent_names), -}; - -static const char * const pwm_g12a_ao_cd_parent_names[] = { - "xtal", "g12a_ao_clk81", + .parent_names = { "xtal", "g12a_ao_clk81", "fclk_div4", "fclk_div5" }, }; static const struct meson_pwm_data pwm_g12a_ao_cd_data = { - .parent_names = pwm_g12a_ao_cd_parent_names, - .num_parents = ARRAY_SIZE(pwm_g12a_ao_cd_parent_names), + .parent_names = { "xtal", "g12a_ao_clk81", NULL, NULL }, }; static const struct of_device_id meson_pwm_matches[] = { @@ -437,13 +406,13 @@ MODULE_DEVICE_TABLE(of, meson_pwm_matches); static int meson_pwm_init_channels(struct pwm_chip *chip) { struct meson_pwm *meson = to_meson_pwm(chip); - struct clk_parent_data mux_parent_data[MESON_MAX_MUX_PARENTS] = {}; + struct clk_parent_data mux_parent_data[MESON_NUM_MUX_PARENTS] = {}; struct device *dev = pwmchip_parent(chip); unsigned int i; char name[255]; int err; - for (i = 0; i < meson->data->num_parents; i++) { + for (i = 0; i < MESON_NUM_MUX_PARENTS; i++) { mux_parent_data[i].index = -1; mux_parent_data[i].name = meson->data->parent_names[i]; } @@ -459,7 +428,7 @@ static int meson_pwm_init_channels(struct pwm_chip *chip) init.ops = &clk_mux_ops; init.flags = 0; init.parent_data = mux_parent_data; - init.num_parents = meson->data->num_parents; + init.num_parents = MESON_NUM_MUX_PARENTS; channel->mux.reg = meson->base + REG_MISC_AB; channel->mux.shift = From fb6ad4aec1d02079250c5935b6946b216e048434 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 29 Jan 2024 23:11:54 -1000 Subject: [PATCH 0919/1038] dm-crypt: Convert from tasklet to BH workqueue The only generic interface to execute asynchronously in the BH context is tasklet; however, it's marked deprecated and has some design flaws. To replace tasklets, BH workqueue support was recently added. A BH workqueue behaves similarly to regular workqueues except that the queued work items are executed in the BH context. This commit converts dm-crypt from tasklet to BH workqueue. It backfills tasklet code that was removed with commit 0a9bab391e33 ("dm-crypt, dm-verity: disable tasklets") and tweaks to use BH workqueue. Like a regular workqueue, a BH workqueue allows freeing the currently executing work item. Converting from tasklet to BH workqueue removes the need for deferring bio_endio() again to a work item, which was buggy anyway. I tested this lightly with "--perf-no_read_workqueue --perf-no_write_workqueue" + some code modifications, but would really -appreciate if someone who knows the code base better could take a look. Signed-off-by: Tejun Heo Link: http://lkml.kernel.org/r/82b964f0-c2c8-a2c6-5b1f-f3145dc2c8e5@redhat.com [snitzer: rebase ontop of commit 0a9bab391e33 reduced this commit's changes] Signed-off-by: Mike Snitzer --- drivers/md/dm-crypt.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 50467f005177..9a74c6316c5d 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2296,7 +2296,11 @@ static void kcryptd_queue_crypt(struct dm_crypt_io *io) * irqs_disabled(): the kernel may run some IO completion from the idle thread, but * it is being executed with irqs disabled. */ - if (!(in_hardirq() || irqs_disabled())) { + if (in_hardirq() || irqs_disabled()) { + INIT_WORK(&io->work, kcryptd_crypt); + queue_work(system_bh_wq, &io->work); + return; + } else { kcryptd_crypt(&io->work); return; } From c375b223338828f29aed76625b33be6d3a21f8af Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 29 Jan 2024 23:11:55 -1000 Subject: [PATCH 0920/1038] dm-verity: Convert from tasklet to BH workqueue The only generic interface to execute asynchronously in the BH context is tasklet; however, it's marked deprecated and has some design flaws. To replace tasklets, BH workqueue support was recently added. A BH workqueue behaves similarly to regular workqueues except that the queued work items are executed in the BH context. This commit converts dm-verity from tasklet to BH workqueue. It backfills tasklet code that was removed with commit 0a9bab391e33 ("dm-crypt, dm-verity: disable tasklets") and tweaks to use BH workqueue (and does some renaming). This is a minimal conversion which doesn't rename the related names including the "try_verify_in_tasklet" option. If this patch is applied, a follow-up patch would be necessary. I couldn't decide whether the option name would need to be updated too. Signed-off-by: Tejun Heo [snitzer: rename 'use_tasklet' to 'use_bh_wq' and 'in_tasklet' to 'in_bh'] Signed-off-by: Mike Snitzer --- drivers/md/dm-verity-target.c | 64 +++++++++++++++++++++++------------ drivers/md/dm-verity.h | 5 +-- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 6b8f9698488e..b6f62d6820ba 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -46,7 +46,7 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); -static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); +static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled); struct dm_verity_prefetch_work { struct work_struct work; @@ -299,7 +299,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, verity_hash_at_level(v, block, level, &hash_block, &offset); - if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { + if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { data = dm_bufio_get(v->bufio, hash_block, &buf); if (data == NULL) { /* @@ -327,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, r = verity_hash(v, verity_io_hash_req(v, io), data, 1 << v->hash_dev_block_bits, - verity_io_real_digest(v, io), !io->in_tasklet); + verity_io_real_digest(v, io), !io->in_bh); if (unlikely(r < 0)) goto release_ret_r; if (likely(memcmp(verity_io_real_digest(v, io), want_digest, v->digest_size) == 0)) aux->hash_verified = 1; - else if (static_branch_unlikely(&use_tasklet_enabled) && - io->in_tasklet) { + else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Error handling code (FEC included) cannot be run in a * tasklet since it may sleep, so fallback to work-queue. @@ -576,7 +575,7 @@ static int verity_verify_io(struct dm_verity_io *io) struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); unsigned int b; - if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { + if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Copy the iterator in case we need to restart * verification in a work-queue. @@ -616,7 +615,7 @@ static int verity_verify_io(struct dm_verity_io *io) continue; } - r = verity_hash_init(v, req, &wait, !io->in_tasklet); + r = verity_hash_init(v, req, &wait, !io->in_bh); if (unlikely(r < 0)) return r; @@ -635,8 +634,7 @@ static int verity_verify_io(struct dm_verity_io *io) if (v->validated_blocks) set_bit(cur_block, v->validated_blocks); continue; - } else if (static_branch_unlikely(&use_tasklet_enabled) && - io->in_tasklet) { + } else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Error handling code (FEC included) cannot be run in a * tasklet since it may sleep, so fallback to work-queue. @@ -690,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) bio->bi_end_io = io->orig_bi_end_io; bio->bi_status = status; - if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet) + if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) verity_fec_finish_io(io); bio_endio(bio); @@ -700,11 +698,28 @@ static void verity_work(struct work_struct *w) { struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); - io->in_tasklet = false; + io->in_bh = false; verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); } +static void verity_bh_work(struct work_struct *w) +{ + struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work); + int err; + + io->in_bh = true; + err = verity_verify_io(io); + if (err == -EAGAIN || err == -ENOMEM) { + /* fallback to retrying with work-queue */ + INIT_WORK(&io->work, verity_work); + queue_work(io->v->verify_wq, &io->work); + return; + } + + verity_finish_io(io, errno_to_blk_status(err)); +} + static void verity_end_io(struct bio *bio) { struct dm_verity_io *io = bio->bi_private; @@ -717,8 +732,13 @@ static void verity_end_io(struct bio *bio) return; } - INIT_WORK(&io->work, verity_work); - queue_work(io->v->verify_wq, &io->work); + if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) { + INIT_WORK(&io->bh_work, verity_bh_work); + queue_work(system_bh_wq, &io->bh_work); + } else { + INIT_WORK(&io->work, verity_work); + queue_work(io->v->verify_wq, &io->work); + } } /* @@ -885,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, args++; if (v->validated_blocks) args++; - if (v->use_tasklet) + if (v->use_bh_wq) args++; if (v->signature_key_desc) args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; @@ -912,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); if (v->validated_blocks) DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); - if (v->use_tasklet) + if (v->use_bh_wq) DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); sz = verity_fec_status_table(v, sz, result, maxlen); if (v->signature_key_desc) @@ -1031,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti) kfree(v->signature_key_desc); - if (v->use_tasklet) - static_branch_dec(&use_tasklet_enabled); + if (v->use_bh_wq) + static_branch_dec(&use_bh_wq_enabled); kfree(v); @@ -1166,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, continue; } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { - v->use_tasklet = true; - static_branch_inc(&use_tasklet_enabled); + v->use_bh_wq = true; + static_branch_inc(&use_bh_wq_enabled); continue; } else if (verity_is_fec_opt_arg(arg_name)) { @@ -1338,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) } v->tfm = crypto_alloc_ahash(v->alg_name, 0, - v->use_tasklet ? CRYPTO_ALG_ASYNC : 0); + v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0); if (IS_ERR(v->tfm)) { ti->error = "Cannot initialize hash function"; r = PTR_ERR(v->tfm); @@ -1463,7 +1483,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), dm_bufio_alloc_callback, NULL, - v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0); + v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0); if (IS_ERR(v->bufio)) { ti->error = "Cannot initialize dm-bufio"; r = PTR_ERR(v->bufio); @@ -1482,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) * reducing wait times when reading from a dm-verity device. * * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI - * allows verify_wq to preempt softirq since verification in tasklet + * allows verify_wq to preempt softirq since verification in BH workqueue * will fall-back to using it for error handling (or if the bufio cache * doesn't have required hashes). */ diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index db93a91169d5..20b1bcf03474 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -54,7 +54,7 @@ struct dm_verity { unsigned char levels; /* the number of tree levels */ unsigned char version; bool hash_failed:1; /* set if hash of any block failed */ - bool use_tasklet:1; /* try to verify in tasklet before work-queue */ + bool use_bh_wq:1; /* try to verify in BH wq before normal work-queue */ unsigned int digest_size; /* digest size for the current hash algorithm */ unsigned int ahash_reqsize;/* the size of temporary space for crypto */ enum verity_mode mode; /* mode for handling verification errors */ @@ -84,9 +84,10 @@ struct dm_verity_io { sector_t block; unsigned int n_blocks; - bool in_tasklet; + bool in_bh; struct work_struct work; + struct work_struct bh_work; char *recheck_buffer; From 17cce771c5fc85f43680143ac8b3b944fdad113f Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 1 Mar 2024 17:08:09 +0100 Subject: [PATCH 0921/1038] mm, slab: remove memcg_from_slab_obj() This empty wrapped exists only for !CONFIG_MEMCG_KMEM and seems it was never used. Probably a leftover from development of a series. Reviewed-by: Chengming Zhou Reviewed-by: Roman Gushchin Acked-by: David Rientjes Signed-off-by: Vlastimil Babka --- mm/slub.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 12066e69688d..186a8ec28228 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2028,11 +2028,6 @@ void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects, obj_cgroup_uncharge(objcg, objects * obj_full_size(s)); } #else /* CONFIG_MEMCG_KMEM */ -static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) -{ - return NULL; -} - static inline void memcg_free_slab_cgroups(struct slab *slab) { } From dd6c6d57ab61d496f6ff7d6ca38611062af142a1 Mon Sep 17 00:00:00 2001 From: Dong Aisheng Date: Mon, 4 Mar 2024 18:29:29 +0800 Subject: [PATCH 0922/1038] pwm: imx-tpm: fix probe crash due to access registers without clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following commit abf6569d6482 ("pwm: imx-tpm: Make use of devm_pwmchip_alloc() function") introduced an issue that accessing registers without clock which results in the following boot crash on MX7ULP platform. Fixed it by enabling clock properly. Unhandled fault: external abort on non-linefetch (0x1008) at 0xf0978004 [f0978004] *pgd=64009811, *pte=40250653, *ppte=40250453 Internal error: : 1008 [#1] SMP ARM Modules linked in: CPU: 0 PID: 1 Comm: swapper/0 Not tainted 6.8.0-rc6-next-20240301 #18 Hardware name: Freescale i.MX7ULP (Device Tree) PC is at pwm_imx_tpm_probe+0x1c/0xd8 LR is at __devm_ioremap_resource+0xf8/0x1dc pc : [] lr : [] psr: 80000053 sp : f0825e10 ip : 00000000 fp : 00000000 r10: c148f8c0 r9 : c41fc338 r8 : c164b000 r7 : 00000000 r6 : c406b400 r5 : c406b410 r4 : f0978000 r3 : 00000005 r2 : 00000000 r1 : a0000053 r0 : f0978000 Flags: Nzcv IRQs on FIQs off Mode SVC_32 ISA ARM Segment none Control: 10c5387d Table: 6000406a DAC: 00000051 ... Call trace: pwm_imx_tpm_probe from platform_probe+0x58/0xb0 platform_probe from really_probe+0xc4/0x2e0 really_probe from __driver_probe_device+0x84/0x19c __driver_probe_device from driver_probe_device+0x2c/0x104 driver_probe_device from __driver_attach+0x90/0x170 __driver_attach from bus_for_each_dev+0x7c/0xd0 bus_for_each_dev from bus_add_driver+0xc4/0x1cc bus_add_driver from driver_register+0x7c/0x114 driver_register from do_one_initcall+0x58/0x270 do_one_initcall from kernel_init_freeable+0x170/0x218 kernel_init_freeable from kernel_init+0x14/0x140 kernel_init from ret_from_fork+0x14/0x20 Fixes: abf6569d6482 ("pwm: imx-tpm: Make use of devm_pwmchip_alloc() function") Signed-off-by: Dong Aisheng Link: https://lore.kernel.org/r/20240304102929.893542-1-aisheng.dong@nxp.com Signed-off-by: Uwe Kleine-König --- drivers/pwm/pwm-imx-tpm.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c index b04b974c5f32..c50ddbac43c8 100644 --- a/drivers/pwm/pwm-imx-tpm.c +++ b/drivers/pwm/pwm-imx-tpm.c @@ -337,6 +337,7 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev) { struct pwm_chip *chip; struct imx_tpm_pwm_chip *tpm; + struct clk *clk; void __iomem *base; int ret; unsigned int npwm; @@ -346,6 +347,11 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); + clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk), + "failed to get PWM clock\n"); + /* get number of channels */ val = readl(base + PWM_IMX_TPM_PARAM); npwm = FIELD_GET(PWM_IMX_TPM_PARAM_CHAN, val); @@ -358,11 +364,7 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev) platform_set_drvdata(pdev, tpm); tpm->base = base; - - tpm->clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(tpm->clk)) - return dev_err_probe(&pdev->dev, PTR_ERR(tpm->clk), - "failed to get PWM clock\n"); + tpm->clk = clk; chip->ops = &imx_tpm_pwm_ops; From ee09bb727bff1f14f3f2d81592741b8a081af2ee Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 1 Mar 2024 11:55:46 +0000 Subject: [PATCH 0923/1038] spi: dt-bindings: samsung: make dma properties not required Since the addition of the driver in 2009, the driver selects between DMA and polling mode depending on the transfer length - DMA mode for transfers bigger than the FIFO depth, polling mode otherwise. All versions of the IP support polling mode, make the dma properties not required. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240301115546.2266676-1-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- Documentation/devicetree/bindings/spi/samsung,spi.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Documentation/devicetree/bindings/spi/samsung,spi.yaml b/Documentation/devicetree/bindings/spi/samsung,spi.yaml index 2f0a0835ecfb..f681372da81f 100644 --- a/Documentation/devicetree/bindings/spi/samsung,spi.yaml +++ b/Documentation/devicetree/bindings/spi/samsung,spi.yaml @@ -76,8 +76,6 @@ required: - compatible - clocks - clock-names - - dmas - - dma-names - interrupts - reg From 69d54ee2e5b0dab9350be2a7019c472b9b8d4c14 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 4 Mar 2024 10:04:23 -0600 Subject: [PATCH 0924/1038] spi: axi-spi-engine: remove p from struct spi_engine_message_state The program pointer p in struct spi_engine_message_state in the AXI SPI Engine controller driver was assigned but never read so it can be removed. Reviewed-by: Kees Cook Signed-off-by: David Lechner Reviewed-by: Gustavo A. R. Silva Link: https://msgid.link/r/20240304-mainline-axi-spi-engine-small-cleanups-v2-1-5b14ed729a31@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index 6177c1a8d56e..d89f75170c9e 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -82,8 +82,6 @@ struct spi_engine_program { * struct spi_engine_message_state - SPI engine per-message state */ struct spi_engine_message_state { - /** @p: Instructions for executing this message. */ - struct spi_engine_program *p; /** @cmd_length: Number of elements in cmd_buf array. */ unsigned cmd_length; /** @cmd_buf: Array of commands not yet written to CMD FIFO. */ @@ -543,7 +541,6 @@ static int spi_engine_transfer_one_message(struct spi_controller *host, /* reinitialize message state for this transfer */ memset(st, 0, sizeof(*st)); - st->p = p; st->cmd_buf = p->instructions; st->cmd_length = p->length; msg->state = st; From c8340ac1015471ec5af234beff535efe15f382e9 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 4 Mar 2024 10:04:24 -0600 Subject: [PATCH 0925/1038] spi: axi-spi-engine: use __counted_by() attribute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the __counted_by() attribute to the flex array at the end of struct spi_engine_program in the AXI SPI Engine controller driver. The assignment of the length field has to be reordered to be before the access to the flex array in order to avoid potential compiler warnings/errors due to adding the __counted_by() attribute. Suggested-by: Nuno Sá Signed-off-by: David Lechner Reviewed-by: Gustavo A. R. Silva Reviewed-by: Kees Cook Link: https://msgid.link/r/20240304-mainline-axi-spi-engine-small-cleanups-v2-2-5b14ed729a31@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index d89f75170c9e..a8f626165f44 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -75,7 +75,7 @@ struct spi_engine_program { unsigned int length; - uint16_t instructions[]; + uint16_t instructions[] __counted_by(length); }; /** @@ -115,9 +115,10 @@ struct spi_engine { static void spi_engine_program_add_cmd(struct spi_engine_program *p, bool dry, uint16_t cmd) { - if (!dry) - p->instructions[p->length] = cmd; p->length++; + + if (!dry) + p->instructions[p->length - 1] = cmd; } static unsigned int spi_engine_get_config(struct spi_device *spi) From 5c708541301e695b611cb0b9c6d732bed9b5d904 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Mon, 4 Mar 2024 10:04:25 -0600 Subject: [PATCH 0926/1038] spi: axi-spi-engine: use struct_size() macro This makes use of the struct_size() macro to calculate the size of the struct axi_spi_engine when allocating it. Suggested-by: Christophe JAILLET Reviewed-by: Kees Cook Signed-off-by: David Lechner Reviewed-by: Gustavo A. R. Silva Link: https://msgid.link/r/20240304-mainline-axi-spi-engine-small-cleanups-v2-3-5b14ed729a31@baylibre.com Signed-off-by: Mark Brown --- drivers/spi/spi-axi-spi-engine.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index a8f626165f44..7cc219d78551 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -502,15 +503,13 @@ static irqreturn_t spi_engine_irq(int irq, void *devid) static int spi_engine_optimize_message(struct spi_message *msg) { struct spi_engine_program p_dry, *p; - size_t size; spi_engine_precompile_message(msg); p_dry.length = 0; spi_engine_compile_message(msg, true, &p_dry); - size = sizeof(*p->instructions) * (p_dry.length + 1); - p = kzalloc(sizeof(*p) + size, GFP_KERNEL); + p = kzalloc(struct_size(p, instructions, p_dry.length + 1), GFP_KERNEL); if (!p) return -ENOMEM; From dcd1332bb5ff996bc92a63948ba32bceae40507c Mon Sep 17 00:00:00 2001 From: Susan LeGendre-McGhee Date: Thu, 15 Feb 2024 11:35:15 -0500 Subject: [PATCH 0927/1038] dm vdo: remove internal ticket references Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 8 ++++---- drivers/md/dm-vdo/data-vio.c | 9 +++++---- drivers/md/dm-vdo/dm-vdo-target.c | 12 +++++------- drivers/md/dm-vdo/memory-alloc.c | 8 ++++---- drivers/md/dm-vdo/packer.c | 16 +++++++--------- drivers/md/dm-vdo/packer.h | 2 +- drivers/md/dm-vdo/repair.c | 4 ++-- drivers/md/dm-vdo/slab-depot.c | 16 +++++++++++----- drivers/md/dm-vdo/sparse-cache.c | 2 +- drivers/md/dm-vdo/vdo.c | 2 +- drivers/md/dm-vdo/vio.c | 1 - 11 files changed, 41 insertions(+), 39 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index e3fadb5f2c2d..5be400743c03 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -542,7 +542,7 @@ static unsigned int distribute_page_over_waitq(struct page_info *info, /* * Increment the busy count once for each pending completion so that this page does not - * stop being busy until all completions have been processed (VDO-83). + * stop being busy until all completions have been processed. */ info->busy += num_pages; @@ -1097,9 +1097,9 @@ static void write_pages(struct vdo_completion *flush_completion) struct vdo_page_cache *cache = ((struct page_info *) flush_completion->parent)->cache; /* - * We need to cache these two values on the stack since in the error case below, it is - * possible for the last page info to cause the page cache to get freed. Hence once we - * launch the last page, it may be unsafe to dereference the cache [VDO-4724]. + * We need to cache these two values on the stack since it is possible for the last + * page info to cause the page cache to get freed. Hence once we launch the last page, + * it may be unsafe to dereference the cache. */ bool has_unflushed_pages = (cache->pages_to_flush > 0); page_count_t pages_in_flush = cache->pages_in_flush; diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index d77adeb5006e..f6c32dc9a822 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -453,10 +453,11 @@ static void attempt_logical_block_lock(struct vdo_completion *completion) /* * If the new request is a pure read request (not read-modify-write) and the lock_holder is - * writing and has received an allocation (VDO-2683), service the read request immediately - * by copying data from the lock_holder to avoid having to flush the write out of the - * packer just to prevent the read from waiting indefinitely. If the lock_holder does not - * yet have an allocation, prevent it from blocking in the packer and wait on it. + * writing and has received an allocation, service the read request immediately by copying + * data from the lock_holder to avoid having to flush the write out of the packer just to + * prevent the read from waiting indefinitely. If the lock_holder does not yet have an + * allocation, prevent it from blocking in the packer and wait on it. This is necessary in + * order to prevent returning data that may not have actually been written. */ if (!data_vio->write && READ_ONCE(lock_holder->allocation_succeeded)) { copy_to_bio(data_vio->user_bio, lock_holder->vio.data + data_vio->offset); diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 7afd1dfec649..0114fa4d48a2 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -945,13 +945,11 @@ static void vdo_io_hints(struct dm_target *ti, struct queue_limits *limits) * Sets the maximum discard size that will be passed into VDO. This value comes from a * table line value passed in during dmsetup create. * - * The value 1024 is the largest usable value on HD systems. A 2048 sector discard on a - * busy HD system takes 31 seconds. We should use a value no higher than 1024, which takes - * 15 to 16 seconds on a busy HD system. - * - * But using large values results in 120 second blocked task warnings in /var/log/kern.log. - * In order to avoid these warnings, we choose to use the smallest reasonable value. See - * VDO-3062 and VDO-3087. + * The value 1024 is the largest usable value on HD systems. A 2048 sector discard on a + * busy HD system takes 31 seconds. We should use a value no higher than 1024, which takes + * 15 to 16 seconds on a busy HD system. However, using large values results in 120 second + * blocked task warnings in kernel logs. In order to avoid these warnings, we choose to + * use the smallest reasonable value. * * The value is displayed in sysfs, and also used by dm-thin to determine whether to pass * down discards. The block layer splits large discards on this boundary when this is set. diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index 3b2bda9248cb..5cd387f9294e 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -235,8 +235,8 @@ int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) if (p == NULL) { /* * It is possible for kmalloc to fail to allocate memory because there is - * no page available (see VDO-3688). A short sleep may allow the page - * reclaimer to free a page. + * no page available. A short sleep may allow the page reclaimer to + * free a page. */ fsleep(1000); p = kmalloc(size, gfp_flags); @@ -251,8 +251,8 @@ int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) UDS_SUCCESS) { /* * It is possible for __vmalloc to fail to allocate memory because there - * are no pages available (see VDO-3661). A short sleep may allow the page - * reclaimer to free enough pages for a small allocation. + * are no pages available. A short sleep may allow the page reclaimer + * to free enough pages for a small allocation. * * For larger allocations, the page_alloc code is racing against the page * reclaimer. If the page reclaimer can stay ahead of page_alloc, the diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c index e391cac6c92d..b0ffb21ec436 100644 --- a/drivers/md/dm-vdo/packer.c +++ b/drivers/md/dm-vdo/packer.c @@ -595,15 +595,13 @@ void vdo_attempt_packing(struct data_vio *data_vio) } /* - * The check of may_vio_block_in_packer() here will set the data_vio's compression state to - * VIO_PACKING if the data_vio is allowed to be compressed (if it has already been - * canceled, we'll fall out here). Once the data_vio is in the VIO_PACKING state, it must - * be guaranteed to be put in a bin before any more requests can be processed by the packer - * thread. Otherwise, a canceling data_vio could attempt to remove the canceled data_vio - * from the packer and fail to rendezvous with it (VDO-2809). We must also make sure that - * we will actually bin the data_vio and not give up on it as being larger than the space - * used in the fullest bin. Hence we must call select_bin() before calling - * may_vio_block_in_packer() (VDO-2826). + * The advance_data_vio_compression_stage() check here verifies that the data_vio is + * allowed to be compressed (if it has already been canceled, we'll fall out here). Once + * the data_vio is in the DATA_VIO_PACKING state, it must be guaranteed to be put in a bin + * before any more requests can be processed by the packer thread. Otherwise, a canceling + * data_vio could attempt to remove the canceled data_vio from the packer and fail to + * rendezvous with it. Thus, we must call select_bin() first to ensure that we will + * actually add the data_vio to a bin before advancing to the DATA_VIO_PACKING stage. */ bin = select_bin(packer, data_vio); if ((bin == NULL) || diff --git a/drivers/md/dm-vdo/packer.h b/drivers/md/dm-vdo/packer.h index 2dcc40bd4417..0f3be44710b5 100644 --- a/drivers/md/dm-vdo/packer.h +++ b/drivers/md/dm-vdo/packer.h @@ -58,7 +58,7 @@ struct compressed_block { * * There is one special bin which is used to hold data_vios which have been canceled and removed * from their bin by the packer. These data_vios need to wait for the canceller to rendezvous with - * them (VDO-2809) and so they sit in this special bin. + * them and so they sit in this special bin. */ struct packer_bin { /* List links for packer.packer_bins */ diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index a75278eb8aa4..847aca9fbe47 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -1504,8 +1504,8 @@ static int extract_new_mappings(struct repair_completion *repair) static noinline int compute_usages(struct repair_completion *repair) { /* - * VDO-5182: function is declared noinline to avoid what is likely a spurious valgrind - * error about this structure being uninitialized. + * This function is declared noinline to avoid a spurious valgrind error regarding the + * following structure being uninitialized. */ struct recovery_point recovery_point = { .sequence_number = repair->tail, diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 42126bd60242..5fa7e0838b32 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -334,7 +334,11 @@ static void launch_write(struct slab_summary_block *block) /* * Flush before writing to ensure that the slab journal tail blocks and reference updates - * covered by this summary update are stable (VDO-2332). + * covered by this summary update are stable. Otherwise, a subsequent recovery could + * encounter a slab summary update that refers to a slab journal tail block that has not + * actually been written. In such cases, the slab journal referenced will be treated as + * empty, causing any data within the slab which predates the existing recovery journal + * entries to be lost. */ pbn = (depot->summary_origin + (VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * allocator->zone_number) + @@ -499,7 +503,7 @@ static void reap_slab_journal(struct slab_journal *journal) * journal block writes can be issued while previous slab summary updates have not yet been * made. Even though those slab journal block writes will be ignored if the slab summary * update is not persisted, they may still overwrite the to-be-reaped slab journal block - * resulting in a loss of reference count updates (VDO-2912). + * resulting in a loss of reference count updates. */ journal->flush_waiter.callback = flush_for_reaping; acquire_vio_from_pool(journal->slab->allocator->vio_pool, @@ -770,7 +774,8 @@ static void write_slab_journal_block(struct vdo_waiter *waiter, void *context) /* * This block won't be read in recovery until the slab summary is updated to refer to it. - * The slab summary update does a flush which is sufficient to protect us from VDO-2331. + * The slab summary update does a flush which is sufficient to protect us from corruption + * due to out of order slab journal, reference block, or block map writes. */ vdo_submit_metadata_vio(uds_forget(vio), block_number, write_slab_journal_endio, complete_write, REQ_OP_WRITE); @@ -1201,7 +1206,8 @@ static void write_reference_block(struct vdo_waiter *waiter, void *context) /* * Flush before writing to ensure that the recovery journal and slab journal entries which - * cover this reference update are stable (VDO-2331). + * cover this reference update are stable. This prevents data corruption that can be caused + * by out of order writes. */ WRITE_ONCE(block->slab->allocator->ref_counts_statistics.blocks_written, block->slab->allocator->ref_counts_statistics.blocks_written + 1); @@ -1775,7 +1781,7 @@ static void add_entries(struct slab_journal *journal) (journal->slab->status == VDO_SLAB_REBUILDING)) { /* * Don't add entries while rebuilding or while a partial write is - * outstanding (VDO-2399). + * outstanding, as it could result in reference count corruption. */ break; } diff --git a/drivers/md/dm-vdo/sparse-cache.c b/drivers/md/dm-vdo/sparse-cache.c index 216c8d6256a9..b43a626a42de 100644 --- a/drivers/md/dm-vdo/sparse-cache.c +++ b/drivers/md/dm-vdo/sparse-cache.c @@ -191,7 +191,7 @@ static inline void __down(struct semaphore *semaphore) * happens, sleep briefly to avoid keeping the CPU locked up in * this loop. We could just call cond_resched, but then we'd * still keep consuming CPU time slices and swamp other threads - * trying to do computational work. [VDO-4980] + * trying to do computational work. */ fsleep(1000); } diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index e0eddd4007b8..a40f059d39b3 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -544,7 +544,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason, int result; struct vdo *vdo; - /* VDO-3769 - Set a generic reason so we don't ever return garbage. */ + /* Initialize with a generic failure reason to prevent returning garbage. */ *reason = "Unspecified error"; result = uds_allocate(1, struct vdo, __func__, &vdo); diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index eb6838ddabbb..4832ea46551f 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -123,7 +123,6 @@ int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, struct vio *vio; int result; - /* If struct vio grows past 256 bytes, we'll lose benefits of VDOSTORY-176. */ BUILD_BUG_ON(sizeof(struct vio) > 256); /* From 924553644ac5ede5c03311ce218151f2bf7384e7 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 9 Feb 2024 14:14:21 -0600 Subject: [PATCH 0928/1038] dm vdo memory-alloc: simplify allocations_allowed() Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/memory-alloc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index 5cd387f9294e..db085c1124be 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -20,11 +20,9 @@ */ static struct thread_registry allocating_threads; -static bool allocations_allowed(void) +static inline bool allocations_allowed(void) { - const bool *pointer = vdo_lookup_thread(&allocating_threads); - - return (pointer != NULL) ? *pointer : false; + return vdo_lookup_thread(&allocating_threads) != NULL; } /* From 672fc9b8c000e8162cb4fe6f2dfc990df968ee3a Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 9 Feb 2024 16:06:05 +0300 Subject: [PATCH 0929/1038] dm vdo slab-depot: delete unnecessary check in allocate_components This is a duplicate check so it can't be true. Delete it. Signed-off-by: Dan Carpenter Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/slab-depot.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 5fa7e0838b32..e10c3db8e665 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -4106,9 +4106,6 @@ static int allocate_components(struct slab_depot *depot, }; } - if (result != VDO_SUCCESS) - return result; - slab_count = vdo_compute_slab_count(depot->first_block, depot->last_block, depot->slab_size_shift); if (thread_config->physical_zone_count > slab_count) { From b259c1a60c379ad73d673d329892c7dc36313656 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Sun, 11 Feb 2024 14:07:18 -0500 Subject: [PATCH 0930/1038] dm vdo flush: initialize return to NULL in allocate_flush Otherwise, error path could result in allocate_flush's subsequent check for flush being non-NULL leading to false positive. Reported-by: Dan Carpenter Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/flush.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index 330b18715027..391b6203efc6 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -100,7 +100,7 @@ static struct vdo_flush *vdo_waiter_as_flush(struct vdo_waiter *waiter) static void *allocate_flush(gfp_t gfp_mask, void *pool_data) { - struct vdo_flush *flush; + struct vdo_flush *flush = NULL; if ((gfp_mask & GFP_NOWAIT) == GFP_NOWAIT) { flush = uds_allocate_memory_nowait(sizeof(struct vdo_flush), __func__); From 444d3f0bfd436cf28348a0a20d0b8ad0cdbb8be3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Sun, 11 Feb 2024 14:49:42 -0500 Subject: [PATCH 0931/1038] dm vdo indexer-volume: fix missing mutex_lock in process_entry Must mutex_lock after dm_bufio_read, before dm_bufio_read error handling, otherwise process_entry error path will return without volume->read_threads_mutex held. This fixes potential double mutex_unlock. Reported-by: Dan Carpenter Signed-off-by: Mike Snitzer Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/volume.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/volume.c index 37c2ef0777e5..60416dc8a9d7 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/volume.c @@ -556,6 +556,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) mutex_unlock(&volume->read_threads_mutex); page_data = dm_bufio_read(volume->client, page_number, &page->buffer); + mutex_lock(&volume->read_threads_mutex); if (IS_ERR(page_data)) { result = -PTR_ERR(page_data); uds_log_warning_strerror(result, @@ -564,7 +565,6 @@ static int process_entry(struct volume *volume, struct queued_read *entry) cancel_page_in_cache(&volume->page_cache, page_number, page); return result; } - mutex_lock(&volume->read_threads_mutex); if (entry->invalid) { uds_log_warning("Page %u invalidated after read", page_number); From 20be466c7a967a5743890e850914c3e86117ac15 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 10:51:19 -0500 Subject: [PATCH 0932/1038] dm vdo: include to resolve current being undeclared Reported when building on loongarch. Reported-by: Randy Dunlap Signed-off-by: Mike Snitzer Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/logger.c | 1 + drivers/md/dm-vdo/thread-registry.c | 1 + drivers/md/dm-vdo/thread-utils.c | 1 + 3 files changed, 3 insertions(+) diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 6ba7e99ee8f9..a50edb0331fc 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -5,6 +5,7 @@ #include "logger.h" +#include #include #include #include diff --git a/drivers/md/dm-vdo/thread-registry.c b/drivers/md/dm-vdo/thread-registry.c index 1314d2b6a26f..03e2f45e8e78 100644 --- a/drivers/md/dm-vdo/thread-registry.c +++ b/drivers/md/dm-vdo/thread-registry.c @@ -5,6 +5,7 @@ #include "thread-registry.h" +#include #include #include "permassert.h" diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index 160679984d72..aeca14bba852 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -5,6 +5,7 @@ #include "thread-utils.h" +#include #include #include #include From 81c751ad1b7f55e0ae04c51bb29c7d855754e3b3 Mon Sep 17 00:00:00 2001 From: Chung Chung Date: Mon, 19 Feb 2024 15:58:24 -0500 Subject: [PATCH 0933/1038] dm vdo: clean up scnprintf usage Ignore scnprintf return status since it is not necessary. Change write_* functions type from int to void since we no longer return any result. Also, clean up any code that checks or uses any scnprintf return results. Check uds_allocate return code which was previous ignored, return and log error when uds_allocate failed. Reported-by: Dan Carpenter Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/message-stats.c | 864 +++++++++--------------------- 1 file changed, 262 insertions(+), 602 deletions(-) diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index af964e55b98c..cac7232f467b 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -11,749 +11,407 @@ #include "thread-device.h" #include "vdo.h" -static int write_u64(char *prefix, u64 value, char *suffix, char **buf, - unsigned int *maxlen) -{ - int count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, - value, suffix == NULL ? "" : suffix); - *buf += count; - *maxlen -= count; - if (count >= *maxlen) - return VDO_UNEXPECTED_EOF; - return VDO_SUCCESS; -} - -static int write_u32(char *prefix, u32 value, char *suffix, char **buf, - unsigned int *maxlen) -{ - int count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, - value, suffix == NULL ? "" : suffix); - *buf += count; - *maxlen -= count; - if (count >= *maxlen) - return VDO_UNEXPECTED_EOF; - return VDO_SUCCESS; -} - -static int write_block_count_t(char *prefix, block_count_t value, char *suffix, - char **buf, unsigned int *maxlen) -{ - int count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, - value, suffix == NULL ? "" : suffix); - *buf += count; - *maxlen -= count; - if (count >= *maxlen) - return VDO_UNEXPECTED_EOF; - return VDO_SUCCESS; -} - -static int write_string(char *prefix, char *value, char *suffix, char **buf, - unsigned int *maxlen) -{ - int count = scnprintf(*buf, *maxlen, "%s%s%s", prefix == NULL ? "" : prefix, - value, suffix == NULL ? "" : suffix); - *buf += count; - *maxlen -= count; - if (count >= *maxlen) - return VDO_UNEXPECTED_EOF; - return VDO_SUCCESS; -} - -static int write_bool(char *prefix, bool value, char *suffix, char **buf, +static void write_u64(char *prefix, u64 value, char *suffix, char **buf, unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%d%s", prefix == NULL ? "" : prefix, - value, suffix == NULL ? "" : suffix); + int count; + + count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; - if (count >= *maxlen) - return VDO_UNEXPECTED_EOF; - return VDO_SUCCESS; } -static int write_u8(char *prefix, u8 value, char *suffix, char **buf, - unsigned int *maxlen) +static void write_u32(char *prefix, u32 value, char *suffix, char **buf, + unsigned int *maxlen) { - int count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, - value, suffix == NULL ? "" : suffix); + int count; + + count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); *buf += count; *maxlen -= count; - if (count >= *maxlen) - return VDO_UNEXPECTED_EOF; - return VDO_SUCCESS; } -static int write_block_allocator_statistics(char *prefix, - struct block_allocator_statistics *stats, - char *suffix, char **buf, - unsigned int *maxlen) +static void write_block_count_t(char *prefix, block_count_t value, char *suffix, + char **buf, unsigned int *maxlen) { - int result; + int count; - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* The total number of slabs from which blocks may be allocated */ - result = write_u64("slabCount : ", stats->slab_count, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* The total number of slabs from which blocks have ever been allocated */ - result = write_u64("slabsOpened : ", stats->slabs_opened, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* The number of times since loading that a slab has been re-opened */ - result = write_u64("slabsReopened : ", stats->slabs_reopened, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; } -static int write_commit_statistics(char *prefix, struct commit_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_string(char *prefix, char *value, char *suffix, char **buf, + unsigned int *maxlen) { - int result; + int count; - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* The total number of items on which processing has started */ - result = write_u64("started : ", stats->started, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* The total number of items for which a write operation has been issued */ - result = write_u64("written : ", stats->written, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* The total number of items for which a write operation has completed */ - result = write_u64("committed : ", stats->committed, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + count = scnprintf(*buf, *maxlen, "%s%s%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; } -static int write_recovery_journal_statistics(char *prefix, - struct recovery_journal_statistics *stats, +static void write_bool(char *prefix, bool value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%d%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_u8(char *prefix, u8 value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_block_allocator_statistics(char *prefix, + struct block_allocator_statistics *stats, char *suffix, char **buf, unsigned int *maxlen) { - int result; + write_string(prefix, "{ ", NULL, buf, maxlen); + /* The total number of slabs from which blocks may be allocated */ + write_u64("slabCount : ", stats->slab_count, ", ", buf, maxlen); + /* The total number of slabs from which blocks have ever been allocated */ + write_u64("slabsOpened : ", stats->slabs_opened, ", ", buf, maxlen); + /* The number of times since loading that a slab has been re-opened */ + write_u64("slabsReopened : ", stats->slabs_reopened, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; +static void write_commit_statistics(char *prefix, struct commit_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* The total number of items on which processing has started */ + write_u64("started : ", stats->started, ", ", buf, maxlen); + /* The total number of items for which a write operation has been issued */ + write_u64("written : ", stats->written, ", ", buf, maxlen); + /* The total number of items for which a write operation has completed */ + write_u64("committed : ", stats->committed, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_recovery_journal_statistics(char *prefix, + struct recovery_journal_statistics *stats, + char *suffix, char **buf, + unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of times the on-disk journal was full */ - result = write_u64("diskFull : ", stats->disk_full, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("diskFull : ", stats->disk_full, ", ", buf, maxlen); /* Number of times the recovery journal requested slab journal commits. */ - result = write_u64("slabJournalCommitsRequested : ", - stats->slab_journal_commits_requested, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("slabJournalCommitsRequested : ", + stats->slab_journal_commits_requested, ", ", buf, maxlen); /* Write/Commit totals for individual journal entries */ - result = write_commit_statistics("entries : ", &stats->entries, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_commit_statistics("entries : ", &stats->entries, ", ", buf, maxlen); /* Write/Commit totals for journal blocks */ - result = write_commit_statistics("blocks : ", &stats->blocks, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_commit_statistics("blocks : ", &stats->blocks, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_packer_statistics(char *prefix, struct packer_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_packer_statistics(char *prefix, struct packer_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of compressed data items written since startup */ - result = write_u64("compressedFragmentsWritten : ", - stats->compressed_fragments_written, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("compressedFragmentsWritten : ", + stats->compressed_fragments_written, ", ", buf, maxlen); /* Number of blocks containing compressed items written since startup */ - result = write_u64("compressedBlocksWritten : ", - stats->compressed_blocks_written, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("compressedBlocksWritten : ", + stats->compressed_blocks_written, ", ", buf, maxlen); /* Number of VIOs that are pending in the packer */ - result = write_u64("compressedFragmentsInPacker : ", - stats->compressed_fragments_in_packer, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("compressedFragmentsInPacker : ", + stats->compressed_fragments_in_packer, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_slab_journal_statistics(char *prefix, - struct slab_journal_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_slab_journal_statistics(char *prefix, + struct slab_journal_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of times the on-disk journal was full */ - result = write_u64("diskFullCount : ", stats->disk_full_count, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("diskFullCount : ", stats->disk_full_count, ", ", buf, maxlen); /* Number of times an entry was added over the flush threshold */ - result = write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); /* Number of times an entry was added over the block threshold */ - result = write_u64("blockedCount : ", stats->blocked_count, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("blockedCount : ", stats->blocked_count, ", ", buf, maxlen); /* Number of times a tail block was written */ - result = write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); /* Number of times we had to wait for the tail to write */ - result = write_u64("tailBusyCount : ", stats->tail_busy_count, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("tailBusyCount : ", stats->tail_busy_count, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_slab_summary_statistics(char *prefix, - struct slab_summary_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_slab_summary_statistics(char *prefix, + struct slab_summary_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of blocks written */ - result = write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_ref_counts_statistics(char *prefix, struct ref_counts_statistics *stats, +static void write_ref_counts_statistics(char *prefix, struct ref_counts_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of reference blocks written */ + write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_block_map_statistics(char *prefix, struct block_map_statistics *stats, char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - /* Number of reference blocks written */ - result = write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; -} - -static int write_block_map_statistics(char *prefix, struct block_map_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) -{ - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* number of dirty (resident) pages */ - result = write_u32("dirtyPages : ", stats->dirty_pages, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("dirtyPages : ", stats->dirty_pages, ", ", buf, maxlen); /* number of clean (resident) pages */ - result = write_u32("cleanPages : ", stats->clean_pages, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("cleanPages : ", stats->clean_pages, ", ", buf, maxlen); /* number of free pages */ - result = write_u32("freePages : ", stats->free_pages, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("freePages : ", stats->free_pages, ", ", buf, maxlen); /* number of pages in failed state */ - result = write_u32("failedPages : ", stats->failed_pages, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("failedPages : ", stats->failed_pages, ", ", buf, maxlen); /* number of pages incoming */ - result = write_u32("incomingPages : ", stats->incoming_pages, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("incomingPages : ", stats->incoming_pages, ", ", buf, maxlen); /* number of pages outgoing */ - result = write_u32("outgoingPages : ", stats->outgoing_pages, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("outgoingPages : ", stats->outgoing_pages, ", ", buf, maxlen); /* how many times free page not avail */ - result = write_u32("cachePressure : ", stats->cache_pressure, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("cachePressure : ", stats->cache_pressure, ", ", buf, maxlen); /* number of get_vdo_page() calls for read */ - result = write_u64("readCount : ", stats->read_count, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("readCount : ", stats->read_count, ", ", buf, maxlen); /* number of get_vdo_page() calls for write */ - result = write_u64("writeCount : ", stats->write_count, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("writeCount : ", stats->write_count, ", ", buf, maxlen); /* number of times pages failed to read */ - result = write_u64("failedReads : ", stats->failed_reads, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("failedReads : ", stats->failed_reads, ", ", buf, maxlen); /* number of times pages failed to write */ - result = write_u64("failedWrites : ", stats->failed_writes, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("failedWrites : ", stats->failed_writes, ", ", buf, maxlen); /* number of gets that are reclaimed */ - result = write_u64("reclaimed : ", stats->reclaimed, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("reclaimed : ", stats->reclaimed, ", ", buf, maxlen); /* number of gets for outgoing pages */ - result = write_u64("readOutgoing : ", stats->read_outgoing, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("readOutgoing : ", stats->read_outgoing, ", ", buf, maxlen); /* number of gets that were already there */ - result = write_u64("foundInCache : ", stats->found_in_cache, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("foundInCache : ", stats->found_in_cache, ", ", buf, maxlen); /* number of gets requiring discard */ - result = write_u64("discardRequired : ", stats->discard_required, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("discardRequired : ", stats->discard_required, ", ", buf, maxlen); /* number of gets enqueued for their page */ - result = write_u64("waitForPage : ", stats->wait_for_page, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("waitForPage : ", stats->wait_for_page, ", ", buf, maxlen); /* number of gets that have to fetch */ - result = write_u64("fetchRequired : ", stats->fetch_required, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("fetchRequired : ", stats->fetch_required, ", ", buf, maxlen); /* number of page fetches */ - result = write_u64("pagesLoaded : ", stats->pages_loaded, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("pagesLoaded : ", stats->pages_loaded, ", ", buf, maxlen); /* number of page saves */ - result = write_u64("pagesSaved : ", stats->pages_saved, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("pagesSaved : ", stats->pages_saved, ", ", buf, maxlen); /* the number of flushes issued */ - result = write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_hash_lock_statistics(char *prefix, struct hash_lock_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_hash_lock_statistics(char *prefix, struct hash_lock_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of times the UDS advice proved correct */ - result = write_u64("dedupeAdviceValid : ", stats->dedupe_advice_valid, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("dedupeAdviceValid : ", stats->dedupe_advice_valid, ", ", buf, maxlen); /* Number of times the UDS advice proved incorrect */ - result = write_u64("dedupeAdviceStale : ", stats->dedupe_advice_stale, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("dedupeAdviceStale : ", stats->dedupe_advice_stale, ", ", buf, maxlen); /* Number of writes with the same data as another in-flight write */ - result = write_u64("concurrentDataMatches : ", stats->concurrent_data_matches, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("concurrentDataMatches : ", stats->concurrent_data_matches, + ", ", buf, maxlen); /* Number of writes whose hash collided with an in-flight write */ - result = write_u64("concurrentHashCollisions : ", - stats->concurrent_hash_collisions, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("concurrentHashCollisions : ", + stats->concurrent_hash_collisions, ", ", buf, maxlen); /* Current number of dedupe queries that are in flight */ - result = write_u32("currDedupeQueries : ", stats->curr_dedupe_queries, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u32("currDedupeQueries : ", stats->curr_dedupe_queries, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_error_statistics(char *prefix, struct error_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_error_statistics(char *prefix, struct error_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* number of times VDO got an invalid dedupe advice PBN from UDS */ - result = write_u64("invalidAdvicePBNCount : ", stats->invalid_advice_pbn_count, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("invalidAdvicePBNCount : ", stats->invalid_advice_pbn_count, + ", ", buf, maxlen); /* number of times a VIO completed with a VDO_NO_SPACE error */ - result = write_u64("noSpaceErrorCount : ", stats->no_space_error_count, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("noSpaceErrorCount : ", stats->no_space_error_count, ", ", + buf, maxlen); /* number of times a VIO completed with a VDO_READ_ONLY error */ - result = write_u64("readOnlyErrorCount : ", stats->read_only_error_count, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("readOnlyErrorCount : ", stats->read_only_error_count, ", ", + buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_bio_stats(char *prefix, struct bio_stats *stats, char *suffix, - char **buf, unsigned int *maxlen) +static void write_bio_stats(char *prefix, struct bio_stats *stats, char *suffix, + char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of REQ_OP_READ bios */ - result = write_u64("read : ", stats->read, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("read : ", stats->read, ", ", buf, maxlen); /* Number of REQ_OP_WRITE bios with data */ - result = write_u64("write : ", stats->write, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("write : ", stats->write, ", ", buf, maxlen); /* Number of bios tagged with REQ_PREFLUSH and containing no data */ - result = write_u64("emptyFlush : ", stats->empty_flush, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("emptyFlush : ", stats->empty_flush, ", ", buf, maxlen); /* Number of REQ_OP_DISCARD bios */ - result = write_u64("discard : ", stats->discard, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("discard : ", stats->discard, ", ", buf, maxlen); /* Number of bios tagged with REQ_PREFLUSH */ - result = write_u64("flush : ", stats->flush, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("flush : ", stats->flush, ", ", buf, maxlen); /* Number of bios tagged with REQ_FUA */ - result = write_u64("fua : ", stats->fua, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("fua : ", stats->fua, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_memory_usage(char *prefix, struct memory_usage *stats, char *suffix, - char **buf, unsigned int *maxlen) +static void write_memory_usage(char *prefix, struct memory_usage *stats, char *suffix, + char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Tracked bytes currently allocated. */ - result = write_u64("bytesUsed : ", stats->bytes_used, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("bytesUsed : ", stats->bytes_used, ", ", buf, maxlen); /* Maximum tracked bytes allocated. */ - result = write_u64("peakBytesUsed : ", stats->peak_bytes_used, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("peakBytesUsed : ", stats->peak_bytes_used, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_index_statistics(char *prefix, struct index_statistics *stats, - char *suffix, char **buf, unsigned int *maxlen) +static void write_index_statistics(char *prefix, struct index_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); /* Number of records stored in the index */ - result = write_u64("entriesIndexed : ", stats->entries_indexed, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("entriesIndexed : ", stats->entries_indexed, ", ", buf, maxlen); /* Number of post calls that found an existing entry */ - result = write_u64("postsFound : ", stats->posts_found, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("postsFound : ", stats->posts_found, ", ", buf, maxlen); /* Number of post calls that added a new entry */ - result = write_u64("postsNotFound : ", stats->posts_not_found, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("postsNotFound : ", stats->posts_not_found, ", ", buf, maxlen); /* Number of query calls that found an existing entry */ - result = write_u64("queriesFound : ", stats->queries_found, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("queriesFound : ", stats->queries_found, ", ", buf, maxlen); /* Number of query calls that added a new entry */ - result = write_u64("queriesNotFound : ", stats->queries_not_found, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("queriesNotFound : ", stats->queries_not_found, ", ", buf, maxlen); /* Number of update calls that found an existing entry */ - result = write_u64("updatesFound : ", stats->updates_found, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("updatesFound : ", stats->updates_found, ", ", buf, maxlen); /* Number of update calls that added a new entry */ - result = write_u64("updatesNotFound : ", stats->updates_not_found, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("updatesNotFound : ", stats->updates_not_found, ", ", buf, maxlen); /* Number of entries discarded */ - result = write_u64("entriesDiscarded : ", stats->entries_discarded, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_u64("entriesDiscarded : ", stats->entries_discarded, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } -static int write_vdo_statistics(char *prefix, struct vdo_statistics *stats, char *suffix, - char **buf, unsigned int *maxlen) +static void write_vdo_statistics(char *prefix, struct vdo_statistics *stats, char *suffix, + char **buf, unsigned int *maxlen) { - int result; - - result = write_string(prefix, "{ ", NULL, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_u32("version : ", stats->version, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string(prefix, "{ ", NULL, buf, maxlen); + write_u32("version : ", stats->version, ", ", buf, maxlen); /* Number of blocks used for data */ - result = write_u64("dataBlocksUsed : ", stats->data_blocks_used, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("dataBlocksUsed : ", stats->data_blocks_used, ", ", buf, maxlen); /* Number of blocks used for VDO metadata */ - result = write_u64("overheadBlocksUsed : ", stats->overhead_blocks_used, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("overheadBlocksUsed : ", stats->overhead_blocks_used, ", ", + buf, maxlen); /* Number of logical blocks that are currently mapped to physical blocks */ - result = write_u64("logicalBlocksUsed : ", stats->logical_blocks_used, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("logicalBlocksUsed : ", stats->logical_blocks_used, ", ", buf, maxlen); /* number of physical blocks */ - result = write_block_count_t("physicalBlocks : ", stats->physical_blocks, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_block_count_t("physicalBlocks : ", stats->physical_blocks, ", ", + buf, maxlen); /* number of logical blocks */ - result = write_block_count_t("logicalBlocks : ", stats->logical_blocks, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_block_count_t("logicalBlocks : ", stats->logical_blocks, ", ", + buf, maxlen); /* Size of the block map page cache, in bytes */ - result = write_u64("blockMapCacheSize : ", stats->block_map_cache_size, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("blockMapCacheSize : ", stats->block_map_cache_size, ", ", + buf, maxlen); /* The physical block size */ - result = write_u64("blockSize : ", stats->block_size, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("blockSize : ", stats->block_size, ", ", buf, maxlen); /* Number of times the VDO has successfully recovered */ - result = write_u64("completeRecoveries : ", stats->complete_recoveries, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("completeRecoveries : ", stats->complete_recoveries, ", ", + buf, maxlen); /* Number of times the VDO has recovered from read-only mode */ - result = write_u64("readOnlyRecoveries : ", stats->read_only_recoveries, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("readOnlyRecoveries : ", stats->read_only_recoveries, ", ", + buf, maxlen); /* String describing the operating mode of the VDO */ - result = write_string("mode : ", stats->mode, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_string("mode : ", stats->mode, ", ", buf, maxlen); /* Whether the VDO is in recovery mode */ - result = write_bool("inRecoveryMode : ", stats->in_recovery_mode, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_bool("inRecoveryMode : ", stats->in_recovery_mode, ", ", buf, maxlen); /* What percentage of recovery mode work has been completed */ - result = write_u8("recoveryPercentage : ", stats->recovery_percentage, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u8("recoveryPercentage : ", stats->recovery_percentage, ", ", buf, maxlen); /* The statistics for the compressed block packer */ - result = write_packer_statistics("packer : ", &stats->packer, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_packer_statistics("packer : ", &stats->packer, ", ", buf, maxlen); /* Counters for events in the block allocator */ - result = write_block_allocator_statistics("allocator : ", &stats->allocator, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_block_allocator_statistics("allocator : ", &stats->allocator, + ", ", buf, maxlen); /* Counters for events in the recovery journal */ - result = write_recovery_journal_statistics("journal : ", &stats->journal, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_recovery_journal_statistics("journal : ", &stats->journal, ", ", + buf, maxlen); /* The statistics for the slab journals */ - result = write_slab_journal_statistics("slabJournal : ", &stats->slab_journal, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_slab_journal_statistics("slabJournal : ", &stats->slab_journal, + ", ", buf, maxlen); /* The statistics for the slab summary */ - result = write_slab_summary_statistics("slabSummary : ", &stats->slab_summary, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_slab_summary_statistics("slabSummary : ", &stats->slab_summary, + ", ", buf, maxlen); /* The statistics for the reference counts */ - result = write_ref_counts_statistics("refCounts : ", &stats->ref_counts, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_ref_counts_statistics("refCounts : ", &stats->ref_counts, ", ", + buf, maxlen); /* The statistics for the block map */ - result = write_block_map_statistics("blockMap : ", &stats->block_map, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_block_map_statistics("blockMap : ", &stats->block_map, ", ", buf, maxlen); /* The dedupe statistics from hash locks */ - result = write_hash_lock_statistics("hashLock : ", &stats->hash_lock, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_hash_lock_statistics("hashLock : ", &stats->hash_lock, ", ", buf, maxlen); /* Counts of error conditions */ - result = write_error_statistics("errors : ", &stats->errors, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_error_statistics("errors : ", &stats->errors, ", ", buf, maxlen); /* The VDO instance */ - result = write_u32("instance : ", stats->instance, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("instance : ", stats->instance, ", ", buf, maxlen); /* Current number of active VIOs */ - result = write_u32("currentVIOsInProgress : ", stats->current_vios_in_progress, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("currentVIOsInProgress : ", stats->current_vios_in_progress, + ", ", buf, maxlen); /* Maximum number of active VIOs */ - result = write_u32("maxVIOs : ", stats->max_vios, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u32("maxVIOs : ", stats->max_vios, ", ", buf, maxlen); /* Number of times the UDS index was too slow in responding */ - result = write_u64("dedupeAdviceTimeouts : ", stats->dedupe_advice_timeouts, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("dedupeAdviceTimeouts : ", stats->dedupe_advice_timeouts, + ", ", buf, maxlen); /* Number of flush requests submitted to the storage device */ - result = write_u64("flushOut : ", stats->flush_out, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("flushOut : ", stats->flush_out, ", ", buf, maxlen); /* Logical block size */ - result = write_u64("logicalBlockSize : ", stats->logical_block_size, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_u64("logicalBlockSize : ", stats->logical_block_size, ", ", buf, maxlen); /* Bios submitted into VDO from above */ - result = write_bio_stats("biosIn : ", &stats->bios_in, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosInPartial : ", &stats->bios_in_partial, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_bio_stats("biosIn : ", &stats->bios_in, ", ", buf, maxlen); + write_bio_stats("biosInPartial : ", &stats->bios_in_partial, ", ", buf, maxlen); /* Bios submitted onward for user data */ - result = write_bio_stats("biosOut : ", &stats->bios_out, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_bio_stats("biosOut : ", &stats->bios_out, ", ", buf, maxlen); /* Bios submitted onward for metadata */ - result = write_bio_stats("biosMeta : ", &stats->bios_meta, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosJournal : ", &stats->bios_journal, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosPageCache : ", &stats->bios_page_cache, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosOutCompleted : ", &stats->bios_out_completed, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosMetaCompleted : ", &stats->bios_meta_completed, - ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosJournalCompleted : ", - &stats->bios_journal_completed, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosPageCacheCompleted : ", - &stats->bios_page_cache_completed, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosAcknowledged : ", &stats->bios_acknowledged, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_bio_stats("biosAcknowledgedPartial : ", - &stats->bios_acknowledged_partial, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_bio_stats("biosMeta : ", &stats->bios_meta, ", ", buf, maxlen); + write_bio_stats("biosJournal : ", &stats->bios_journal, ", ", buf, maxlen); + write_bio_stats("biosPageCache : ", &stats->bios_page_cache, ", ", buf, maxlen); + write_bio_stats("biosOutCompleted : ", &stats->bios_out_completed, ", ", + buf, maxlen); + write_bio_stats("biosMetaCompleted : ", &stats->bios_meta_completed, + ", ", buf, maxlen); + write_bio_stats("biosJournalCompleted : ", + &stats->bios_journal_completed, ", ", buf, maxlen); + write_bio_stats("biosPageCacheCompleted : ", + &stats->bios_page_cache_completed, ", ", buf, maxlen); + write_bio_stats("biosAcknowledged : ", &stats->bios_acknowledged, ", ", + buf, maxlen); + write_bio_stats("biosAcknowledgedPartial : ", + &stats->bios_acknowledged_partial, ", ", buf, maxlen); /* Current number of bios in progress */ - result = write_bio_stats("biosInProgress : ", &stats->bios_in_progress, ", ", - buf, maxlen); - if (result != VDO_SUCCESS) - return result; + write_bio_stats("biosInProgress : ", &stats->bios_in_progress, ", ", + buf, maxlen); /* Memory usage stats. */ - result = write_memory_usage("memoryUsage : ", &stats->memory_usage, ", ", buf, - maxlen); - if (result != VDO_SUCCESS) - return result; + write_memory_usage("memoryUsage : ", &stats->memory_usage, ", ", buf, maxlen); /* The statistics for the UDS index */ - result = write_index_statistics("index : ", &stats->index, ", ", buf, maxlen); - if (result != VDO_SUCCESS) - return result; - result = write_string(NULL, "}", suffix, buf, maxlen); - if (result != VDO_SUCCESS) - return result; - return VDO_SUCCESS; + write_index_statistics("index : ", &stats->index, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); } int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) @@ -762,11 +420,13 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) int result; result = uds_allocate(1, struct vdo_statistics, __func__, &stats); - if (result != VDO_SUCCESS) + if (result != UDS_SUCCESS) { + uds_log_error("Cannot allocate memory to write VDO statistics"); return result; + } vdo_fetch_statistics(vdo, stats); - result = write_vdo_statistics(NULL, stats, NULL, &buf, &maxlen); + write_vdo_statistics(NULL, stats, NULL, &buf, &maxlen); uds_free(stats); - return result; + return VDO_SUCCESS; } From 61234f0bdabb5f3b9e26e582e6457a1a3b01cb79 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Mon, 19 Feb 2024 22:23:07 -0500 Subject: [PATCH 0934/1038] dm vdo: remove unnecessary indexer.h includes Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/encodings.h | 1 - drivers/md/dm-vdo/vdo.h | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/md/dm-vdo/encodings.h b/drivers/md/dm-vdo/encodings.h index 18794fd59b0b..e5ff2b0aaa79 100644 --- a/drivers/md/dm-vdo/encodings.h +++ b/drivers/md/dm-vdo/encodings.h @@ -11,7 +11,6 @@ #include #include -#include "indexer.h" #include "numeric.h" #include "constants.h" diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h index 3938e519ae6a..01558fc67f24 100644 --- a/drivers/md/dm-vdo/vdo.h +++ b/drivers/md/dm-vdo/vdo.h @@ -17,7 +17,6 @@ #include "admin-state.h" #include "encodings.h" #include "funnel-workqueue.h" -#include "indexer.h" #include "packer.h" #include "physical-zone.h" #include "statistics.h" From 17b1a73feaf31bea77010e226c8e434cba1f95ea Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 8 Feb 2024 15:55:29 -0600 Subject: [PATCH 0935/1038] dm vdo: move indexer files into sub-directory The goal is to assist high-level understanding of which code is conceptually specific to VDO's indexer. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/Makefile | 34 ++++++++++--------- drivers/md/dm-vdo/data-vio.h | 3 +- drivers/md/dm-vdo/dedupe.c | 3 +- .../md/dm-vdo/{ => indexer}/chapter-index.c | 5 +-- .../md/dm-vdo/{ => indexer}/chapter-index.h | 0 drivers/md/dm-vdo/{ => indexer}/config.c | 0 drivers/md/dm-vdo/{ => indexer}/config.h | 0 drivers/md/dm-vdo/{ => indexer}/delta-index.c | 5 +-- drivers/md/dm-vdo/{ => indexer}/delta-index.h | 5 +-- .../{ => indexer}/funnel-requestqueue.c | 0 .../{ => indexer}/funnel-requestqueue.h | 0 drivers/md/dm-vdo/{ => indexer}/geometry.c | 5 +-- drivers/md/dm-vdo/{ => indexer}/geometry.h | 0 drivers/md/dm-vdo/{ => indexer}/hash-utils.h | 3 +- .../md/dm-vdo/{ => indexer}/index-layout.c | 5 +-- .../md/dm-vdo/{ => indexer}/index-layout.h | 0 .../md/dm-vdo/{ => indexer}/index-page-map.c | 5 +-- .../md/dm-vdo/{ => indexer}/index-page-map.h | 0 .../md/dm-vdo/{ => indexer}/index-session.c | 7 ++-- .../md/dm-vdo/{ => indexer}/index-session.h | 3 +- drivers/md/dm-vdo/{ => indexer}/index.c | 5 +-- drivers/md/dm-vdo/{ => indexer}/index.h | 0 drivers/md/dm-vdo/{ => indexer}/indexer.h | 0 drivers/md/dm-vdo/{ => indexer}/io-factory.c | 0 drivers/md/dm-vdo/{ => indexer}/io-factory.h | 0 .../md/dm-vdo/{ => indexer}/open-chapter.c | 5 +-- .../md/dm-vdo/{ => indexer}/open-chapter.h | 0 drivers/md/dm-vdo/{ => indexer}/radix-sort.c | 0 drivers/md/dm-vdo/{ => indexer}/radix-sort.h | 0 .../md/dm-vdo/{ => indexer}/sparse-cache.c | 7 ++-- .../md/dm-vdo/{ => indexer}/sparse-cache.h | 0 .../md/dm-vdo/{ => indexer}/volume-index.c | 9 ++--- .../md/dm-vdo/{ => indexer}/volume-index.h | 3 +- drivers/md/dm-vdo/{ => indexer}/volume.c | 13 +++---- drivers/md/dm-vdo/{ => indexer}/volume.h | 5 +-- drivers/md/dm-vdo/uds-sysfs.c | 3 +- 36 files changed, 77 insertions(+), 56 deletions(-) rename drivers/md/dm-vdo/{ => indexer}/chapter-index.c (99%) rename drivers/md/dm-vdo/{ => indexer}/chapter-index.h (100%) rename drivers/md/dm-vdo/{ => indexer}/config.c (100%) rename drivers/md/dm-vdo/{ => indexer}/config.h (100%) rename drivers/md/dm-vdo/{ => indexer}/delta-index.c (99%) rename drivers/md/dm-vdo/{ => indexer}/delta-index.h (99%) rename drivers/md/dm-vdo/{ => indexer}/funnel-requestqueue.c (100%) rename drivers/md/dm-vdo/{ => indexer}/funnel-requestqueue.h (100%) rename drivers/md/dm-vdo/{ => indexer}/geometry.c (99%) rename drivers/md/dm-vdo/{ => indexer}/geometry.h (100%) rename drivers/md/dm-vdo/{ => indexer}/hash-utils.h (99%) rename drivers/md/dm-vdo/{ => indexer}/index-layout.c (99%) rename drivers/md/dm-vdo/{ => indexer}/index-layout.h (100%) rename drivers/md/dm-vdo/{ => indexer}/index-page-map.c (99%) rename drivers/md/dm-vdo/{ => indexer}/index-page-map.h (100%) rename drivers/md/dm-vdo/{ => indexer}/index-session.c (99%) rename drivers/md/dm-vdo/{ => indexer}/index-session.h (99%) rename drivers/md/dm-vdo/{ => indexer}/index.c (99%) rename drivers/md/dm-vdo/{ => indexer}/index.h (100%) rename drivers/md/dm-vdo/{ => indexer}/indexer.h (100%) rename drivers/md/dm-vdo/{ => indexer}/io-factory.c (100%) rename drivers/md/dm-vdo/{ => indexer}/io-factory.h (100%) rename drivers/md/dm-vdo/{ => indexer}/open-chapter.c (99%) rename drivers/md/dm-vdo/{ => indexer}/open-chapter.h (100%) rename drivers/md/dm-vdo/{ => indexer}/radix-sort.c (100%) rename drivers/md/dm-vdo/{ => indexer}/radix-sort.h (100%) rename drivers/md/dm-vdo/{ => indexer}/sparse-cache.c (99%) rename drivers/md/dm-vdo/{ => indexer}/sparse-cache.h (100%) rename drivers/md/dm-vdo/{ => indexer}/volume-index.c (99%) rename drivers/md/dm-vdo/{ => indexer}/volume-index.h (99%) rename drivers/md/dm-vdo/{ => indexer}/volume.c (99%) rename drivers/md/dm-vdo/{ => indexer}/volume.h (99%) diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 32266ab04cc1..502a7a0acbdb 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -1,50 +1,39 @@ # SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I$(srctree)/$(src) -I$(srctree)/$(src)/indexer + obj-$(CONFIG_DM_VDO) += dm-vdo.o dm-vdo-objs := \ action-manager.o \ admin-state.o \ block-map.o \ - chapter-index.o \ completion.o \ - config.o \ data-vio.o \ dedupe.o \ - delta-index.o \ dm-vdo-target.o \ dump.o \ encodings.o \ errors.o \ flush.o \ funnel-queue.o \ - funnel-requestqueue.o \ funnel-workqueue.o \ - geometry.o \ - index-layout.o \ - index.o \ - index-page-map.o \ - index-session.o \ int-map.o \ - io-factory.o \ io-submitter.o \ logger.o \ logical-zone.o \ memory-alloc.o \ message-stats.o \ murmurhash3.o \ - open-chapter.o \ packer.o \ permassert.o \ physical-zone.o \ pool-sysfs.o \ pool-sysfs-stats.o \ priority-table.o \ - radix-sort.o \ recovery-journal.o \ repair.o \ slab-depot.o \ - sparse-cache.o \ status-codes.o \ string-utils.o \ sysfs.o \ @@ -54,6 +43,19 @@ dm-vdo-objs := \ uds-sysfs.o \ vdo.o \ vio.o \ - volume-index.o \ - volume.o \ - wait-queue.o + wait-queue.o \ + indexer/chapter-index.o \ + indexer/config.o \ + indexer/delta-index.o \ + indexer/funnel-requestqueue.o \ + indexer/geometry.o \ + indexer/index.o \ + indexer/index-layout.o \ + indexer/index-page-map.o \ + indexer/index-session.o \ + indexer/io-factory.o \ + indexer/open-chapter.o \ + indexer/radix-sort.o \ + indexer/sparse-cache.o \ + indexer/volume.o \ + indexer/volume-index.o diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h index e7729623a6bb..44fd0d8ccb76 100644 --- a/drivers/md/dm-vdo/data-vio.h +++ b/drivers/md/dm-vdo/data-vio.h @@ -10,9 +10,10 @@ #include #include -#include "indexer.h" #include "permassert.h" +#include "indexer.h" + #include "block-map.h" #include "completion.h" #include "constants.h" diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 942a50ef8b0d..9468d7fad443 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -126,13 +126,14 @@ #include #include -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "string-utils.h" +#include "indexer.h" + #include "action-manager.h" #include "admin-state.h" #include "completion.h" diff --git a/drivers/md/dm-vdo/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c similarity index 99% rename from drivers/md/dm-vdo/chapter-index.c rename to drivers/md/dm-vdo/indexer/chapter-index.c index 9b9185c2c237..6487825ada90 100644 --- a/drivers/md/dm-vdo/chapter-index.c +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -6,12 +6,13 @@ #include "chapter-index.h" #include "errors.h" -#include "hash-utils.h" -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "permassert.h" +#include "hash-utils.h" +#include "indexer.h" + int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, const struct index_geometry *geometry, u64 volume_nonce) { diff --git a/drivers/md/dm-vdo/chapter-index.h b/drivers/md/dm-vdo/indexer/chapter-index.h similarity index 100% rename from drivers/md/dm-vdo/chapter-index.h rename to drivers/md/dm-vdo/indexer/chapter-index.h diff --git a/drivers/md/dm-vdo/config.c b/drivers/md/dm-vdo/indexer/config.c similarity index 100% rename from drivers/md/dm-vdo/config.c rename to drivers/md/dm-vdo/indexer/config.c diff --git a/drivers/md/dm-vdo/config.h b/drivers/md/dm-vdo/indexer/config.h similarity index 100% rename from drivers/md/dm-vdo/config.h rename to drivers/md/dm-vdo/indexer/config.h diff --git a/drivers/md/dm-vdo/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c similarity index 99% rename from drivers/md/dm-vdo/delta-index.c rename to drivers/md/dm-vdo/indexer/delta-index.c index 66f51b5f8fd2..4aace707545a 100644 --- a/drivers/md/dm-vdo/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -10,10 +10,8 @@ #include #include -#include "config.h" #include "cpu.h" #include "errors.h" -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" @@ -21,6 +19,9 @@ #include "string-utils.h" #include "time-utils.h" +#include "config.h" +#include "indexer.h" + /* * The entries in a delta index could be stored in a single delta list, but to reduce search times * and update costs it uses multiple delta lists. These lists are stored in a single chunk of diff --git a/drivers/md/dm-vdo/delta-index.h b/drivers/md/dm-vdo/indexer/delta-index.h similarity index 99% rename from drivers/md/dm-vdo/delta-index.h rename to drivers/md/dm-vdo/indexer/delta-index.h index b3b38fb440bf..3d2ea19aef61 100644 --- a/drivers/md/dm-vdo/delta-index.h +++ b/drivers/md/dm-vdo/indexer/delta-index.h @@ -8,11 +8,12 @@ #include -#include "config.h" -#include "io-factory.h" #include "numeric.h" #include "time-utils.h" +#include "config.h" +#include "io-factory.h" + /* * A delta index is a key-value store, where each entry maps an address (the key) to a payload (the * value). The entries are sorted by address, and only the delta between successive addresses is diff --git a/drivers/md/dm-vdo/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c similarity index 100% rename from drivers/md/dm-vdo/funnel-requestqueue.c rename to drivers/md/dm-vdo/indexer/funnel-requestqueue.c diff --git a/drivers/md/dm-vdo/funnel-requestqueue.h b/drivers/md/dm-vdo/indexer/funnel-requestqueue.h similarity index 100% rename from drivers/md/dm-vdo/funnel-requestqueue.h rename to drivers/md/dm-vdo/indexer/funnel-requestqueue.h diff --git a/drivers/md/dm-vdo/geometry.c b/drivers/md/dm-vdo/indexer/geometry.c similarity index 99% rename from drivers/md/dm-vdo/geometry.c rename to drivers/md/dm-vdo/indexer/geometry.c index 04c07195a01c..38c18283cdde 100644 --- a/drivers/md/dm-vdo/geometry.c +++ b/drivers/md/dm-vdo/indexer/geometry.c @@ -8,13 +8,14 @@ #include #include -#include "delta-index.h" #include "errors.h" -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "permassert.h" +#include "delta-index.h" +#include "indexer.h" + /* * An index volume is divided into a fixed number of fixed-size chapters, each consisting of a * fixed number of fixed-size pages. The volume layout is defined by two constants and four diff --git a/drivers/md/dm-vdo/geometry.h b/drivers/md/dm-vdo/indexer/geometry.h similarity index 100% rename from drivers/md/dm-vdo/geometry.h rename to drivers/md/dm-vdo/indexer/geometry.h diff --git a/drivers/md/dm-vdo/hash-utils.h b/drivers/md/dm-vdo/indexer/hash-utils.h similarity index 99% rename from drivers/md/dm-vdo/hash-utils.h rename to drivers/md/dm-vdo/indexer/hash-utils.h index e3b865bbe9b2..6a8dd8ffea6c 100644 --- a/drivers/md/dm-vdo/hash-utils.h +++ b/drivers/md/dm-vdo/indexer/hash-utils.h @@ -6,9 +6,10 @@ #ifndef UDS_HASH_UTILS_H #define UDS_HASH_UTILS_H +#include "numeric.h" + #include "geometry.h" #include "indexer.h" -#include "numeric.h" /* Utilities for extracting portions of a request name for various uses. */ diff --git a/drivers/md/dm-vdo/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c similarity index 99% rename from drivers/md/dm-vdo/index-layout.c rename to drivers/md/dm-vdo/indexer/index-layout.c index 2da507b26fd5..af533aa270a8 100644 --- a/drivers/md/dm-vdo/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -7,13 +7,14 @@ #include -#include "config.h" #include "logger.h" #include "memory-alloc.h" #include "murmurhash3.h" #include "numeric.h" -#include "open-chapter.h" #include "time-utils.h" + +#include "config.h" +#include "open-chapter.h" #include "volume-index.h" /* diff --git a/drivers/md/dm-vdo/index-layout.h b/drivers/md/dm-vdo/indexer/index-layout.h similarity index 100% rename from drivers/md/dm-vdo/index-layout.h rename to drivers/md/dm-vdo/indexer/index-layout.h diff --git a/drivers/md/dm-vdo/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c similarity index 99% rename from drivers/md/dm-vdo/index-page-map.c rename to drivers/md/dm-vdo/indexer/index-page-map.c index 1bb12066ad1a..90d97c33a9c3 100644 --- a/drivers/md/dm-vdo/index-page-map.c +++ b/drivers/md/dm-vdo/indexer/index-page-map.c @@ -6,8 +6,6 @@ #include "index-page-map.h" #include "errors.h" -#include "hash-utils.h" -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" @@ -15,6 +13,9 @@ #include "string-utils.h" #include "thread-utils.h" +#include "hash-utils.h" +#include "indexer.h" + /* * The index page map is conceptually a two-dimensional array indexed by chapter number and index * page number within the chapter. Each entry contains the number of the last delta list on that diff --git a/drivers/md/dm-vdo/index-page-map.h b/drivers/md/dm-vdo/indexer/index-page-map.h similarity index 100% rename from drivers/md/dm-vdo/index-page-map.h rename to drivers/md/dm-vdo/indexer/index-page-map.h diff --git a/drivers/md/dm-vdo/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c similarity index 99% rename from drivers/md/dm-vdo/index-session.c rename to drivers/md/dm-vdo/indexer/index-session.c index a482ccd3981e..07b478f57c68 100644 --- a/drivers/md/dm-vdo/index-session.c +++ b/drivers/md/dm-vdo/indexer/index-session.c @@ -7,13 +7,14 @@ #include -#include "funnel-requestqueue.h" -#include "index.h" -#include "index-layout.h" #include "logger.h" #include "memory-alloc.h" #include "time-utils.h" +#include "funnel-requestqueue.h" +#include "index.h" +#include "index-layout.h" + /* * The index session contains a lock (the request_mutex) which ensures that only one thread can * change the state of its index at a time. The state field indicates the current state of the diff --git a/drivers/md/dm-vdo/index-session.h b/drivers/md/dm-vdo/indexer/index-session.h similarity index 99% rename from drivers/md/dm-vdo/index-session.h rename to drivers/md/dm-vdo/indexer/index-session.h index 733d10f8a56c..066648f6e062 100644 --- a/drivers/md/dm-vdo/index-session.h +++ b/drivers/md/dm-vdo/indexer/index-session.h @@ -9,9 +9,10 @@ #include #include +#include "thread-utils.h" + #include "config.h" #include "indexer.h" -#include "thread-utils.h" /* * The index session mediates all interactions with a UDS index. Once the index session is created, diff --git a/drivers/md/dm-vdo/index.c b/drivers/md/dm-vdo/indexer/index.c similarity index 99% rename from drivers/md/dm-vdo/index.c rename to drivers/md/dm-vdo/indexer/index.c index 9d4a8e5cbaad..35e3b45cdb71 100644 --- a/drivers/md/dm-vdo/index.c +++ b/drivers/md/dm-vdo/indexer/index.c @@ -6,10 +6,11 @@ #include "index.h" -#include "funnel-requestqueue.h" -#include "hash-utils.h" #include "logger.h" #include "memory-alloc.h" + +#include "funnel-requestqueue.h" +#include "hash-utils.h" #include "sparse-cache.h" static const u64 NO_LAST_SAVE = U64_MAX; diff --git a/drivers/md/dm-vdo/index.h b/drivers/md/dm-vdo/indexer/index.h similarity index 100% rename from drivers/md/dm-vdo/index.h rename to drivers/md/dm-vdo/indexer/index.h diff --git a/drivers/md/dm-vdo/indexer.h b/drivers/md/dm-vdo/indexer/indexer.h similarity index 100% rename from drivers/md/dm-vdo/indexer.h rename to drivers/md/dm-vdo/indexer/indexer.h diff --git a/drivers/md/dm-vdo/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c similarity index 100% rename from drivers/md/dm-vdo/io-factory.c rename to drivers/md/dm-vdo/indexer/io-factory.c diff --git a/drivers/md/dm-vdo/io-factory.h b/drivers/md/dm-vdo/indexer/io-factory.h similarity index 100% rename from drivers/md/dm-vdo/io-factory.h rename to drivers/md/dm-vdo/indexer/io-factory.h diff --git a/drivers/md/dm-vdo/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c similarity index 99% rename from drivers/md/dm-vdo/open-chapter.c rename to drivers/md/dm-vdo/indexer/open-chapter.c index d9d6e5d45bfb..da16afaec07f 100644 --- a/drivers/md/dm-vdo/open-chapter.c +++ b/drivers/md/dm-vdo/indexer/open-chapter.c @@ -7,13 +7,14 @@ #include -#include "config.h" -#include "hash-utils.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" +#include "config.h" +#include "hash-utils.h" + /* * Each index zone has a dedicated open chapter zone structure which gets an equal share of the * open chapter space. Records are assigned to zones based on their record name. Within each zone, diff --git a/drivers/md/dm-vdo/open-chapter.h b/drivers/md/dm-vdo/indexer/open-chapter.h similarity index 100% rename from drivers/md/dm-vdo/open-chapter.h rename to drivers/md/dm-vdo/indexer/open-chapter.h diff --git a/drivers/md/dm-vdo/radix-sort.c b/drivers/md/dm-vdo/indexer/radix-sort.c similarity index 100% rename from drivers/md/dm-vdo/radix-sort.c rename to drivers/md/dm-vdo/indexer/radix-sort.c diff --git a/drivers/md/dm-vdo/radix-sort.h b/drivers/md/dm-vdo/indexer/radix-sort.h similarity index 100% rename from drivers/md/dm-vdo/radix-sort.h rename to drivers/md/dm-vdo/indexer/radix-sort.h diff --git a/drivers/md/dm-vdo/sparse-cache.c b/drivers/md/dm-vdo/indexer/sparse-cache.c similarity index 99% rename from drivers/md/dm-vdo/sparse-cache.c rename to drivers/md/dm-vdo/indexer/sparse-cache.c index b43a626a42de..f2141de6ed00 100644 --- a/drivers/md/dm-vdo/sparse-cache.c +++ b/drivers/md/dm-vdo/indexer/sparse-cache.c @@ -9,13 +9,14 @@ #include #include -#include "chapter-index.h" -#include "config.h" -#include "index.h" #include "logger.h" #include "memory-alloc.h" #include "permassert.h" +#include "chapter-index.h" +#include "config.h" +#include "index.h" + /* * Since the cache is small, it is implemented as a simple array of cache entries. Searching for a * specific virtual chapter is implemented as a linear search. The cache replacement policy is diff --git a/drivers/md/dm-vdo/sparse-cache.h b/drivers/md/dm-vdo/indexer/sparse-cache.h similarity index 100% rename from drivers/md/dm-vdo/sparse-cache.h rename to drivers/md/dm-vdo/indexer/sparse-cache.h diff --git a/drivers/md/dm-vdo/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c similarity index 99% rename from drivers/md/dm-vdo/volume-index.c rename to drivers/md/dm-vdo/indexer/volume-index.c index 36e3c2e3d799..8cbd9280c4bd 100644 --- a/drivers/md/dm-vdo/volume-index.c +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -10,17 +10,18 @@ #include #include -#include "config.h" #include "errors.h" -#include "geometry.h" -#include "hash-utils.h" -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "thread-utils.h" +#include "config.h" +#include "geometry.h" +#include "hash-utils.h" +#include "indexer.h" + /* * The volume index is a combination of two separate subindexes, one containing sparse hook entries * (retained for all chapters), and one containing the remaining entries (retained only for the diff --git a/drivers/md/dm-vdo/volume-index.h b/drivers/md/dm-vdo/indexer/volume-index.h similarity index 99% rename from drivers/md/dm-vdo/volume-index.h rename to drivers/md/dm-vdo/indexer/volume-index.h index 66bf14fddc90..583998c547b7 100644 --- a/drivers/md/dm-vdo/volume-index.h +++ b/drivers/md/dm-vdo/indexer/volume-index.h @@ -8,10 +8,11 @@ #include +#include "thread-utils.h" + #include "config.h" #include "delta-index.h" #include "indexer.h" -#include "thread-utils.h" /* * The volume index is the primary top-level index for UDS. It contains records which map a record diff --git a/drivers/md/dm-vdo/volume.c b/drivers/md/dm-vdo/indexer/volume.c similarity index 99% rename from drivers/md/dm-vdo/volume.c rename to drivers/md/dm-vdo/indexer/volume.c index 60416dc8a9d7..eca83b6cab35 100644 --- a/drivers/md/dm-vdo/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -9,19 +9,20 @@ #include #include -#include "chapter-index.h" -#include "config.h" #include "errors.h" -#include "geometry.h" -#include "hash-utils.h" -#include "index.h" #include "logger.h" #include "memory-alloc.h" #include "permassert.h" -#include "sparse-cache.h" #include "string-utils.h" #include "thread-utils.h" +#include "chapter-index.h" +#include "config.h" +#include "geometry.h" +#include "hash-utils.h" +#include "index.h" +#include "sparse-cache.h" + /* * The first block of the volume layout is reserved for the volume header, which is no longer used. * The remainder of the volume is divided into chapters consisting of several pages of records, and diff --git a/drivers/md/dm-vdo/volume.h b/drivers/md/dm-vdo/indexer/volume.h similarity index 99% rename from drivers/md/dm-vdo/volume.h rename to drivers/md/dm-vdo/indexer/volume.h index 290de5cbf9ec..8679a5e55347 100644 --- a/drivers/md/dm-vdo/volume.h +++ b/drivers/md/dm-vdo/indexer/volume.h @@ -11,16 +11,17 @@ #include #include +#include "permassert.h" +#include "thread-utils.h" + #include "chapter-index.h" #include "config.h" #include "geometry.h" #include "indexer.h" #include "index-layout.h" #include "index-page-map.h" -#include "permassert.h" #include "radix-sort.h" #include "sparse-cache.h" -#include "thread-utils.h" /* * The volume manages deduplication records on permanent storage. The term "volume" can also refer diff --git a/drivers/md/dm-vdo/uds-sysfs.c b/drivers/md/dm-vdo/uds-sysfs.c index 1548092e7de1..2c4fb277ba38 100644 --- a/drivers/md/dm-vdo/uds-sysfs.c +++ b/drivers/md/dm-vdo/uds-sysfs.c @@ -9,11 +9,12 @@ #include #include -#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "string-utils.h" +#include "indexer.h" + #define UDS_SYSFS_NAME "uds" static struct { From eebd4e163024944ed53b6ad6d147e49df24dba32 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 21 Feb 2024 17:17:31 +0800 Subject: [PATCH 0936/1038] dm vdo: fix various function names referenced in comment blocks No functional modification involved. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/admin-state.c | 2 +- drivers/md/dm-vdo/dedupe.c | 2 +- drivers/md/dm-vdo/encodings.c | 2 +- drivers/md/dm-vdo/flush.c | 2 +- drivers/md/dm-vdo/logical-zone.c | 4 ++-- drivers/md/dm-vdo/physical-zone.c | 2 +- drivers/md/dm-vdo/slab-depot.c | 6 +++--- drivers/md/dm-vdo/vdo.c | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/md/dm-vdo/admin-state.c b/drivers/md/dm-vdo/admin-state.c index 94533a802edb..1423f4cebb8a 100644 --- a/drivers/md/dm-vdo/admin-state.c +++ b/drivers/md/dm-vdo/admin-state.c @@ -300,7 +300,7 @@ static bool check_code(bool valid, const struct admin_state_code *code, const ch } /** - * vdo_drain_operation() - Check that an operation is a drain. + * assert_vdo_drain_operation() - Check that an operation is a drain. * @waiter The completion to finish with an error if the operation is not a drain. * * Return: true if the specified operation is a drain. diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 9468d7fad443..c8018c5799bd 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -2839,7 +2839,7 @@ static const char *index_state_to_string(struct hash_zones *zones, } /** - * vdo_dump_hash_zone() - Dump information about a hash zone to the log for debugging. + * dump_hash_zone() - Dump information about a hash zone to the log for debugging. * @zone: The zone to dump. */ static void dump_hash_zone(const struct hash_zone *zone) diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index 9e45411fe816..5012042b26af 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -1172,7 +1172,7 @@ static struct vdo_component unpack_vdo_component_41_0(struct packed_vdo_componen } /** - * vdo_decode_component() - Decode the component data for the vdo itself out of the super block. + * decode_vdo_component() - Decode the component data for the vdo itself out of the super block. * * Return: VDO_SUCCESS or an error. */ diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index 391b6203efc6..1bc13470a608 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -88,7 +88,7 @@ static inline struct vdo_flush *completion_as_vdo_flush(struct vdo_completion *c } /** - * waiter_as_flush() - Convert a vdo_flush's generic wait queue entry back to the vdo_flush. + * vdo_waiter_as_flush() - Convert a vdo_flush's generic wait queue entry back to the vdo_flush. * @waiter: The wait queue entry to convert. * * Return: The wait queue entry as a vdo_flush. diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index cfbf1701ca84..e11f9c859349 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -295,8 +295,8 @@ static void notify_flusher(struct vdo_completion *completion) } /** - * void attempt_generation_complete_notification() - Notify the flusher if some generation no - * longer has active VIOs. + * attempt_generation_complete_notification() - Notify the flusher if some generation no + * longer has active VIOs. * @completion: The zone completion. */ static void attempt_generation_complete_notification(struct vdo_completion *completion) diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index 3bcf6f1ba77f..a9380966b56f 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -258,7 +258,7 @@ static int make_pbn_lock_pool(size_t capacity, struct pbn_lock_pool **pool_ptr) } /** - * vdo_free_pbn_lock_pool() - Free a PBN lock pool. + * free_pbn_lock_pool() - Free a PBN lock pool. * @pool: The lock pool to free. * * This also frees all the PBN locks it allocated, so the caller must ensure that all locks have diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index e10c3db8e665..8c6376e79a23 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -2035,8 +2035,8 @@ static inline slab_block_number find_zero_byte_in_word(const u8 *word_ptr, } /** - * vdo_find_free_block() - Find the first block with a reference count of zero in the specified - * range of reference counter indexes. + * find_free_block() - Find the first block with a reference count of zero in the specified + * range of reference counter indexes. * @slab: The slab counters to scan. * @index_ptr: A pointer to hold the array index of the free block. * @@ -5032,7 +5032,7 @@ get_ref_counts_statistics(const struct slab_depot *depot) } /** - * get_depot_slab_journal_statistics() - Get the aggregated slab journal statistics for the depot. + * get_slab_journal_statistics() - Get the aggregated slab journal statistics for the depot. * @depot: The slab depot. * * Return: The aggregated statistics for all slab journals in the depot. diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index a40f059d39b3..c161dffeb91a 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -267,8 +267,8 @@ static int __must_check initialize_thread_config(struct thread_count_config coun } /** - * vdo_read_geometry_block() - Synchronously read the geometry block from a vdo's underlying block - * device. + * read_geometry_block() - Synchronously read the geometry block from a vdo's underlying block + * device. * @vdo: The vdo whose geometry is to be read. * * Return: VDO_SUCCESS or an error code. From bbe434d94e007d3db258cfbc16336290af39d0bc Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Fri, 19 Jan 2024 22:27:27 -0500 Subject: [PATCH 0937/1038] dm vdo indexer delta-index: fix typos in comments Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/indexer/delta-index.c | 2 +- drivers/md/dm-vdo/indexer/delta-index.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c index 4aace707545a..6a85f93bbcb1 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -35,7 +35,7 @@ * * Each delta list is stored as a bit stream. Within the delta list encoding, bits and bytes are * numbered in little endian order. Within a byte, bit 0 is the least significant bit (0x1), and - * bit 7 is the most significant bit (0x80). Within a bit stream, bit 7 is the most signficant bit + * bit 7 is the most significant bit (0x80). Within a bit stream, bit 7 is the most significant bit * of byte 0, and bit 8 is the least significant bit of byte 1. Within a byte array, a byte's * number corresponds to its index in the array. * diff --git a/drivers/md/dm-vdo/indexer/delta-index.h b/drivers/md/dm-vdo/indexer/delta-index.h index 3d2ea19aef61..53f6c6ac0bc7 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.h +++ b/drivers/md/dm-vdo/indexer/delta-index.h @@ -141,7 +141,7 @@ struct delta_index_page { * record at the end of the list. * * (5) If at_end is false and is_collision is true, the delta_list entry fields refer to a - * collision entry in the list, and the delta_list entry can be used a a reference to this + * collision entry in the list, and the delta_list entry can be used as a reference to this * entry. * * (6) If at_end is false and is_collision is false, the delta_list entry fields refer to a From e1e510fcad19c67e122003028c15293115a5b9d7 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 22 Feb 2024 20:44:29 -0500 Subject: [PATCH 0938/1038] dm vdo: update module comments Update outdated comments referring to separate VDO and UDS modules. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dm-vdo-target.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 0114fa4d48a2..429a285cbc4a 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -2907,9 +2907,7 @@ static int __init vdo_init(void) { int result = 0; - /* - * UDS module level initialization must be done first, as VDO initialization depends on it - */ + /* Memory tracking must be initialized first for accurate accounting. */ uds_memory_init(); uds_init_sysfs(); @@ -2917,7 +2915,7 @@ static int __init vdo_init(void) vdo_initialize_device_registry_once(); uds_log_info("loaded version %s", CURRENT_VERSION); - /* Add VDO errors to the already existing set of errors in UDS. */ + /* Add VDO errors to the set of errors registered by the indexer. */ result = vdo_register_status_codes(); if (result != UDS_SUCCESS) { uds_log_error("vdo_register_status_codes failed %d", result); @@ -2939,11 +2937,8 @@ static int __init vdo_init(void) static void __exit vdo_exit(void) { vdo_module_destroy(); - /* - * UDS module level exit processing must be done after all VDO module exit processing is - * complete. - */ uds_put_sysfs(); + /* Memory tracking cleanup must be done last. */ uds_memory_exit(); } From 04530b487bf35962ac3fbf490a6ca07d7a5d8869 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 22 Feb 2024 20:48:16 -0500 Subject: [PATCH 0939/1038] dm vdo: remove outdated pointer_map reference Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/vdo.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index c161dffeb91a..d16404588881 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -67,10 +67,7 @@ struct sync_completion { struct completion completion; }; -/* - * We don't expect this set to ever get really large, so a linked list is adequate. We can use a - * pointer_map if we need to later. - */ +/* A linked list is adequate for the small number of entries we expect. */ struct device_registry { struct list_head links; /* TODO: Convert to rcu per kernel recommendation. */ From 6008d526b06bab2cbea194948aec0cd04461dcc6 Mon Sep 17 00:00:00 2001 From: Bruce Johnston Date: Mon, 26 Feb 2024 17:04:43 -0500 Subject: [PATCH 0940/1038] dm-vdo: change unnamed enums to defines Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 6 ++-- drivers/md/dm-vdo/data-vio.c | 8 ++--- drivers/md/dm-vdo/dedupe.c | 20 ++++------- drivers/md/dm-vdo/dm-vdo-target.c | 28 +++++++-------- drivers/md/dm-vdo/dump.c | 11 +++--- drivers/md/dm-vdo/encodings.c | 4 +-- drivers/md/dm-vdo/errors.c | 4 +-- drivers/md/dm-vdo/indexer/config.c | 10 +++--- drivers/md/dm-vdo/indexer/delta-index.c | 40 ++++++---------------- drivers/md/dm-vdo/indexer/index-layout.c | 20 ++++------- drivers/md/dm-vdo/indexer/index-page-map.c | 4 +-- drivers/md/dm-vdo/indexer/io-factory.c | 2 +- drivers/md/dm-vdo/indexer/open-chapter.c | 8 ++--- drivers/md/dm-vdo/indexer/radix-sort.c | 6 ++-- drivers/md/dm-vdo/indexer/sparse-cache.c | 6 ++-- drivers/md/dm-vdo/indexer/volume-index.c | 6 ++-- drivers/md/dm-vdo/indexer/volume.c | 12 +++---- drivers/md/dm-vdo/int-map.c | 12 +++---- drivers/md/dm-vdo/logical-zone.c | 4 +-- drivers/md/dm-vdo/packer.c | 4 +-- drivers/md/dm-vdo/physical-zone.c | 6 ++-- drivers/md/dm-vdo/priority-table.c | 4 +-- drivers/md/dm-vdo/recovery-journal.c | 16 ++++----- drivers/md/dm-vdo/thread-utils.c | 8 ++--- drivers/md/dm-vdo/vdo.c | 2 +- 25 files changed, 91 insertions(+), 160 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 5be400743c03..b7274e94b269 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -114,10 +114,8 @@ const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY = { .pbn_low_word = __cpu_to_le32(VDO_ZERO_BLOCK & UINT_MAX), }; -enum { - LOG_INTERVAL = 4000, - DISPLAY_INTERVAL = 100000, -}; +#define LOG_INTERVAL 4000 +#define DISPLAY_INTERVAL 100000 /* * For adjusting VDO page cache statistic fields which are only mutated on the logical zone thread. diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index f6c32dc9a822..1630993e536f 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -114,9 +114,7 @@ static blk_opf_t PASSTHROUGH_FLAGS = (REQ_PRIO | REQ_META | REQ_SYNC | REQ_RAHEA * them are awakened. */ -enum { - DATA_VIO_RELEASE_BATCH_SIZE = 128, -}; +#define DATA_VIO_RELEASE_BATCH_SIZE 128 static const unsigned int VDO_SECTORS_PER_BLOCK_MASK = VDO_SECTORS_PER_BLOCK - 1; static const u32 COMPRESSION_STATUS_MASK = 0xff; @@ -1044,8 +1042,8 @@ void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios) * In order that syslog can empty its buffer, sleep after 35 elements for 4ms (till the * second clock tick). These numbers were picked based on experiments with lab machines. */ - enum { ELEMENTS_PER_BATCH = 35 }; - enum { SLEEP_FOR_SYSLOG = 4000 }; + static const int ELEMENTS_PER_BATCH = 35; + static const int SLEEP_FOR_SYSLOG = 4000; if (pool == NULL) return; diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index c8018c5799bd..b453a204239a 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -154,11 +154,9 @@ struct uds_attribute { const char *(*show_string)(struct hash_zones *hash_zones); }; -enum timer_state { - DEDUPE_QUERY_TIMER_IDLE, - DEDUPE_QUERY_TIMER_RUNNING, - DEDUPE_QUERY_TIMER_FIRED, -}; +#define DEDUPE_QUERY_TIMER_IDLE 0 +#define DEDUPE_QUERY_TIMER_RUNNING 1 +#define DEDUPE_QUERY_TIMER_FIRED 2 enum dedupe_context_state { DEDUPE_CONTEXT_IDLE, @@ -185,11 +183,9 @@ static const char *SUSPENDED = "suspended"; static const char *UNKNOWN = "unknown"; /* Version 2 uses the kernel space UDS index and is limited to 16 bytes */ -enum { - UDS_ADVICE_VERSION = 2, - /* version byte + state byte + 64-bit little-endian PBN */ - UDS_ADVICE_SIZE = 1 + 1 + sizeof(u64), -}; +#define UDS_ADVICE_VERSION 2 +/* version byte + state byte + 64-bit little-endian PBN */ +#define UDS_ADVICE_SIZE (1 + 1 + sizeof(u64)) enum hash_lock_state { /* State for locks that are not in use or are being initialized. */ @@ -279,9 +275,7 @@ struct hash_lock { struct vdo_wait_queue waiters; }; -enum { - LOCK_POOL_CAPACITY = MAXIMUM_VDO_USER_VIOS, -}; +#define LOCK_POOL_CAPACITY MAXIMUM_VDO_USER_VIOS struct hash_zones { struct action_manager *manager; diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 429a285cbc4a..175ee56a89e1 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -42,7 +42,7 @@ #define CURRENT_VERSION "8.3.0.65" -enum { +enum admin_phases { GROW_LOGICAL_PHASE_START, GROW_LOGICAL_PHASE_GROW_BLOCK_MAP, GROW_LOGICAL_PHASE_END, @@ -142,10 +142,8 @@ static const char * const ADMIN_PHASE_NAMES[] = { "SUSPEND_PHASE_END", }; -enum { - /* If we bump this, update the arrays below */ - TABLE_VERSION = 4, -}; +/* If we bump this, update the arrays below */ +#define TABLE_VERSION 4 /* arrays for handling different table versions */ static const u8 REQUIRED_ARGC[] = { 10, 12, 9, 7, 6 }; @@ -159,17 +157,15 @@ static const u8 POOL_NAME_ARG_INDEX[] = { 8, 10, 8 }; * need to scan 16 words, so it's not likely to be a big deal compared to other resource usage. */ -enum { - /* - * This minimum size for the bit array creates a numbering space of 0-999, which allows - * successive starts of the same volume to have different instance numbers in any - * reasonably-sized test. Changing instances on restart allows vdoMonReport to detect that - * the ephemeral stats have reset to zero. - */ - BIT_COUNT_MINIMUM = 1000, - /** Grow the bit array by this many bits when needed */ - BIT_COUNT_INCREMENT = 100, -}; +/* + * This minimum size for the bit array creates a numbering space of 0-999, which allows + * successive starts of the same volume to have different instance numbers in any + * reasonably-sized test. Changing instances on restart allows vdoMonReport to detect that + * the ephemeral stats have reset to zero. + */ +#define BIT_COUNT_MINIMUM 1000 +/* Grow the bit array by this many bits when needed */ +#define BIT_COUNT_INCREMENT 100 struct instance_tracker { unsigned int bit_count; diff --git a/drivers/md/dm-vdo/dump.c b/drivers/md/dm-vdo/dump.c index 91bc8ed36aa7..2a0890b54186 100644 --- a/drivers/md/dm-vdo/dump.c +++ b/drivers/md/dm-vdo/dump.c @@ -41,10 +41,10 @@ enum dump_option_flags { FLAG_SKIP_DEFAULT = (1 << SKIP_DEFAULT) }; -enum { - FLAGS_ALL_POOLS = (FLAG_SHOW_VIO_POOL), - DEFAULT_DUMP_FLAGS = (FLAG_SHOW_QUEUES | FLAG_SHOW_VDO_STATUS) -}; +#define FLAGS_ALL_POOLS (FLAG_SHOW_VIO_POOL) +#define DEFAULT_DUMP_FLAGS (FLAG_SHOW_QUEUES | FLAG_SHOW_VDO_STATUS) +/* Another static buffer... log10(256) = 2.408+, round up: */ +#define DIGITS_PER_U64 (1 + sizeof(u64) * 2409 / 1000) static inline bool is_arg_string(const char *arg, const char *this_option) { @@ -222,9 +222,6 @@ void dump_data_vio(void *data) * one does run, the log output will be garbled anyway. */ static char vio_completion_dump_buffer[100 + MAX_VDO_WORK_QUEUE_NAME_LEN]; - /* Another static buffer... log10(256) = 2.408+, round up: */ - enum { DIGITS_PER_U64 = 1 + sizeof(u64) * 2409 / 1000 }; - static char vio_block_number_dump_buffer[sizeof("P L D") + 3 * DIGITS_PER_U64]; static char vio_flush_generation_buffer[sizeof(" FG") + DIGITS_PER_U64]; static char flags_dump_buffer[8]; diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index 5012042b26af..4a0a6afea670 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -55,9 +55,7 @@ static const struct header GEOMETRY_BLOCK_HEADER_4_0 = { const u8 VDO_GEOMETRY_MAGIC_NUMBER[VDO_GEOMETRY_MAGIC_NUMBER_SIZE + 1] = "dmvdo001"; -enum { - PAGE_HEADER_4_1_SIZE = 8 + 8 + 8 + 1 + 1 + 1 + 1, -}; +#define PAGE_HEADER_4_1_SIZE (8 + 8 + 8 + 1 + 1 + 1 + 1) static const struct version_number BLOCK_MAP_4_1 = { .major_version = 4, diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c index e8599599a377..dc1f0533bd7a 100644 --- a/drivers/md/dm-vdo/errors.c +++ b/drivers/md/dm-vdo/errors.c @@ -79,9 +79,7 @@ struct error_block { const struct error_info *infos; }; -enum { - MAX_ERROR_BLOCKS = 6, -}; +#define MAX_ERROR_BLOCKS 6 static struct { int allocated; diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c index 0bf315e7b5d1..b572350a3d5f 100644 --- a/drivers/md/dm-vdo/indexer/config.c +++ b/drivers/md/dm-vdo/indexer/config.c @@ -15,12 +15,10 @@ static const u8 INDEX_CONFIG_MAGIC[] = "ALBIC"; static const u8 INDEX_CONFIG_VERSION_6_02[] = "06.02"; static const u8 INDEX_CONFIG_VERSION_8_02[] = "08.02"; -enum { - DEFAULT_VOLUME_READ_THREADS = 2, - MAX_VOLUME_READ_THREADS = 16, - INDEX_CONFIG_MAGIC_LENGTH = sizeof(INDEX_CONFIG_MAGIC) - 1, - INDEX_CONFIG_VERSION_LENGTH = sizeof(INDEX_CONFIG_VERSION_6_02) - 1, -}; +#define DEFAULT_VOLUME_READ_THREADS 2 +#define MAX_VOLUME_READ_THREADS 16 +#define INDEX_CONFIG_MAGIC_LENGTH (sizeof(INDEX_CONFIG_MAGIC) - 1) +#define INDEX_CONFIG_VERSION_LENGTH ((int)(sizeof(INDEX_CONFIG_VERSION_6_02) - 1)) static bool is_version(const u8 *version, u8 *buffer) { diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c index 6a85f93bbcb1..8eece0ba6d93 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -70,17 +70,13 @@ * This is the largest field size supported by get_field() and set_field(). Any field that is * larger is not guaranteed to fit in a single byte-aligned u32. */ -enum { - MAX_FIELD_BITS = (sizeof(u32) - 1) * BITS_PER_BYTE + 1, -}; +#define MAX_FIELD_BITS ((sizeof(u32) - 1) * BITS_PER_BYTE + 1) /* * This is the largest field size supported by get_big_field() and set_big_field(). Any field that * is larger is not guaranteed to fit in a single byte-aligned u64. */ -enum { - MAX_BIG_FIELD_BITS = (sizeof(u64) - 1) * BITS_PER_BYTE + 1, -}; +#define MAX_BIG_FIELD_BITS ((sizeof(u64) - 1) * BITS_PER_BYTE + 1) /* * This is the number of guard bytes needed at the end of the memory byte array when using the bit @@ -88,45 +84,33 @@ enum { * bytes beyond the end of the desired field. The definition is written to make it clear how this * value is derived. */ -enum { - POST_FIELD_GUARD_BYTES = sizeof(u64) - 1, -}; +#define POST_FIELD_GUARD_BYTES (sizeof(u64) - 1) /* The number of guard bits that are needed in the tail guard list */ -enum { - GUARD_BITS = POST_FIELD_GUARD_BYTES * BITS_PER_BYTE -}; +#define GUARD_BITS (POST_FIELD_GUARD_BYTES * BITS_PER_BYTE) /* * The maximum size of a single delta list in bytes. We count guard bytes in this value because a * buffer of this size can be used with move_bits(). */ -enum { - DELTA_LIST_MAX_BYTE_COUNT = - ((U16_MAX + BITS_PER_BYTE) / BITS_PER_BYTE + POST_FIELD_GUARD_BYTES) -}; +#define DELTA_LIST_MAX_BYTE_COUNT \ + ((U16_MAX + BITS_PER_BYTE) / BITS_PER_BYTE + POST_FIELD_GUARD_BYTES) /* The number of extra bytes and bits needed to store a collision entry */ -enum { - COLLISION_BYTES = UDS_RECORD_NAME_SIZE, - COLLISION_BITS = COLLISION_BYTES * BITS_PER_BYTE -}; +#define COLLISION_BYTES UDS_RECORD_NAME_SIZE +#define COLLISION_BITS (COLLISION_BYTES * BITS_PER_BYTE) /* * Immutable delta lists are packed into pages containing a header that encodes the delta list * information into 19 bits per list (64KB bit offset). */ - -enum { IMMUTABLE_HEADER_SIZE = 19 }; +#define IMMUTABLE_HEADER_SIZE 19 /* * Constants and structures for the saved delta index. "DI" is for delta_index, and -##### is a * number to increment when the format of the data changes. */ - -enum { - MAGIC_SIZE = 8, -}; +#define MAGIC_SIZE 8 static const char DELTA_INDEX_MAGIC[] = "DI-00002"; @@ -216,9 +200,7 @@ static void rebalance_delta_zone(const struct delta_zone *delta_zone, u32 first, static inline size_t get_zone_memory_size(unsigned int zone_count, size_t memory_size) { /* Round up so that each zone is a multiple of 64K in size. */ - enum { - ALLOC_BOUNDARY = 64 * 1024, - }; + size_t ALLOC_BOUNDARY = 64 * 1024; return (memory_size / zone_count + ALLOC_BOUNDARY - 1) & -ALLOC_BOUNDARY; } diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index af533aa270a8..a0227a75814b 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -54,11 +54,9 @@ * Each save also has a unique nonce. */ -enum { - MAGIC_SIZE = 32, - NONCE_INFO_SIZE = 32, - MAX_SAVES = 2, -}; +#define MAGIC_SIZE 32 +#define NONCE_INFO_SIZE 32 +#define MAX_SAVES 2 enum region_kind { RL_KIND_EMPTY = 0, @@ -82,9 +80,7 @@ enum region_type { RH_TYPE_UNSAVED = 4, }; -enum { - RL_SOLE_INSTANCE = 65535, -}; +#define RL_SOLE_INSTANCE 65535 /* * Super block version 2 is the first released version. @@ -98,11 +94,9 @@ enum { * order to make room to prepend LVM metadata to a volume originally created without lvm. This * allows the index to retain most its deduplication records. */ -enum { - SUPER_VERSION_MINIMUM = 3, - SUPER_VERSION_CURRENT = 3, - SUPER_VERSION_MAXIMUM = 7, -}; +#define SUPER_VERSION_MINIMUM 3 +#define SUPER_VERSION_CURRENT 3 +#define SUPER_VERSION_MAXIMUM 7 static const u8 LAYOUT_MAGIC[MAGIC_SIZE] = "*ALBIREO*SINGLE*FILE*LAYOUT*001*"; static const u64 REGION_MAGIC = 0x416c6252676e3031; /* 'AlbRgn01' */ diff --git a/drivers/md/dm-vdo/indexer/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c index 90d97c33a9c3..37037ac8eee9 100644 --- a/drivers/md/dm-vdo/indexer/index-page-map.c +++ b/drivers/md/dm-vdo/indexer/index-page-map.c @@ -25,9 +25,7 @@ static const u8 PAGE_MAP_MAGIC[] = "ALBIPM02"; -enum { - PAGE_MAP_MAGIC_LENGTH = sizeof(PAGE_MAP_MAGIC) - 1, -}; +#define PAGE_MAP_MAGIC_LENGTH (sizeof(PAGE_MAP_MAGIC) - 1) static inline u32 get_entry_count(const struct index_geometry *geometry) { diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c index 02242df94e37..fecd436986ae 100644 --- a/drivers/md/dm-vdo/indexer/io-factory.c +++ b/drivers/md/dm-vdo/indexer/io-factory.c @@ -37,7 +37,7 @@ struct buffered_reader { u8 *end; }; -enum { MAX_READ_AHEAD_BLOCKS = 4 }; +#define MAX_READ_AHEAD_BLOCKS 4 /* * The buffered writer allows efficient I/O by buffering writes and committing page-sized segments diff --git a/drivers/md/dm-vdo/indexer/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c index da16afaec07f..cd2d35e39c20 100644 --- a/drivers/md/dm-vdo/indexer/open-chapter.c +++ b/drivers/md/dm-vdo/indexer/open-chapter.c @@ -46,11 +46,9 @@ static const u8 OPEN_CHAPTER_MAGIC[] = "ALBOC"; static const u8 OPEN_CHAPTER_VERSION[] = "02.00"; -enum { - OPEN_CHAPTER_MAGIC_LENGTH = sizeof(OPEN_CHAPTER_MAGIC) - 1, - OPEN_CHAPTER_VERSION_LENGTH = sizeof(OPEN_CHAPTER_VERSION) - 1, - LOAD_RATIO = 2, -}; +#define OPEN_CHAPTER_MAGIC_LENGTH (sizeof(OPEN_CHAPTER_MAGIC) - 1) +#define OPEN_CHAPTER_VERSION_LENGTH (sizeof(OPEN_CHAPTER_VERSION) - 1) +#define LOAD_RATIO 2 static inline size_t records_size(const struct open_chapter_zone *open_chapter) { diff --git a/drivers/md/dm-vdo/indexer/radix-sort.c b/drivers/md/dm-vdo/indexer/radix-sort.c index 1f17c708a652..b86d55f0827e 100644 --- a/drivers/md/dm-vdo/indexer/radix-sort.c +++ b/drivers/md/dm-vdo/indexer/radix-sort.c @@ -17,10 +17,8 @@ * keys to be sorted. */ -enum { - /* Piles smaller than this are handled with a simple insertion sort. */ - INSERTION_SORT_THRESHOLD = 12, -}; +/* Piles smaller than this are handled with a simple insertion sort. */ +#define INSERTION_SORT_THRESHOLD 12 /* Sort keys are pointers to immutable fixed-length arrays of bytes. */ typedef const u8 *sort_key_t; diff --git a/drivers/md/dm-vdo/indexer/sparse-cache.c b/drivers/md/dm-vdo/indexer/sparse-cache.c index f2141de6ed00..9e8672cba3fa 100644 --- a/drivers/md/dm-vdo/indexer/sparse-cache.c +++ b/drivers/md/dm-vdo/indexer/sparse-cache.c @@ -77,10 +77,8 @@ * considered to be a member of the cache for uds_sparse_cache_contains(). */ -enum { - SKIP_SEARCH_THRESHOLD = 20000, - ZONE_ZERO = 0, -}; +#define SKIP_SEARCH_THRESHOLD 20000 +#define ZONE_ZERO 0 /* * These counters are essentially fields of the struct cached_chapter_index, but are segregated diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c index 8cbd9280c4bd..a88e515ceef6 100644 --- a/drivers/md/dm-vdo/indexer/volume-index.c +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -94,7 +94,8 @@ struct chapter_range { u32 chapter_count; }; -enum { MAGIC_SIZE = 8 }; +#define MAGIC_SIZE 8 + static const char MAGIC_START_5[] = "MI5-0005"; struct sub_index_data { @@ -193,10 +194,11 @@ unsigned int uds_get_volume_index_zone(const struct volume_index *volume_index, return get_volume_sub_index_zone(get_volume_sub_index(volume_index, name), name); } +#define DELTA_LIST_SIZE 256 + static int compute_volume_sub_index_parameters(const struct uds_configuration *config, struct sub_index_parameters *params) { - enum { DELTA_LIST_SIZE = 256 }; u64 entries_in_volume_index, address_span; u32 chapters_in_volume_index, invalid_chapters; u32 rounded_chapters; diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c index eca83b6cab35..002a4264a163 100644 --- a/drivers/md/dm-vdo/indexer/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -60,13 +60,11 @@ * in-memory volume index. */ -enum { - /* The maximum allowable number of contiguous bad chapters */ - MAX_BAD_CHAPTERS = 100, - VOLUME_CACHE_MAX_ENTRIES = (U16_MAX >> 1), - VOLUME_CACHE_QUEUED_FLAG = (1 << 15), - VOLUME_CACHE_MAX_QUEUED_READS = 4096, -}; +/* The maximum allowable number of contiguous bad chapters */ +#define MAX_BAD_CHAPTERS 100 +#define VOLUME_CACHE_MAX_ENTRIES (U16_MAX >> 1) +#define VOLUME_CACHE_QUEUED_FLAG (1 << 15) +#define VOLUME_CACHE_MAX_QUEUED_READS 4096 static const u64 BAD_CHAPTER = U64_MAX; diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index 99ccbb1339c6..e0953e013f15 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -56,13 +56,11 @@ #include "numeric.h" #include "permassert.h" -enum { - DEFAULT_CAPACITY = 16, /* the number of neighborhoods in a new table */ - NEIGHBORHOOD = 255, /* the number of buckets in each neighborhood */ - MAX_PROBES = 1024, /* limit on the number of probes for a free bucket */ - NULL_HOP_OFFSET = 0, /* the hop offset value terminating the hop list */ - DEFAULT_LOAD = 75 /* a compromise between memory use and performance */ -}; +#define DEFAULT_CAPACITY 16 /* the number of neighborhoods in a new table */ +#define NEIGHBORHOOD 255 /* the number of buckets in each neighborhood */ +#define MAX_PROBES 1024 /* limit on the number of probes for a free bucket */ +#define NULL_HOP_OFFSET 0 /* the hop offset value terminating the hop list */ +#define DEFAULT_LOAD 75 /* a compromise between memory use and performance */ /** * struct bucket - hash bucket diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index e11f9c859349..c5b3b1c111e3 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -21,9 +21,7 @@ #include "physical-zone.h" #include "vdo.h" -enum { - ALLOCATIONS_PER_ZONE = 128, -}; +#define ALLOCATIONS_PER_ZONE 128 /** * as_logical_zone() - Convert a generic vdo_completion to a logical_zone. diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c index b0ffb21ec436..3283c8d56c59 100644 --- a/drivers/md/dm-vdo/packer.c +++ b/drivers/md/dm-vdo/packer.c @@ -30,9 +30,7 @@ static const struct version_number COMPRESSED_BLOCK_1_0 = { .minor_version = 0, }; -enum { - COMPRESSED_BLOCK_1_0_SIZE = 4 + 4 + (2 * VDO_MAX_COMPRESSION_SLOTS), -}; +#define COMPRESSED_BLOCK_1_0_SIZE (4 + 4 + (2 * VDO_MAX_COMPRESSION_SLOTS)) /** * vdo_get_compressed_block_fragment() - Get a reference to a compressed fragment from a compressed diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index a9380966b56f..62d142b28282 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -23,10 +23,8 @@ #include "status-codes.h" #include "vdo.h" -enum { - /* Each user data_vio needs a PBN read lock and write lock. */ - LOCK_POOL_CAPACITY = 2 * MAXIMUM_VDO_USER_VIOS, -}; +/* Each user data_vio needs a PBN read lock and write lock. */ +#define LOCK_POOL_CAPACITY (2 * MAXIMUM_VDO_USER_VIOS) struct pbn_lock_implementation { enum pbn_lock_type type; diff --git a/drivers/md/dm-vdo/priority-table.c b/drivers/md/dm-vdo/priority-table.c index 9408219b5700..a59e9d40ca90 100644 --- a/drivers/md/dm-vdo/priority-table.c +++ b/drivers/md/dm-vdo/priority-table.c @@ -14,9 +14,7 @@ #include "status-codes.h" /* We use a single 64-bit search vector, so the maximum priority is 63 */ -enum { - MAX_PRIORITY = 63 -}; +#define MAX_PRIORITY 63 /* * All the entries with the same priority are queued in a circular list in a bucket for that diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 1e15bfe42cfc..c1d355346bcf 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -26,15 +26,13 @@ static const u64 RECOVERY_COUNT_MASK = 0xff; -enum { - /* - * The number of reserved blocks must be large enough to prevent a new recovery journal - * block write from overwriting a block which appears to still be a valid head block of the - * journal. Currently, that means reserving enough space for all 2048 data_vios. - */ - RECOVERY_JOURNAL_RESERVED_BLOCKS = - (MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2, -}; +/* + * The number of reserved blocks must be large enough to prevent a new recovery journal + * block write from overwriting a block which appears to still be a valid head block of the + * journal. Currently, that means reserving enough space for all 2048 data_vios. + */ +#define RECOVERY_JOURNAL_RESERVED_BLOCKS \ + ((MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2) /** * DOC: Lock Counters. diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index aeca14bba852..244abc6ad848 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -27,11 +27,9 @@ struct thread { struct completion thread_done; }; -enum { - ONCE_NOT_DONE = 0, - ONCE_IN_PROGRESS = 1, - ONCE_COMPLETE = 2, -}; +#define ONCE_NOT_DONE 0 +#define ONCE_IN_PROGRESS 1 +#define ONCE_COMPLETE 2 /* Run a function once only, and record that fact in the atomic value. */ void vdo_perform_once(atomic_t *once, void (*function)(void)) diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index d16404588881..6baf319d79c6 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -60,7 +60,7 @@ #include "status-codes.h" #include "vio.h" -enum { PARANOID_THREAD_CONSISTENCY_CHECKS = 0 }; +#define PARANOID_THREAD_CONSISTENCY_CHECKS 0 struct sync_completion { struct vdo_completion vdo_completion; From 0eea6b6e78daa45ca13e9b186da042f9b6139b50 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 10:55:50 -0600 Subject: [PATCH 0941/1038] dm vdo memory-alloc: change from uds_ to vdo_ namespace Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/action-manager.c | 2 +- drivers/md/dm-vdo/admin-state.c | 2 +- drivers/md/dm-vdo/block-map.c | 60 ++++++------ drivers/md/dm-vdo/data-vio.c | 24 ++--- drivers/md/dm-vdo/dedupe.c | 34 +++---- drivers/md/dm-vdo/dm-vdo-target.c | 56 +++++------ drivers/md/dm-vdo/dump.c | 2 +- drivers/md/dm-vdo/encodings.c | 4 +- drivers/md/dm-vdo/flush.c | 12 +-- drivers/md/dm-vdo/funnel-queue.c | 4 +- drivers/md/dm-vdo/funnel-workqueue.c | 30 +++--- drivers/md/dm-vdo/indexer/chapter-index.c | 6 +- drivers/md/dm-vdo/indexer/config.c | 4 +- drivers/md/dm-vdo/indexer/delta-index.c | 20 ++-- .../md/dm-vdo/indexer/funnel-requestqueue.c | 4 +- drivers/md/dm-vdo/indexer/geometry.c | 4 +- drivers/md/dm-vdo/indexer/index-layout.c | 58 ++++++------ drivers/md/dm-vdo/indexer/index-page-map.c | 20 ++-- drivers/md/dm-vdo/indexer/index-session.c | 6 +- drivers/md/dm-vdo/indexer/index.c | 26 +++--- drivers/md/dm-vdo/indexer/io-factory.c | 14 +-- drivers/md/dm-vdo/indexer/open-chapter.c | 8 +- drivers/md/dm-vdo/indexer/radix-sort.c | 4 +- drivers/md/dm-vdo/indexer/sparse-cache.c | 22 ++--- drivers/md/dm-vdo/indexer/volume-index.c | 16 ++-- drivers/md/dm-vdo/indexer/volume.c | 36 ++++---- drivers/md/dm-vdo/int-map.c | 14 +-- drivers/md/dm-vdo/io-submitter.c | 10 +- drivers/md/dm-vdo/logical-zone.c | 8 +- drivers/md/dm-vdo/memory-alloc.c | 38 ++++---- drivers/md/dm-vdo/memory-alloc.h | 52 +++++------ drivers/md/dm-vdo/message-stats.c | 4 +- drivers/md/dm-vdo/packer.c | 14 +-- drivers/md/dm-vdo/physical-zone.c | 16 ++-- drivers/md/dm-vdo/pool-sysfs.c | 2 +- drivers/md/dm-vdo/priority-table.c | 4 +- drivers/md/dm-vdo/recovery-journal.c | 36 ++++---- drivers/md/dm-vdo/repair.c | 24 ++--- drivers/md/dm-vdo/slab-depot.c | 80 ++++++++-------- drivers/md/dm-vdo/slab-depot.h | 2 +- drivers/md/dm-vdo/thread-utils.c | 10 +- drivers/md/dm-vdo/uds-sysfs.c | 4 +- drivers/md/dm-vdo/vdo.c | 92 +++++++++---------- drivers/md/dm-vdo/vio.c | 20 ++-- 44 files changed, 453 insertions(+), 455 deletions(-) diff --git a/drivers/md/dm-vdo/action-manager.c b/drivers/md/dm-vdo/action-manager.c index 973901fc3174..709be4c17d27 100644 --- a/drivers/md/dm-vdo/action-manager.c +++ b/drivers/md/dm-vdo/action-manager.c @@ -107,7 +107,7 @@ int vdo_make_action_manager(zone_count_t zones, struct action_manager **manager_ptr) { struct action_manager *manager; - int result = uds_allocate(1, struct action_manager, __func__, &manager); + int result = vdo_allocate(1, struct action_manager, __func__, &manager); if (result != VDO_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/admin-state.c b/drivers/md/dm-vdo/admin-state.c index 1423f4cebb8a..d695af42d140 100644 --- a/drivers/md/dm-vdo/admin-state.c +++ b/drivers/md/dm-vdo/admin-state.c @@ -206,7 +206,7 @@ bool vdo_finish_operation(struct admin_state *state, int result) if (!state->starting) { vdo_set_admin_state_code(state, state->next_state); if (state->waiter != NULL) - vdo_launch_completion(uds_forget(state->waiter)); + vdo_launch_completion(vdo_forget(state->waiter)); } return true; diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index b7274e94b269..b09974ad41d2 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -221,12 +221,12 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache) u64 size = cache->page_count * (u64) VDO_BLOCK_SIZE; int result; - result = uds_allocate(cache->page_count, struct page_info, "page infos", + result = vdo_allocate(cache->page_count, struct page_info, "page infos", &cache->infos); if (result != UDS_SUCCESS) return result; - result = uds_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages); + result = vdo_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages); if (result != UDS_SUCCESS) return result; @@ -1341,7 +1341,7 @@ int vdo_invalidate_page_cache(struct vdo_page_cache *cache) } /* Reset the page map by re-allocating it. */ - vdo_int_map_free(uds_forget(cache->page_map)); + vdo_int_map_free(vdo_forget(cache->page_map)); return vdo_int_map_create(cache->page_count, &cache->page_map); } @@ -2346,17 +2346,17 @@ static int make_segment(struct forest *old_forest, block_count_t new_pages, forest->segments = index + 1; - result = uds_allocate(forest->segments, struct boundary, + result = vdo_allocate(forest->segments, struct boundary, "forest boundary array", &forest->boundaries); if (result != VDO_SUCCESS) return result; - result = uds_allocate(forest->segments, struct tree_page *, + result = vdo_allocate(forest->segments, struct tree_page *, "forest page pointers", &forest->pages); if (result != VDO_SUCCESS) return result; - result = uds_allocate(new_pages, struct tree_page, + result = vdo_allocate(new_pages, struct tree_page, "new forest pages", &forest->pages[index]); if (result != VDO_SUCCESS) return result; @@ -2382,7 +2382,7 @@ static int make_segment(struct forest *old_forest, block_count_t new_pages, struct block_map_tree *tree = &(forest->trees[root]); height_t height; - int result = uds_allocate(forest->segments, + int result = vdo_allocate(forest->segments, struct block_map_tree_segment, "tree root segments", &tree->segments); if (result != VDO_SUCCESS) @@ -2424,15 +2424,15 @@ static void deforest(struct forest *forest, size_t first_page_segment) size_t segment; for (segment = first_page_segment; segment < forest->segments; segment++) - uds_free(forest->pages[segment]); - uds_free(forest->pages); + vdo_free(forest->pages[segment]); + vdo_free(forest->pages); } for (root = 0; root < forest->map->root_count; root++) - uds_free(forest->trees[root].segments); + vdo_free(forest->trees[root].segments); - uds_free(forest->boundaries); - uds_free(forest); + vdo_free(forest->boundaries); + vdo_free(forest); } /** @@ -2459,7 +2459,7 @@ static int make_forest(struct block_map *map, block_count_t entries) return VDO_SUCCESS; } - result = uds_allocate_extended(struct forest, map->root_count, + result = vdo_allocate_extended(struct forest, map->root_count, struct block_map_tree, __func__, &forest); if (result != VDO_SUCCESS) @@ -2485,7 +2485,7 @@ static void replace_forest(struct block_map *map) if (map->next_forest != NULL) { if (map->forest != NULL) deforest(map->forest, map->forest->segments); - map->forest = uds_forget(map->next_forest); + map->forest = vdo_forget(map->next_forest); } map->entry_count = map->next_entry_count; @@ -2501,11 +2501,11 @@ static void finish_cursor(struct cursor *cursor) struct cursors *cursors = cursor->parent; struct vdo_completion *completion = cursors->completion; - return_vio_to_pool(cursors->pool, uds_forget(cursor->vio)); + return_vio_to_pool(cursors->pool, vdo_forget(cursor->vio)); if (--cursors->active_roots > 0) return; - uds_free(cursors); + vdo_free(cursors); vdo_finish_completion(completion); } @@ -2681,7 +2681,7 @@ void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, struct cursors *cursors; int result; - result = uds_allocate_extended(struct cursors, map->root_count, + result = vdo_allocate_extended(struct cursors, map->root_count, struct cursor, __func__, &cursors); if (result != VDO_SUCCESS) { vdo_fail_completion(completion, result); @@ -2729,7 +2729,7 @@ static int __must_check initialize_block_map_zone(struct block_map *map, zone->thread_id = vdo->thread_config.logical_threads[zone_number]; zone->block_map = map; - result = uds_allocate_extended(struct dirty_lists, maximum_age, + result = vdo_allocate_extended(struct dirty_lists, maximum_age, dirty_era_t, __func__, &zone->dirty_lists); if (result != VDO_SUCCESS) @@ -2822,19 +2822,19 @@ static void uninitialize_block_map_zone(struct block_map_zone *zone) { struct vdo_page_cache *cache = &zone->page_cache; - uds_free(uds_forget(zone->dirty_lists)); - free_vio_pool(uds_forget(zone->vio_pool)); - vdo_int_map_free(uds_forget(zone->loading_pages)); + vdo_free(vdo_forget(zone->dirty_lists)); + free_vio_pool(vdo_forget(zone->vio_pool)); + vdo_int_map_free(vdo_forget(zone->loading_pages)); if (cache->infos != NULL) { struct page_info *info; for (info = cache->infos; info < cache->infos + cache->page_count; info++) - free_vio(uds_forget(info->vio)); + free_vio(vdo_forget(info->vio)); } - vdo_int_map_free(uds_forget(cache->page_map)); - uds_free(uds_forget(cache->infos)); - uds_free(uds_forget(cache->pages)); + vdo_int_map_free(vdo_forget(cache->page_map)); + vdo_free(vdo_forget(cache->infos)); + vdo_free(vdo_forget(cache->pages)); } void vdo_free_block_map(struct block_map *map) @@ -2849,9 +2849,9 @@ void vdo_free_block_map(struct block_map *map) vdo_abandon_block_map_growth(map); if (map->forest != NULL) - deforest(uds_forget(map->forest), 0); - uds_free(uds_forget(map->action_manager)); - uds_free(map); + deforest(vdo_forget(map->forest), 0); + vdo_free(vdo_forget(map->action_manager)); + vdo_free(map); } /* @journal may be NULL. */ @@ -2871,7 +2871,7 @@ int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical if (result != UDS_SUCCESS) return result; - result = uds_allocate_extended(struct block_map, + result = vdo_allocate_extended(struct block_map, vdo->thread_config.logical_zone_count, struct block_map_zone, __func__, &map); if (result != UDS_SUCCESS) @@ -3053,7 +3053,7 @@ void vdo_grow_block_map(struct block_map *map, struct vdo_completion *parent) void vdo_abandon_block_map_growth(struct block_map *map) { - struct forest *forest = uds_forget(map->next_forest); + struct forest *forest = vdo_forget(map->next_forest); if (forest != NULL) deforest(forest, forest->segments - 1); diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 1630993e536f..dcdd767e40e5 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -789,20 +789,20 @@ static int initialize_data_vio(struct data_vio *data_vio, struct vdo *vdo) int result; BUILD_BUG_ON(VDO_BLOCK_SIZE > PAGE_SIZE); - result = uds_allocate_memory(VDO_BLOCK_SIZE, 0, "data_vio data", + result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "data_vio data", &data_vio->vio.data); if (result != VDO_SUCCESS) return uds_log_error_strerror(result, "data_vio data allocation failure"); - result = uds_allocate_memory(VDO_BLOCK_SIZE, 0, "compressed block", + result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "compressed block", &data_vio->compression.block); if (result != VDO_SUCCESS) { return uds_log_error_strerror(result, "data_vio compressed block allocation failure"); } - result = uds_allocate_memory(VDO_BLOCK_SIZE, 0, "vio scratch", + result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "vio scratch", &data_vio->scratch_block); if (result != VDO_SUCCESS) return uds_log_error_strerror(result, @@ -825,10 +825,10 @@ static void destroy_data_vio(struct data_vio *data_vio) if (data_vio == NULL) return; - vdo_free_bio(uds_forget(data_vio->vio.bio)); - uds_free(uds_forget(data_vio->vio.data)); - uds_free(uds_forget(data_vio->compression.block)); - uds_free(uds_forget(data_vio->scratch_block)); + vdo_free_bio(vdo_forget(data_vio->vio.bio)); + vdo_free(vdo_forget(data_vio->vio.data)); + vdo_free(vdo_forget(data_vio->compression.block)); + vdo_free(vdo_forget(data_vio->scratch_block)); } /** @@ -845,7 +845,7 @@ int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, struct data_vio_pool *pool; data_vio_count_t i; - result = uds_allocate_extended(struct data_vio_pool, pool_size, struct data_vio, + result = vdo_allocate_extended(struct data_vio_pool, pool_size, struct data_vio, __func__, &pool); if (result != UDS_SUCCESS) return result; @@ -867,7 +867,7 @@ int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, result = uds_make_funnel_queue(&pool->queue); if (result != UDS_SUCCESS) { - free_data_vio_pool(uds_forget(pool)); + free_data_vio_pool(vdo_forget(pool)); return result; } @@ -924,8 +924,8 @@ void free_data_vio_pool(struct data_vio_pool *pool) destroy_data_vio(data_vio); } - uds_free_funnel_queue(uds_forget(pool->queue)); - uds_free(pool); + uds_free_funnel_queue(vdo_forget(pool->queue)); + vdo_free(pool); } static bool acquire_permit(struct limiter *limiter) @@ -1431,7 +1431,7 @@ void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset) allocation->pbn = VDO_ZERO_BLOCK; vdo_release_physical_zone_pbn_lock(allocation->zone, locked_pbn, - uds_forget(allocation->lock)); + vdo_forget(allocation->lock)); } /** diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index b453a204239a..7cdbe825116f 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -700,7 +700,7 @@ static void unlock_duplicate_pbn(struct vdo_completion *completion) "must have a duplicate lock to release"); vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, agent->duplicate.pbn, - uds_forget(lock->duplicate_lock)); + vdo_forget(lock->duplicate_lock)); if (lock->state == VDO_HASH_LOCK_BYPASSING) { complete_data_vio(completion); return; @@ -896,7 +896,7 @@ static int __must_check acquire_lock(struct hash_zone *zone, result = vdo_int_map_put(zone->hash_lock_map, hash_lock_key(new_lock), new_lock, (replace_lock != NULL), (void **) &lock); if (result != VDO_SUCCESS) { - return_hash_lock_to_pool(zone, uds_forget(new_lock)); + return_hash_lock_to_pool(zone, vdo_forget(new_lock)); return result; } @@ -915,7 +915,7 @@ static int __must_check acquire_lock(struct hash_zone *zone, lock->registered = true; } else { /* There's already a lock for the hash, so we don't need the borrowed lock. */ - return_hash_lock_to_pool(zone, uds_forget(new_lock)); + return_hash_lock_to_pool(zone, vdo_forget(new_lock)); } *lock_ptr = lock; @@ -1980,7 +1980,7 @@ static void transfer_allocation_lock(struct data_vio *data_vio) * Since the lock is being transferred, the holder count doesn't change (and isn't even * safe to examine on this thread). */ - hash_lock->duplicate_lock = uds_forget(allocation->lock); + hash_lock->duplicate_lock = vdo_forget(allocation->lock); } /** @@ -2025,7 +2025,7 @@ void vdo_share_compressed_write_lock(struct data_vio *data_vio, static void dedupe_kobj_release(struct kobject *directory) { - uds_free(container_of(directory, struct hash_zones, dedupe_directory)); + vdo_free(container_of(directory, struct hash_zones, dedupe_directory)); } static ssize_t dedupe_status_show(struct kobject *directory, struct attribute *attr, @@ -2083,12 +2083,12 @@ static void start_uds_queue(void *ptr) */ struct vdo_thread *thread = vdo_get_work_queue_owner(vdo_get_current_work_queue()); - uds_register_allocating_thread(&thread->allocating_thread, NULL); + vdo_register_allocating_thread(&thread->allocating_thread, NULL); } static void finish_uds_queue(void *ptr __always_unused) { - uds_unregister_allocating_thread(); + vdo_unregister_allocating_thread(); } static void close_index(struct hash_zones *zones) @@ -2259,7 +2259,7 @@ static int initialize_index(struct vdo *vdo, struct hash_zones *zones) result = vdo_make_thread(vdo, vdo->thread_config.dedupe_thread, &uds_queue_type, 1, NULL); if (result != VDO_SUCCESS) { - uds_destroy_index_session(uds_forget(zones->index_session)); + uds_destroy_index_session(vdo_forget(zones->index_session)); uds_log_error("UDS index queue initialization failed (%d)", result); return result; } @@ -2417,7 +2417,7 @@ static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zone vdo_set_completion_callback(&zone->completion, timeout_index_operations_callback, zone->thread_id); INIT_LIST_HEAD(&zone->lock_pool); - result = uds_allocate(LOCK_POOL_CAPACITY, struct hash_lock, "hash_lock array", + result = vdo_allocate(LOCK_POOL_CAPACITY, struct hash_lock, "hash_lock array", &zone->lock_array); if (result != VDO_SUCCESS) return result; @@ -2471,14 +2471,14 @@ int vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr) if (zone_count == 0) return VDO_SUCCESS; - result = uds_allocate_extended(struct hash_zones, zone_count, struct hash_zone, + result = vdo_allocate_extended(struct hash_zones, zone_count, struct hash_zone, __func__, &zones); if (result != VDO_SUCCESS) return result; result = initialize_index(vdo, zones); if (result != VDO_SUCCESS) { - uds_free(zones); + vdo_free(zones); return result; } @@ -2510,7 +2510,7 @@ void vdo_finish_dedupe_index(struct hash_zones *zones) if (zones == NULL) return; - uds_destroy_index_session(uds_forget(zones->index_session)); + uds_destroy_index_session(vdo_forget(zones->index_session)); } /** @@ -2524,14 +2524,14 @@ void vdo_free_hash_zones(struct hash_zones *zones) if (zones == NULL) return; - uds_free(uds_forget(zones->manager)); + vdo_free(vdo_forget(zones->manager)); for (i = 0; i < zones->zone_count; i++) { struct hash_zone *zone = &zones->zones[i]; - uds_free_funnel_queue(uds_forget(zone->timed_out_complete)); - vdo_int_map_free(uds_forget(zone->hash_lock_map)); - uds_free(uds_forget(zone->lock_array)); + uds_free_funnel_queue(vdo_forget(zone->timed_out_complete)); + vdo_int_map_free(vdo_forget(zone->hash_lock_map)); + vdo_free(vdo_forget(zone->lock_array)); } if (zones->index_session != NULL) @@ -2539,7 +2539,7 @@ void vdo_free_hash_zones(struct hash_zones *zones) ratelimit_state_exit(&zones->ratelimiter); if (vdo_get_admin_state_code(&zones->state) == VDO_ADMIN_STATE_NEW) - uds_free(zones); + vdo_free(zones); else kobject_put(&zones->dedupe_directory); } diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 175ee56a89e1..86c30fbd75ca 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -189,12 +189,12 @@ static void free_device_config(struct device_config *config) if (config->owned_device != NULL) dm_put_device(config->owning_target, config->owned_device); - uds_free(config->parent_device_name); - uds_free(config->original_string); + vdo_free(config->parent_device_name); + vdo_free(config->original_string); /* Reduce the chance a use-after-free (as in BZ 1669960) happens to work. */ memset(config, 0, sizeof(*config)); - uds_free(config); + vdo_free(config); } /** @@ -249,15 +249,15 @@ static void free_string_array(char **string_array) unsigned int offset; for (offset = 0; string_array[offset] != NULL; offset++) - uds_free(string_array[offset]); - uds_free(string_array); + vdo_free(string_array[offset]); + vdo_free(string_array); } /* * Split the input string into substrings, separated at occurrences of the indicated character, * returning a null-terminated list of string pointers. * - * The string pointers and the pointer array itself should both be freed with uds_free() when no + * The string pointers and the pointer array itself should both be freed with vdo_free() when no * longer needed. This can be done with vdo_free_string_array (below) if the pointers in the array * are not changed. Since the array and copied strings are allocated by this function, it may only * be used in contexts where allocation is permitted. @@ -278,7 +278,7 @@ static int split_string(const char *string, char separator, char ***substring_ar substring_count++; } - result = uds_allocate(substring_count + 1, char *, "string-splitting array", + result = vdo_allocate(substring_count + 1, char *, "string-splitting array", &substrings); if (result != UDS_SUCCESS) return result; @@ -287,7 +287,7 @@ static int split_string(const char *string, char separator, char ***substring_ar if (*s == separator) { ptrdiff_t length = s - string; - result = uds_allocate(length + 1, char, "split string", + result = vdo_allocate(length + 1, char, "split string", &substrings[current_substring]); if (result != UDS_SUCCESS) { free_string_array(substrings); @@ -308,7 +308,7 @@ static int split_string(const char *string, char separator, char ***substring_ar BUG_ON(current_substring != (substring_count - 1)); length = strlen(string); - result = uds_allocate(length + 1, char, "split string", + result = vdo_allocate(length + 1, char, "split string", &substrings[current_substring]); if (result != UDS_SUCCESS) { free_string_array(substrings); @@ -337,7 +337,7 @@ static int join_strings(char **substring_array, size_t array_length, char separa for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++) string_length += strlen(substring_array[i]) + 1; - result = uds_allocate(string_length, char, __func__, &output); + result = vdo_allocate(string_length, char, __func__, &output); if (result != VDO_SUCCESS) return result; @@ -731,7 +731,7 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti, return VDO_BAD_CONFIGURATION; } - result = uds_allocate(1, struct device_config, "device_config", &config); + result = vdo_allocate(1, struct device_config, "device_config", &config); if (result != VDO_SUCCESS) { handle_parse_error(config, error_ptr, "Could not allocate config structure"); @@ -777,7 +777,7 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti, if (config->version >= 1) dm_shift_arg(&arg_set); - result = uds_duplicate_string(dm_shift_arg(&arg_set), "parent device name", + result = vdo_duplicate_string(dm_shift_arg(&arg_set), "parent device name", &config->parent_device_name); if (result != VDO_SUCCESS) { handle_parse_error(config, error_ptr, @@ -1100,7 +1100,7 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, } vdo = get_vdo_for_target(ti); - uds_register_allocating_thread(&allocating_thread, NULL); + vdo_register_allocating_thread(&allocating_thread, NULL); vdo_register_thread_device_id(&instance_thread, &vdo->instance); /* @@ -1115,7 +1115,7 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, } vdo_unregister_thread_device_id(); - uds_unregister_allocating_thread(); + vdo_unregister_allocating_thread(); return result; } @@ -1536,7 +1536,7 @@ static int grow_bit_array(void) unsigned long *new_words; int result; - result = uds_reallocate_memory(instances.words, + result = vdo_reallocate_memory(instances.words, get_bit_array_size(instances.bit_count), get_bit_array_size(new_count), "instance number bit array", &new_words); @@ -1702,7 +1702,7 @@ static int grow_layout(struct vdo *vdo, block_count_t old_size, block_count_t ne VDO_SLAB_SUMMARY_PARTITION), &vdo->next_layout); if (result != VDO_SUCCESS) { - dm_kcopyd_client_destroy(uds_forget(vdo->partition_copier)); + dm_kcopyd_client_destroy(vdo_forget(vdo->partition_copier)); return result; } @@ -1715,7 +1715,7 @@ static int grow_layout(struct vdo *vdo, block_count_t old_size, block_count_t ne if (min_new_size > new_size) { /* Copying the journal and summary would destroy some old metadata. */ vdo_uninitialize_layout(&vdo->next_layout); - dm_kcopyd_client_destroy(uds_forget(vdo->partition_copier)); + dm_kcopyd_client_destroy(vdo_forget(vdo->partition_copier)); return VDO_INCREMENT_TOO_SMALL; } @@ -1901,7 +1901,7 @@ static int vdo_ctr(struct dm_target *ti, unsigned int argc, char **argv) const char *device_name; struct vdo *vdo; - uds_register_allocating_thread(&allocating_thread, NULL); + vdo_register_allocating_thread(&allocating_thread, NULL); device_name = vdo_get_device_name(ti); vdo = vdo_find_matching(vdo_is_named, device_name); if (vdo == NULL) { @@ -1912,14 +1912,14 @@ static int vdo_ctr(struct dm_target *ti, unsigned int argc, char **argv) vdo_unregister_thread_device_id(); } - uds_unregister_allocating_thread(); + vdo_unregister_allocating_thread(); return result; } static void vdo_dtr(struct dm_target *ti) { struct device_config *config = ti->private; - struct vdo *vdo = uds_forget(config->vdo); + struct vdo *vdo = vdo_forget(config->vdo); list_del_init(&config->config_list); if (list_empty(&vdo->device_config_list)) { @@ -1930,17 +1930,17 @@ static void vdo_dtr(struct dm_target *ti) struct registered_thread allocating_thread, instance_thread; vdo_register_thread_device_id(&instance_thread, &instance); - uds_register_allocating_thread(&allocating_thread, NULL); + vdo_register_allocating_thread(&allocating_thread, NULL); device_name = vdo_get_device_name(ti); uds_log_info("stopping device '%s'", device_name); if (vdo->dump_on_shutdown) vdo_dump_all(vdo, "device shutdown"); - vdo_destroy(uds_forget(vdo)); + vdo_destroy(vdo_forget(vdo)); uds_log_info("device '%s' stopped", device_name); vdo_unregister_thread_device_id(); - uds_unregister_allocating_thread(); + vdo_unregister_allocating_thread(); release_instance(instance); } else if (config == vdo->device_config) { /* @@ -2323,7 +2323,7 @@ static void handle_load_error(struct vdo_completion *completion) (vdo->admin.phase == LOAD_PHASE_MAKE_DIRTY)) { uds_log_error_strerror(completion->result, "aborting load"); vdo->admin.phase = LOAD_PHASE_DRAIN_JOURNAL; - load_callback(uds_forget(completion)); + load_callback(vdo_forget(completion)); return; } @@ -2633,7 +2633,7 @@ static void grow_physical_callback(struct vdo_completion *completion) case GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS: vdo_uninitialize_layout(&vdo->layout); vdo->layout = vdo->next_layout; - uds_forget(vdo->next_layout.head); + vdo_forget(vdo->next_layout.head); vdo->states.vdo.config.physical_blocks = vdo->layout.size; vdo_update_slab_depot_size(vdo->depot); vdo_save_components(vdo, completion); @@ -2893,7 +2893,7 @@ static void vdo_module_destroy(void) ASSERT_LOG_ONLY(instances.count == 0, "should have no instance numbers still in use, but have %u", instances.count); - uds_free(instances.words); + vdo_free(instances.words); memset(&instances, 0, sizeof(struct instance_tracker)); uds_log_info("unloaded version %s", CURRENT_VERSION); @@ -2904,7 +2904,7 @@ static int __init vdo_init(void) int result = 0; /* Memory tracking must be initialized first for accurate accounting. */ - uds_memory_init(); + vdo_memory_init(); uds_init_sysfs(); vdo_initialize_thread_device_registry(); @@ -2935,7 +2935,7 @@ static void __exit vdo_exit(void) vdo_module_destroy(); uds_put_sysfs(); /* Memory tracking cleanup must be done last. */ - uds_memory_exit(); + vdo_memory_exit(); } module_init(vdo_init); diff --git a/drivers/md/dm-vdo/dump.c b/drivers/md/dm-vdo/dump.c index 2a0890b54186..52ee9a72781c 100644 --- a/drivers/md/dm-vdo/dump.c +++ b/drivers/md/dm-vdo/dump.c @@ -79,7 +79,7 @@ static void do_dump(struct vdo *vdo, unsigned int dump_options_requested, if ((dump_options_requested & FLAG_SHOW_VDO_STATUS) != 0) vdo_dump_status(vdo); - uds_report_memory_usage(); + vdo_report_memory_usage(); uds_log_info("end of %s dump", UDS_LOGGING_MODULE_NAME); } diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index 4a0a6afea670..56d94339d2af 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -799,7 +799,7 @@ static int allocate_partition(struct layout *layout, u8 id, struct partition *partition; int result; - result = uds_allocate(1, struct partition, __func__, &partition); + result = vdo_allocate(1, struct partition, __func__, &partition); if (result != UDS_SUCCESS) return result; @@ -928,7 +928,7 @@ void vdo_uninitialize_layout(struct layout *layout) struct partition *part = layout->head; layout->head = part->next; - uds_free(part); + vdo_free(part); } memset(layout, 0, sizeof(struct layout)); diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index 1bc13470a608..8d8d9cf4a24c 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -103,9 +103,9 @@ static void *allocate_flush(gfp_t gfp_mask, void *pool_data) struct vdo_flush *flush = NULL; if ((gfp_mask & GFP_NOWAIT) == GFP_NOWAIT) { - flush = uds_allocate_memory_nowait(sizeof(struct vdo_flush), __func__); + flush = vdo_allocate_memory_nowait(sizeof(struct vdo_flush), __func__); } else { - int result = uds_allocate(1, struct vdo_flush, __func__, &flush); + int result = vdo_allocate(1, struct vdo_flush, __func__, &flush); if (result != VDO_SUCCESS) uds_log_error_strerror(result, "failed to allocate spare flush"); @@ -123,7 +123,7 @@ static void *allocate_flush(gfp_t gfp_mask, void *pool_data) static void free_flush(void *element, void *pool_data __always_unused) { - uds_free(element); + vdo_free(element); } /** @@ -134,7 +134,7 @@ static void free_flush(void *element, void *pool_data __always_unused) */ int vdo_make_flusher(struct vdo *vdo) { - int result = uds_allocate(1, struct flusher, __func__, &vdo->flusher); + int result = vdo_allocate(1, struct flusher, __func__, &vdo->flusher); if (result != VDO_SUCCESS) return result; @@ -162,8 +162,8 @@ void vdo_free_flusher(struct flusher *flusher) return; if (flusher->flush_pool != NULL) - mempool_destroy(uds_forget(flusher->flush_pool)); - uds_free(flusher); + mempool_destroy(vdo_forget(flusher->flush_pool)); + vdo_free(flusher); } /** diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c index d5d96bb38b94..67f7b52ecc86 100644 --- a/drivers/md/dm-vdo/funnel-queue.c +++ b/drivers/md/dm-vdo/funnel-queue.c @@ -14,7 +14,7 @@ int uds_make_funnel_queue(struct funnel_queue **queue_ptr) int result; struct funnel_queue *queue; - result = uds_allocate(1, struct funnel_queue, "funnel queue", &queue); + result = vdo_allocate(1, struct funnel_queue, "funnel queue", &queue); if (result != UDS_SUCCESS) return result; @@ -32,7 +32,7 @@ int uds_make_funnel_queue(struct funnel_queue **queue_ptr) void uds_free_funnel_queue(struct funnel_queue *queue) { - uds_free(queue); + vdo_free(queue); } static struct funnel_queue_entry *get_oldest(struct funnel_queue *queue) diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c index 8f0ada13e549..ebf8dce67086 100644 --- a/drivers/md/dm-vdo/funnel-workqueue.c +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -276,8 +276,8 @@ static void free_simple_work_queue(struct simple_work_queue *queue) for (i = 0; i <= VDO_WORK_Q_MAX_PRIORITY; i++) uds_free_funnel_queue(queue->priority_lists[i]); - uds_free(queue->common.name); - uds_free(queue); + vdo_free(queue->common.name); + vdo_free(queue); } static void free_round_robin_work_queue(struct round_robin_work_queue *queue) @@ -290,9 +290,9 @@ static void free_round_robin_work_queue(struct round_robin_work_queue *queue) for (i = 0; i < count; i++) free_simple_work_queue(queue_table[i]); - uds_free(queue_table); - uds_free(queue->common.name); - uds_free(queue); + vdo_free(queue_table); + vdo_free(queue->common.name); + vdo_free(queue); } void vdo_free_work_queue(struct vdo_work_queue *queue) @@ -323,7 +323,7 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na "queue priority count %u within limit %u", type->max_priority, VDO_WORK_Q_MAX_PRIORITY); - result = uds_allocate(1, struct simple_work_queue, "simple work queue", &queue); + result = vdo_allocate(1, struct simple_work_queue, "simple work queue", &queue); if (result != UDS_SUCCESS) return result; @@ -333,9 +333,9 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na queue->common.owner = owner; init_waitqueue_head(&queue->waiting_worker_threads); - result = uds_duplicate_string(name, "queue name", &queue->common.name); + result = vdo_duplicate_string(name, "queue name", &queue->common.name); if (result != VDO_SUCCESS) { - uds_free(queue); + vdo_free(queue); return -ENOMEM; } @@ -399,15 +399,15 @@ int vdo_make_work_queue(const char *thread_name_prefix, const char *name, return result; } - result = uds_allocate(1, struct round_robin_work_queue, "round-robin work queue", + result = vdo_allocate(1, struct round_robin_work_queue, "round-robin work queue", &queue); if (result != UDS_SUCCESS) return result; - result = uds_allocate(thread_count, struct simple_work_queue *, + result = vdo_allocate(thread_count, struct simple_work_queue *, "subordinate work queues", &queue->service_queues); if (result != UDS_SUCCESS) { - uds_free(queue); + vdo_free(queue); return result; } @@ -415,10 +415,10 @@ int vdo_make_work_queue(const char *thread_name_prefix, const char *name, queue->common.round_robin_mode = true; queue->common.owner = owner; - result = uds_duplicate_string(name, "queue name", &queue->common.name); + result = vdo_duplicate_string(name, "queue name", &queue->common.name); if (result != VDO_SUCCESS) { - uds_free(queue->service_queues); - uds_free(queue); + vdo_free(queue->service_queues); + vdo_free(queue); return -ENOMEM; } @@ -433,7 +433,7 @@ int vdo_make_work_queue(const char *thread_name_prefix, const char *name, if (result != VDO_SUCCESS) { queue->num_service_queues = i; /* Destroy previously created subordinates. */ - vdo_free_work_queue(uds_forget(*queue_ptr)); + vdo_free_work_queue(vdo_forget(*queue_ptr)); return result; } } diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c index 6487825ada90..9477150362ae 100644 --- a/drivers/md/dm-vdo/indexer/chapter-index.c +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -20,7 +20,7 @@ int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, size_t memory_size; struct open_chapter_index *index; - result = uds_allocate(1, struct open_chapter_index, "open chapter index", &index); + result = vdo_allocate(1, struct open_chapter_index, "open chapter index", &index); if (result != UDS_SUCCESS) return result; @@ -37,7 +37,7 @@ int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, geometry->chapter_payload_bits, memory_size, 'm'); if (result != UDS_SUCCESS) { - uds_free(index); + vdo_free(index); return result; } @@ -52,7 +52,7 @@ void uds_free_open_chapter_index(struct open_chapter_index *chapter_index) return; uds_uninitialize_delta_index(&chapter_index->delta_index); - uds_free(chapter_index); + vdo_free(chapter_index); } /* Re-initialize an open chapter index for a new chapter. */ diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c index b572350a3d5f..cd20ee8b9a02 100644 --- a/drivers/md/dm-vdo/indexer/config.c +++ b/drivers/md/dm-vdo/indexer/config.c @@ -325,7 +325,7 @@ int uds_make_configuration(const struct uds_parameters *params, if (result != UDS_SUCCESS) return result; - result = uds_allocate(1, struct uds_configuration, __func__, &config); + result = vdo_allocate(1, struct uds_configuration, __func__, &config); if (result != UDS_SUCCESS) return result; @@ -356,7 +356,7 @@ void uds_free_configuration(struct uds_configuration *config) { if (config != NULL) { uds_free_index_geometry(config->geometry); - uds_free(config); + vdo_free(config); } } diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c index 8eece0ba6d93..11f7b85b6710 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -296,12 +296,12 @@ void uds_uninitialize_delta_index(struct delta_index *delta_index) return; for (z = 0; z < delta_index->zone_count; z++) { - uds_free(uds_forget(delta_index->delta_zones[z].new_offsets)); - uds_free(uds_forget(delta_index->delta_zones[z].delta_lists)); - uds_free(uds_forget(delta_index->delta_zones[z].memory)); + vdo_free(vdo_forget(delta_index->delta_zones[z].new_offsets)); + vdo_free(vdo_forget(delta_index->delta_zones[z].delta_lists)); + vdo_free(vdo_forget(delta_index->delta_zones[z].memory)); } - uds_free(delta_index->delta_zones); + vdo_free(delta_index->delta_zones); memset(delta_index, 0, sizeof(struct delta_index)); } @@ -311,17 +311,17 @@ static int initialize_delta_zone(struct delta_zone *delta_zone, size_t size, { int result; - result = uds_allocate(size, u8, "delta list", &delta_zone->memory); + result = vdo_allocate(size, u8, "delta list", &delta_zone->memory); if (result != UDS_SUCCESS) return result; - result = uds_allocate(list_count + 2, u64, "delta list temp", + result = vdo_allocate(list_count + 2, u64, "delta list temp", &delta_zone->new_offsets); if (result != UDS_SUCCESS) return result; /* Allocate the delta lists. */ - result = uds_allocate(list_count + 2, struct delta_list, "delta lists", + result = vdo_allocate(list_count + 2, struct delta_list, "delta lists", &delta_zone->delta_lists); if (result != UDS_SUCCESS) return result; @@ -352,7 +352,7 @@ int uds_initialize_delta_index(struct delta_index *delta_index, unsigned int zon unsigned int z; size_t zone_memory; - result = uds_allocate(zone_count, struct delta_zone, "Delta Index Zones", + result = vdo_allocate(zone_count, struct delta_zone, "Delta Index Zones", &delta_index->delta_zones); if (result != UDS_SUCCESS) return result; @@ -1047,7 +1047,7 @@ int uds_finish_restoring_delta_index(struct delta_index *delta_index, unsigned int z; u8 *data; - result = uds_allocate(DELTA_LIST_MAX_BYTE_COUNT, u8, __func__, &data); + result = vdo_allocate(DELTA_LIST_MAX_BYTE_COUNT, u8, __func__, &data); if (result != UDS_SUCCESS) return result; @@ -1062,7 +1062,7 @@ int uds_finish_restoring_delta_index(struct delta_index *delta_index, } } - uds_free(data); + vdo_free(data); return saved_result; } diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c index d2b49e39550c..95a402ec31c9 100644 --- a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c +++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c @@ -198,7 +198,7 @@ int uds_make_request_queue(const char *queue_name, int result; struct uds_request_queue *queue; - result = uds_allocate(1, struct uds_request_queue, __func__, &queue); + result = vdo_allocate(1, struct uds_request_queue, __func__, &queue); if (result != UDS_SUCCESS) return result; @@ -275,5 +275,5 @@ void uds_request_queue_finish(struct uds_request_queue *queue) uds_free_funnel_queue(queue->main_queue); uds_free_funnel_queue(queue->retry_queue); - uds_free(queue); + vdo_free(queue); } diff --git a/drivers/md/dm-vdo/indexer/geometry.c b/drivers/md/dm-vdo/indexer/geometry.c index 38c18283cdde..c735e6cb4425 100644 --- a/drivers/md/dm-vdo/indexer/geometry.c +++ b/drivers/md/dm-vdo/indexer/geometry.c @@ -61,7 +61,7 @@ int uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, int result; struct index_geometry *geometry; - result = uds_allocate(1, struct index_geometry, "geometry", &geometry); + result = vdo_allocate(1, struct index_geometry, "geometry", &geometry); if (result != UDS_SUCCESS) return result; @@ -121,7 +121,7 @@ int uds_copy_index_geometry(struct index_geometry *source, void uds_free_index_geometry(struct index_geometry *geometry) { - uds_free(geometry); + vdo_free(geometry); } u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry, diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index a0227a75814b..c1bcff03cc55 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -484,7 +484,7 @@ static int __must_check make_index_save_region_table(struct index_save_layout *i type = RH_TYPE_UNSAVED; } - result = uds_allocate_extended(struct region_table, region_count, + result = vdo_allocate_extended(struct region_table, region_count, struct layout_region, "layout region table for ISL", &table); if (result != UDS_SUCCESS) @@ -545,7 +545,7 @@ static int __must_check write_index_save_header(struct index_save_layout *isl, u8 *buffer; size_t offset = 0; - result = uds_allocate(table->encoded_size, u8, "index save data", &buffer); + result = vdo_allocate(table->encoded_size, u8, "index save data", &buffer); if (result != UDS_SUCCESS) return result; @@ -564,7 +564,7 @@ static int __must_check write_index_save_header(struct index_save_layout *isl, } result = uds_write_to_buffered_writer(writer, buffer, offset); - uds_free(buffer); + vdo_free(buffer); if (result != UDS_SUCCESS) return result; @@ -584,12 +584,12 @@ static int write_index_save_layout(struct index_layout *layout, result = open_region_writer(layout, &isl->header, &writer); if (result != UDS_SUCCESS) { - uds_free(table); + vdo_free(table); return result; } result = write_index_save_header(isl, table, writer); - uds_free(table); + vdo_free(table); uds_free_buffered_writer(writer); return result; @@ -667,7 +667,7 @@ static int __must_check make_layout_region_table(struct index_layout *layout, struct region_table *table; struct layout_region *lr; - result = uds_allocate_extended(struct region_table, region_count, + result = vdo_allocate_extended(struct region_table, region_count, struct layout_region, "layout region table", &table); if (result != UDS_SUCCESS) @@ -715,7 +715,7 @@ static int __must_check write_layout_header(struct index_layout *layout, u8 *buffer; size_t offset = 0; - result = uds_allocate(table->encoded_size, u8, "layout data", &buffer); + result = vdo_allocate(table->encoded_size, u8, "layout data", &buffer); if (result != UDS_SUCCESS) return result; @@ -739,7 +739,7 @@ static int __must_check write_layout_header(struct index_layout *layout, } result = uds_write_to_buffered_writer(writer, buffer, offset); - uds_free(buffer); + vdo_free(buffer); if (result != UDS_SUCCESS) return result; @@ -785,12 +785,12 @@ static int __must_check save_layout(struct index_layout *layout, off_t offset) result = open_layout_writer(layout, &layout->header, offset, &writer); if (result != UDS_SUCCESS) { - uds_free(table); + vdo_free(table); return result; } result = write_layout_header(layout, table, writer); - uds_free(table); + vdo_free(table); uds_free_buffered_writer(writer); return result; @@ -805,7 +805,7 @@ static int create_index_layout(struct index_layout *layout, struct uds_configura if (result != UDS_SUCCESS) return result; - result = uds_allocate(sizes.save_count, struct index_save_layout, __func__, + result = vdo_allocate(sizes.save_count, struct index_save_layout, __func__, &layout->index.saves); if (result != UDS_SUCCESS) return result; @@ -1162,7 +1162,7 @@ static int __must_check load_region_table(struct buffered_reader *reader, header.version); } - result = uds_allocate_extended(struct region_table, header.region_count, + result = vdo_allocate_extended(struct region_table, header.region_count, struct layout_region, "single file layout region table", &table); if (result != UDS_SUCCESS) @@ -1176,7 +1176,7 @@ static int __must_check load_region_table(struct buffered_reader *reader, result = uds_read_from_buffered_reader(reader, region_buffer, sizeof(region_buffer)); if (result != UDS_SUCCESS) { - uds_free(table); + vdo_free(table); return uds_log_error_strerror(UDS_CORRUPT_DATA, "cannot read region table layouts"); } @@ -1201,13 +1201,13 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, u8 *buffer; size_t offset = 0; - result = uds_allocate(saved_size, u8, "super block data", &buffer); + result = vdo_allocate(saved_size, u8, "super block data", &buffer); if (result != UDS_SUCCESS) return result; result = uds_read_from_buffered_reader(reader, buffer, saved_size); if (result != UDS_SUCCESS) { - uds_free(buffer); + vdo_free(buffer); return uds_log_error_strerror(result, "cannot read region table header"); } @@ -1232,7 +1232,7 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, super->start_offset = 0; } - uds_free(buffer); + vdo_free(buffer); if (memcmp(super->magic_label, LAYOUT_MAGIC, MAGIC_SIZE) != 0) return uds_log_error_strerror(UDS_CORRUPT_DATA, @@ -1335,7 +1335,7 @@ static int __must_check reconstitute_layout(struct index_layout *layout, int result; u64 next_block = first_block; - result = uds_allocate(layout->super.max_saves, struct index_save_layout, + result = vdo_allocate(layout->super.max_saves, struct index_save_layout, __func__, &layout->index.saves); if (result != UDS_SUCCESS) return result; @@ -1386,19 +1386,19 @@ static int __must_check load_super_block(struct index_layout *layout, size_t blo return result; if (table->header.type != RH_TYPE_SUPER) { - uds_free(table); + vdo_free(table); return uds_log_error_strerror(UDS_CORRUPT_DATA, "not a superblock region table"); } result = read_super_block_data(reader, layout, table->header.payload); if (result != UDS_SUCCESS) { - uds_free(table); + vdo_free(table); return uds_log_error_strerror(result, "unknown superblock format"); } if (super->block_size != block_size) { - uds_free(table); + vdo_free(table); return uds_log_error_strerror(UDS_CORRUPT_DATA, "superblock saved block_size %u differs from supplied block_size %zu", super->block_size, block_size); @@ -1406,7 +1406,7 @@ static int __must_check load_super_block(struct index_layout *layout, size_t blo first_block -= (super->volume_offset - super->start_offset); result = reconstitute_layout(layout, table, first_block); - uds_free(table); + vdo_free(table); return result; } @@ -1545,7 +1545,7 @@ static int __must_check load_index_save(struct index_save_layout *isl, if (table->header.region_blocks != isl->index_save.block_count) { u64 region_blocks = table->header.region_blocks; - uds_free(table); + vdo_free(table); return uds_log_error_strerror(UDS_CORRUPT_DATA, "unexpected index save %u region block count %llu", instance, @@ -1553,14 +1553,14 @@ static int __must_check load_index_save(struct index_save_layout *isl, } if (table->header.type == RH_TYPE_UNSAVED) { - uds_free(table); + vdo_free(table); reset_index_save_layout(isl, 0); return UDS_SUCCESS; } if (table->header.type != RH_TYPE_SAVE) { - uds_free(table); + vdo_free(table); return uds_log_error_strerror(UDS_CORRUPT_DATA, "unexpected index save %u header type %u", instance, table->header.type); @@ -1568,14 +1568,14 @@ static int __must_check load_index_save(struct index_save_layout *isl, result = read_index_save_data(reader, isl, table->header.payload); if (result != UDS_SUCCESS) { - uds_free(table); + vdo_free(table); return uds_log_error_strerror(result, "unknown index save %u data format", instance); } result = reconstruct_index_save(isl, table); - uds_free(table); + vdo_free(table); if (result != UDS_SUCCESS) { return uds_log_error_strerror(result, "cannot reconstruct index save %u", instance); @@ -1695,7 +1695,7 @@ int uds_make_index_layout(struct uds_configuration *config, bool new_layout, if (result != UDS_SUCCESS) return result; - result = uds_allocate(1, struct index_layout, __func__, &layout); + result = vdo_allocate(1, struct index_layout, __func__, &layout); if (result != UDS_SUCCESS) return result; @@ -1731,11 +1731,11 @@ void uds_free_index_layout(struct index_layout *layout) if (layout == NULL) return; - uds_free(layout->index.saves); + vdo_free(layout->index.saves); if (layout->factory != NULL) uds_put_io_factory(layout->factory); - uds_free(layout); + vdo_free(layout); } int uds_replace_index_layout_storage(struct index_layout *layout, diff --git a/drivers/md/dm-vdo/indexer/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c index 37037ac8eee9..ddb6d843cbd9 100644 --- a/drivers/md/dm-vdo/indexer/index-page-map.c +++ b/drivers/md/dm-vdo/indexer/index-page-map.c @@ -38,13 +38,13 @@ int uds_make_index_page_map(const struct index_geometry *geometry, int result; struct index_page_map *map; - result = uds_allocate(1, struct index_page_map, "page map", &map); + result = vdo_allocate(1, struct index_page_map, "page map", &map); if (result != UDS_SUCCESS) return result; map->geometry = geometry; map->entries_per_chapter = geometry->index_pages_per_chapter - 1; - result = uds_allocate(get_entry_count(geometry), u16, "Index Page Map Entries", + result = vdo_allocate(get_entry_count(geometry), u16, "Index Page Map Entries", &map->entries); if (result != UDS_SUCCESS) { uds_free_index_page_map(map); @@ -58,8 +58,8 @@ int uds_make_index_page_map(const struct index_geometry *geometry, void uds_free_index_page_map(struct index_page_map *map) { if (map != NULL) { - uds_free(map->entries); - uds_free(map); + vdo_free(map->entries); + vdo_free(map); } } @@ -118,7 +118,7 @@ int uds_write_index_page_map(struct index_page_map *map, struct buffered_writer u64 saved_size = uds_compute_index_page_map_save_size(map->geometry); u32 i; - result = uds_allocate(saved_size, u8, "page map data", &buffer); + result = vdo_allocate(saved_size, u8, "page map data", &buffer); if (result != UDS_SUCCESS) return result; @@ -129,7 +129,7 @@ int uds_write_index_page_map(struct index_page_map *map, struct buffered_writer encode_u16_le(buffer, &offset, map->entries[i]); result = uds_write_to_buffered_writer(writer, buffer, offset); - uds_free(buffer); + vdo_free(buffer); if (result != UDS_SUCCESS) return result; @@ -145,20 +145,20 @@ int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader * u64 saved_size = uds_compute_index_page_map_save_size(map->geometry); u32 i; - result = uds_allocate(saved_size, u8, "page map data", &buffer); + result = vdo_allocate(saved_size, u8, "page map data", &buffer); if (result != UDS_SUCCESS) return result; result = uds_read_from_buffered_reader(reader, buffer, saved_size); if (result != UDS_SUCCESS) { - uds_free(buffer); + vdo_free(buffer); return result; } memcpy(&magic, buffer, PAGE_MAP_MAGIC_LENGTH); offset += PAGE_MAP_MAGIC_LENGTH; if (memcmp(magic, PAGE_MAP_MAGIC, PAGE_MAP_MAGIC_LENGTH) != 0) { - uds_free(buffer); + vdo_free(buffer); return UDS_CORRUPT_DATA; } @@ -166,7 +166,7 @@ int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader * for (i = 0; i < get_entry_count(map->geometry); i++) decode_u16_le(buffer, &offset, &map->entries[i]); - uds_free(buffer); + vdo_free(buffer); uds_log_debug("read index page map, last update %llu", (unsigned long long) map->last_update); return UDS_SUCCESS; diff --git a/drivers/md/dm-vdo/indexer/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c index 07b478f57c68..0f920a583021 100644 --- a/drivers/md/dm-vdo/indexer/index-session.c +++ b/drivers/md/dm-vdo/indexer/index-session.c @@ -221,7 +221,7 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde int result; struct uds_index_session *session; - result = uds_allocate(1, struct uds_index_session, __func__, &session); + result = vdo_allocate(1, struct uds_index_session, __func__, &session); if (result != UDS_SUCCESS) return result; @@ -233,7 +233,7 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde result = uds_make_request_queue("callbackW", &handle_callbacks, &session->callback_queue); if (result != UDS_SUCCESS) { - uds_free(session); + vdo_free(session); return result; } @@ -673,7 +673,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) uds_request_queue_finish(index_session->callback_queue); index_session->callback_queue = NULL; uds_log_debug("Destroyed index session"); - uds_free(index_session); + vdo_free(index_session); return uds_status_to_errno(result); } diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c index 35e3b45cdb71..c576033b8a53 100644 --- a/drivers/md/dm-vdo/indexer/index.c +++ b/drivers/md/dm-vdo/indexer/index.c @@ -88,7 +88,7 @@ static int launch_zone_message(struct uds_zone_message message, unsigned int zon int result; struct uds_request *request; - result = uds_allocate(1, struct uds_request, __func__, &request); + result = vdo_allocate(1, struct uds_request, __func__, &request); if (result != UDS_SUCCESS) return result; @@ -623,7 +623,7 @@ static void execute_zone_request(struct uds_request *request) } /* Once the message is processed it can be freed. */ - uds_free(uds_forget(request)); + vdo_free(vdo_forget(request)); return; } @@ -755,8 +755,8 @@ static void free_chapter_writer(struct chapter_writer *writer) stop_chapter_writer(writer); uds_free_open_chapter_index(writer->open_chapter_index); - uds_free(writer->collated_records); - uds_free(writer); + vdo_free(writer->collated_records); + vdo_free(writer); } static int make_chapter_writer(struct uds_index *index, @@ -767,7 +767,7 @@ static int make_chapter_writer(struct uds_index *index, size_t collated_records_size = (sizeof(struct uds_volume_record) * index->volume->geometry->records_per_chapter); - result = uds_allocate_extended(struct chapter_writer, index->zone_count, + result = vdo_allocate_extended(struct chapter_writer, index->zone_count, struct open_chapter_zone *, "Chapter Writer", &writer); if (result != UDS_SUCCESS) @@ -777,7 +777,7 @@ static int make_chapter_writer(struct uds_index *index, mutex_init(&writer->mutex); uds_init_cond(&writer->cond); - result = uds_allocate_cache_aligned(collated_records_size, "collated records", + result = vdo_allocate_cache_aligned(collated_records_size, "collated records", &writer->collated_records); if (result != UDS_SUCCESS) { free_chapter_writer(writer); @@ -1118,7 +1118,7 @@ static void free_index_zone(struct index_zone *zone) uds_free_open_chapter(zone->open_chapter); uds_free_open_chapter(zone->writing_chapter); - uds_free(zone); + vdo_free(zone); } static int make_index_zone(struct uds_index *index, unsigned int zone_number) @@ -1126,7 +1126,7 @@ static int make_index_zone(struct uds_index *index, unsigned int zone_number) int result; struct index_zone *zone; - result = uds_allocate(1, struct index_zone, "index zone", &zone); + result = vdo_allocate(1, struct index_zone, "index zone", &zone); if (result != UDS_SUCCESS) return result; @@ -1163,7 +1163,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op u64 nonce; unsigned int z; - result = uds_allocate_extended(struct uds_index, config->zone_count, + result = vdo_allocate_extended(struct uds_index, config->zone_count, struct uds_request_queue *, "index", &index); if (result != UDS_SUCCESS) return result; @@ -1176,7 +1176,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op return result; } - result = uds_allocate(index->zone_count, struct index_zone *, "zones", + result = vdo_allocate(index->zone_count, struct index_zone *, "zones", &index->zones); if (result != UDS_SUCCESS) { uds_free_index(index); @@ -1289,12 +1289,12 @@ void uds_free_index(struct uds_index *index) if (index->zones != NULL) { for (i = 0; i < index->zone_count; i++) free_index_zone(index->zones[i]); - uds_free(index->zones); + vdo_free(index->zones); } uds_free_volume(index->volume); - uds_free_index_layout(uds_forget(index->layout)); - uds_free(index); + uds_free_index_layout(vdo_forget(index->layout)); + vdo_free(index); } /* Wait for the chapter writer to complete any outstanding writes. */ diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c index fecd436986ae..749c950c0189 100644 --- a/drivers/md/dm-vdo/indexer/io-factory.c +++ b/drivers/md/dm-vdo/indexer/io-factory.c @@ -64,7 +64,7 @@ int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_p int result; struct io_factory *factory; - result = uds_allocate(1, struct io_factory, __func__, &factory); + result = vdo_allocate(1, struct io_factory, __func__, &factory); if (result != UDS_SUCCESS) return result; @@ -85,7 +85,7 @@ int uds_replace_storage(struct io_factory *factory, struct block_device *bdev) void uds_put_io_factory(struct io_factory *factory) { if (atomic_add_return(-1, &factory->ref_count) <= 0) - uds_free(factory); + vdo_free(factory); } size_t uds_get_writable_size(struct io_factory *factory) @@ -129,7 +129,7 @@ void uds_free_buffered_reader(struct buffered_reader *reader) dm_bufio_client_destroy(reader->client); uds_put_io_factory(reader->factory); - uds_free(reader); + vdo_free(reader); } /* Create a buffered reader for an index region starting at offset. */ @@ -144,7 +144,7 @@ int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block if (result != UDS_SUCCESS) return result; - result = uds_allocate(1, struct buffered_reader, "buffered reader", &reader); + result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader); if (result != UDS_SUCCESS) { dm_bufio_client_destroy(client); return result; @@ -177,7 +177,7 @@ static int position_reader(struct buffered_reader *reader, sector_t block_number return UDS_OUT_OF_RANGE; if (reader->buffer != NULL) - dm_bufio_release(uds_forget(reader->buffer)); + dm_bufio_release(vdo_forget(reader->buffer)); data = dm_bufio_read(reader->client, block_number, &buffer); if (IS_ERR(data)) @@ -282,7 +282,7 @@ int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block if (result != UDS_SUCCESS) return result; - result = uds_allocate(1, struct buffered_writer, "buffered writer", &writer); + result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer); if (result != UDS_SUCCESS) { dm_bufio_client_destroy(client); return result; @@ -369,7 +369,7 @@ void uds_free_buffered_writer(struct buffered_writer *writer) dm_bufio_client_destroy(writer->client); uds_put_io_factory(writer->factory); - uds_free(writer); + vdo_free(writer); } /* diff --git a/drivers/md/dm-vdo/indexer/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c index cd2d35e39c20..4a4dc94915dd 100644 --- a/drivers/md/dm-vdo/indexer/open-chapter.c +++ b/drivers/md/dm-vdo/indexer/open-chapter.c @@ -68,7 +68,7 @@ int uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zo size_t capacity = geometry->records_per_chapter / zone_count; size_t slot_count = (1 << bits_per(capacity * LOAD_RATIO)); - result = uds_allocate_extended(struct open_chapter_zone, slot_count, + result = vdo_allocate_extended(struct open_chapter_zone, slot_count, struct open_chapter_zone_slot, "open chapter", &open_chapter); if (result != UDS_SUCCESS) @@ -76,7 +76,7 @@ int uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zo open_chapter->slot_count = slot_count; open_chapter->capacity = capacity; - result = uds_allocate_cache_aligned(records_size(open_chapter), "record pages", + result = vdo_allocate_cache_aligned(records_size(open_chapter), "record pages", &open_chapter->records); if (result != UDS_SUCCESS) { uds_free_open_chapter(open_chapter); @@ -194,8 +194,8 @@ void uds_remove_from_open_chapter(struct open_chapter_zone *open_chapter, void uds_free_open_chapter(struct open_chapter_zone *open_chapter) { if (open_chapter != NULL) { - uds_free(open_chapter->records); - uds_free(open_chapter); + vdo_free(open_chapter->records); + vdo_free(open_chapter); } } diff --git a/drivers/md/dm-vdo/indexer/radix-sort.c b/drivers/md/dm-vdo/indexer/radix-sort.c index b86d55f0827e..74ea18b8e9be 100644 --- a/drivers/md/dm-vdo/indexer/radix-sort.c +++ b/drivers/md/dm-vdo/indexer/radix-sort.c @@ -211,7 +211,7 @@ int uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter) unsigned int stack_size = count / INSERTION_SORT_THRESHOLD; struct radix_sorter *radix_sorter; - result = uds_allocate_extended(struct radix_sorter, stack_size, struct task, + result = vdo_allocate_extended(struct radix_sorter, stack_size, struct task, __func__, &radix_sorter); if (result != UDS_SUCCESS) return result; @@ -224,7 +224,7 @@ int uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter) void uds_free_radix_sorter(struct radix_sorter *sorter) { - uds_free(sorter); + vdo_free(sorter); } /* diff --git a/drivers/md/dm-vdo/indexer/sparse-cache.c b/drivers/md/dm-vdo/indexer/sparse-cache.c index 9e8672cba3fa..e297ba2d6ceb 100644 --- a/drivers/md/dm-vdo/indexer/sparse-cache.c +++ b/drivers/md/dm-vdo/indexer/sparse-cache.c @@ -222,12 +222,12 @@ static int __must_check initialize_cached_chapter_index(struct cached_chapter_in chapter->virtual_chapter = NO_CHAPTER; chapter->index_pages_count = geometry->index_pages_per_chapter; - result = uds_allocate(chapter->index_pages_count, struct delta_index_page, + result = vdo_allocate(chapter->index_pages_count, struct delta_index_page, __func__, &chapter->index_pages); if (result != UDS_SUCCESS) return result; - return uds_allocate(chapter->index_pages_count, struct dm_buffer *, + return vdo_allocate(chapter->index_pages_count, struct dm_buffer *, "sparse index volume pages", &chapter->page_buffers); } @@ -241,7 +241,7 @@ static int __must_check make_search_list(struct sparse_cache *cache, bytes = (sizeof(struct search_list) + (cache->capacity * sizeof(struct cached_chapter_index *))); - result = uds_allocate_cache_aligned(bytes, "search list", &list); + result = vdo_allocate_cache_aligned(bytes, "search list", &list); if (result != UDS_SUCCESS) return result; @@ -264,7 +264,7 @@ int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int ca unsigned int bytes; bytes = (sizeof(struct sparse_cache) + (capacity * sizeof(struct cached_chapter_index))); - result = uds_allocate_cache_aligned(bytes, "sparse cache", &cache); + result = vdo_allocate_cache_aligned(bytes, "sparse cache", &cache); if (result != UDS_SUCCESS) return result; @@ -294,7 +294,7 @@ int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int ca } /* purge_search_list() needs some temporary lists for sorting. */ - result = uds_allocate(capacity * 2, struct cached_chapter_index *, + result = vdo_allocate(capacity * 2, struct cached_chapter_index *, "scratch entries", &cache->scratch_entries); if (result != UDS_SUCCESS) goto out; @@ -338,7 +338,7 @@ static void release_cached_chapter_index(struct cached_chapter_index *chapter) for (i = 0; i < chapter->index_pages_count; i++) { if (chapter->page_buffers[i] != NULL) - dm_bufio_release(uds_forget(chapter->page_buffers[i])); + dm_bufio_release(vdo_forget(chapter->page_buffers[i])); } } @@ -349,18 +349,18 @@ void uds_free_sparse_cache(struct sparse_cache *cache) if (cache == NULL) return; - uds_free(cache->scratch_entries); + vdo_free(cache->scratch_entries); for (i = 0; i < cache->zone_count; i++) - uds_free(cache->search_lists[i]); + vdo_free(cache->search_lists[i]); for (i = 0; i < cache->capacity; i++) { release_cached_chapter_index(&cache->chapters[i]); - uds_free(cache->chapters[i].index_pages); - uds_free(cache->chapters[i].page_buffers); + vdo_free(cache->chapters[i].index_pages); + vdo_free(cache->chapters[i].page_buffers); } - uds_free(cache); + vdo_free(cache); } /* diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c index a88e515ceef6..1a762e6dd709 100644 --- a/drivers/md/dm-vdo/indexer/volume-index.c +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -279,8 +279,8 @@ static int compute_volume_sub_index_parameters(const struct uds_configuration *c static void uninitialize_volume_sub_index(struct volume_sub_index *sub_index) { - uds_free(uds_forget(sub_index->flush_chapters)); - uds_free(uds_forget(sub_index->zones)); + vdo_free(vdo_forget(sub_index->flush_chapters)); + vdo_free(vdo_forget(sub_index->zones)); uds_uninitialize_delta_index(&sub_index->delta_index); } @@ -290,11 +290,11 @@ void uds_free_volume_index(struct volume_index *volume_index) return; if (volume_index->zones != NULL) - uds_free(uds_forget(volume_index->zones)); + vdo_free(vdo_forget(volume_index->zones)); uninitialize_volume_sub_index(&volume_index->vi_non_hook); uninitialize_volume_sub_index(&volume_index->vi_hook); - uds_free(volume_index); + vdo_free(volume_index); } @@ -1211,12 +1211,12 @@ static int initialize_volume_sub_index(const struct uds_configuration *config, (zone_count * sizeof(struct volume_sub_index_zone))); /* The following arrays are initialized to all zeros. */ - result = uds_allocate(params.list_count, u64, "first chapter to flush", + result = vdo_allocate(params.list_count, u64, "first chapter to flush", &sub_index->flush_chapters); if (result != UDS_SUCCESS) return result; - return uds_allocate(zone_count, struct volume_sub_index_zone, + return vdo_allocate(zone_count, struct volume_sub_index_zone, "volume index zones", &sub_index->zones); } @@ -1228,7 +1228,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non struct volume_index *volume_index; int result; - result = uds_allocate(1, struct volume_index, "volume index", &volume_index); + result = vdo_allocate(1, struct volume_index, "volume index", &volume_index); if (result != UDS_SUCCESS) return result; @@ -1249,7 +1249,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non volume_index->sparse_sample_rate = config->sparse_sample_rate; - result = uds_allocate(config->zone_count, struct volume_index_zone, + result = vdo_allocate(config->zone_count, struct volume_index_zone, "volume index zones", &volume_index->zones); if (result != UDS_SUCCESS) { uds_free_volume_index(volume_index); diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c index 002a4264a163..2d8901732f5d 100644 --- a/drivers/md/dm-vdo/indexer/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -198,7 +198,7 @@ static void wait_for_pending_searches(struct page_cache *cache, u32 physical_pag static void release_page_buffer(struct cached_page *page) { if (page->buffer != NULL) - dm_bufio_release(uds_forget(page->buffer)); + dm_bufio_release(vdo_forget(page->buffer)); } static void clear_cache_page(struct page_cache *cache, struct cached_page *page) @@ -1482,7 +1482,7 @@ int __must_check uds_replace_volume_storage(struct volume *volume, if (volume->sparse_cache != NULL) uds_invalidate_sparse_cache(volume->sparse_cache); if (volume->client != NULL) - dm_bufio_client_destroy(uds_forget(volume->client)); + dm_bufio_client_destroy(vdo_forget(volume->client)); return uds_open_volume_bufio(layout, volume->geometry->bytes_per_page, volume->reserved_buffers, &volume->client); @@ -1507,22 +1507,22 @@ static int __must_check initialize_page_cache(struct page_cache *cache, if (result != UDS_SUCCESS) return result; - result = uds_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read, + result = vdo_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read, "volume read queue", &cache->read_queue); if (result != UDS_SUCCESS) return result; - result = uds_allocate(cache->zone_count, struct search_pending_counter, + result = vdo_allocate(cache->zone_count, struct search_pending_counter, "Volume Cache Zones", &cache->search_pending_counters); if (result != UDS_SUCCESS) return result; - result = uds_allocate(cache->indexable_pages, u16, "page cache index", + result = vdo_allocate(cache->indexable_pages, u16, "page cache index", &cache->index); if (result != UDS_SUCCESS) return result; - result = uds_allocate(cache->cache_slots, struct cached_page, "page cache cache", + result = vdo_allocate(cache->cache_slots, struct cached_page, "page cache cache", &cache->cache); if (result != UDS_SUCCESS) return result; @@ -1546,7 +1546,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout unsigned int reserved_buffers; int result; - result = uds_allocate(1, struct volume, "volume", &volume); + result = vdo_allocate(1, struct volume, "volume", &volume); if (result != UDS_SUCCESS) return result; @@ -1583,7 +1583,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout return result; } - result = uds_allocate(geometry->records_per_page, + result = vdo_allocate(geometry->records_per_page, const struct uds_volume_record *, "record pointers", &volume->record_pointers); if (result != UDS_SUCCESS) { @@ -1624,7 +1624,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout uds_init_cond(&volume->read_threads_read_done_cond); uds_init_cond(&volume->read_threads_cond); - result = uds_allocate(config->read_threads, struct thread *, "reader threads", + result = vdo_allocate(config->read_threads, struct thread *, "reader threads", &volume->reader_threads); if (result != UDS_SUCCESS) { uds_free_volume(volume); @@ -1654,10 +1654,10 @@ static void uninitialize_page_cache(struct page_cache *cache) for (i = 0; i < cache->cache_slots; i++) release_page_buffer(&cache->cache[i]); } - uds_free(cache->index); - uds_free(cache->cache); - uds_free(cache->search_pending_counters); - uds_free(cache->read_queue); + vdo_free(cache->index); + vdo_free(cache->cache); + vdo_free(cache->search_pending_counters); + vdo_free(cache->read_queue); } void uds_free_volume(struct volume *volume) @@ -1675,7 +1675,7 @@ void uds_free_volume(struct volume *volume) mutex_unlock(&volume->read_threads_mutex); for (i = 0; i < volume->read_thread_count; i++) vdo_join_threads(volume->reader_threads[i]); - uds_free(volume->reader_threads); + vdo_free(volume->reader_threads); volume->reader_threads = NULL; } @@ -1683,11 +1683,11 @@ void uds_free_volume(struct volume *volume) uninitialize_page_cache(&volume->page_cache); uds_free_sparse_cache(volume->sparse_cache); if (volume->client != NULL) - dm_bufio_client_destroy(uds_forget(volume->client)); + dm_bufio_client_destroy(vdo_forget(volume->client)); uds_free_index_page_map(volume->index_page_map); uds_free_radix_sorter(volume->radix_sorter); - uds_free(volume->geometry); - uds_free(volume->record_pointers); - uds_free(volume); + vdo_free(volume->geometry); + vdo_free(volume->record_pointers); + vdo_free(volume); } diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index e0953e013f15..0bd742ecbe2e 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -164,7 +164,7 @@ static int allocate_buckets(struct int_map *map, size_t capacity) * without have to wrap back around to element zero. */ map->bucket_count = capacity + (NEIGHBORHOOD - 1); - return uds_allocate(map->bucket_count, struct bucket, + return vdo_allocate(map->bucket_count, struct bucket, "struct int_map buckets", &map->buckets); } @@ -182,7 +182,7 @@ int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) int result; size_t capacity; - result = uds_allocate(1, struct int_map, "struct int_map", &map); + result = vdo_allocate(1, struct int_map, "struct int_map", &map); if (result != UDS_SUCCESS) return result; @@ -197,7 +197,7 @@ int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) result = allocate_buckets(map, capacity); if (result != UDS_SUCCESS) { - vdo_int_map_free(uds_forget(map)); + vdo_int_map_free(vdo_forget(map)); return result; } @@ -217,8 +217,8 @@ void vdo_int_map_free(struct int_map *map) if (map == NULL) return; - uds_free(uds_forget(map->buckets)); - uds_free(uds_forget(map)); + vdo_free(vdo_forget(map->buckets)); + vdo_free(vdo_forget(map)); } /** @@ -399,14 +399,14 @@ static int resize_buckets(struct int_map *map) result = vdo_int_map_put(map, entry->key, entry->value, true, NULL); if (result != UDS_SUCCESS) { /* Destroy the new partial map and restore the map from the stack. */ - uds_free(uds_forget(map->buckets)); + vdo_free(vdo_forget(map->buckets)); *map = old_map; return result; } } /* Destroy the old bucket array. */ - uds_free(uds_forget(old_map.buckets)); + vdo_free(vdo_forget(old_map.buckets)); return UDS_SUCCESS; } diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 6c050f2b3b44..23549b7e9e6d 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -380,7 +380,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter struct io_submitter *io_submitter; int result; - result = uds_allocate_extended(struct io_submitter, thread_count, + result = vdo_allocate_extended(struct io_submitter, thread_count, struct bio_queue_data, "bio submission data", &io_submitter); if (result != UDS_SUCCESS) @@ -422,7 +422,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter * Clean up the partially initialized bio-queue entirely and indicate that * initialization failed. */ - vdo_int_map_free(uds_forget(bio_queue_data->map)); + vdo_int_map_free(vdo_forget(bio_queue_data->map)); uds_log_error("bio queue initialization failed %d", result); vdo_cleanup_io_submitter(io_submitter); vdo_free_io_submitter(io_submitter); @@ -470,8 +470,8 @@ void vdo_free_io_submitter(struct io_submitter *io_submitter) for (i = io_submitter->num_bio_queues_used - 1; i >= 0; i--) { io_submitter->num_bio_queues_used--; /* vdo_destroy() will free the work queue, so just give up our reference to it. */ - uds_forget(io_submitter->bio_queue_data[i].queue); - vdo_int_map_free(uds_forget(io_submitter->bio_queue_data[i].map)); + vdo_forget(io_submitter->bio_queue_data[i].queue); + vdo_int_map_free(vdo_forget(io_submitter->bio_queue_data[i].map)); } - uds_free(io_submitter); + vdo_free(io_submitter); } diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index c5b3b1c111e3..ca5bc3be7978 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -94,7 +94,7 @@ int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr) if (zone_count == 0) return VDO_SUCCESS; - result = uds_allocate_extended(struct logical_zones, zone_count, + result = vdo_allocate_extended(struct logical_zones, zone_count, struct logical_zone, __func__, &zones); if (result != VDO_SUCCESS) return result; @@ -132,12 +132,12 @@ void vdo_free_logical_zones(struct logical_zones *zones) if (zones == NULL) return; - uds_free(uds_forget(zones->manager)); + vdo_free(vdo_forget(zones->manager)); for (index = 0; index < zones->zone_count; index++) - vdo_int_map_free(uds_forget(zones->zones[index].lbn_operations)); + vdo_int_map_free(vdo_forget(zones->zones[index].lbn_operations)); - uds_free(zones); + vdo_free(zones); } static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what) diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index db085c1124be..8d5df3e45a24 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -37,7 +37,7 @@ static inline bool allocations_allowed(void) * @new_thread: registered_thread structure to use for the current thread * @flag_ptr: Location of the allocation-allowed flag */ -void uds_register_allocating_thread(struct registered_thread *new_thread, +void vdo_register_allocating_thread(struct registered_thread *new_thread, const bool *flag_ptr) { if (flag_ptr == NULL) { @@ -50,7 +50,7 @@ void uds_register_allocating_thread(struct registered_thread *new_thread, } /* Unregister the current thread as an allocating thread. */ -void uds_unregister_allocating_thread(void) +void vdo_unregister_allocating_thread(void) { vdo_unregister_thread(&allocating_threads); } @@ -148,7 +148,7 @@ static void remove_vmalloc_block(void *ptr) spin_unlock_irqrestore(&memory_stats.lock, flags); if (block != NULL) - uds_free(block); + vdo_free(block); else uds_log_info("attempting to remove ptr %px not found in vmalloc list", ptr); } @@ -196,7 +196,7 @@ static inline bool use_kmalloc(size_t size) * * Return: UDS_SUCCESS or an error code */ -int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) +int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) { /* * The __GFP_RETRY_MAYFAIL flag means the VM implementation will retry memory reclaim @@ -245,8 +245,7 @@ int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) } else { struct vmalloc_block_info *block; - if (uds_allocate(1, struct vmalloc_block_info, __func__, &block) == - UDS_SUCCESS) { + if (vdo_allocate(1, struct vmalloc_block_info, __func__, &block) == UDS_SUCCESS) { /* * It is possible for __vmalloc to fail to allocate memory because there * are no pages available. A short sleep may allow the page reclaimer @@ -259,7 +258,6 @@ int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) */ for (;;) { p = __vmalloc(size, gfp_flags | __GFP_NOWARN); - if (p != NULL) break; @@ -273,7 +271,7 @@ int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) } if (p == NULL) { - uds_free(block); + vdo_free(block); } else { block->ptr = p; block->size = PAGE_ALIGN(size); @@ -304,7 +302,7 @@ int uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr) * * Return: pointer to the allocated memory, or NULL if the required space is not available. */ -void *uds_allocate_memory_nowait(size_t size, const char *what __maybe_unused) +void *vdo_allocate_memory_nowait(size_t size, const char *what __maybe_unused) { void *p = kmalloc(size, GFP_NOWAIT | __GFP_ZERO); @@ -314,7 +312,7 @@ void *uds_allocate_memory_nowait(size_t size, const char *what __maybe_unused) return p; } -void uds_free(void *ptr) +void vdo_free(void *ptr) { if (ptr != NULL) { if (is_vmalloc_addr(ptr)) { @@ -339,18 +337,18 @@ void uds_free(void *ptr) * * Return: UDS_SUCCESS or an error code */ -int uds_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what, +int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what, void *new_ptr) { int result; if (size == 0) { - uds_free(ptr); + vdo_free(ptr); *(void **) new_ptr = NULL; return UDS_SUCCESS; } - result = uds_allocate(size, char, what, new_ptr); + result = vdo_allocate(size, char, what, new_ptr); if (result != UDS_SUCCESS) return result; @@ -359,18 +357,18 @@ int uds_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *w size = old_size; memcpy(*((void **) new_ptr), ptr, size); - uds_free(ptr); + vdo_free(ptr); } return UDS_SUCCESS; } -int uds_duplicate_string(const char *string, const char *what, char **new_string) +int vdo_duplicate_string(const char *string, const char *what, char **new_string) { int result; u8 *dup; - result = uds_allocate(strlen(string) + 1, u8, what, &dup); + result = vdo_allocate(strlen(string) + 1, u8, what, &dup); if (result != UDS_SUCCESS) return result; @@ -379,13 +377,13 @@ int uds_duplicate_string(const char *string, const char *what, char **new_string return UDS_SUCCESS; } -void uds_memory_init(void) +void vdo_memory_init(void) { spin_lock_init(&memory_stats.lock); vdo_initialize_thread_registry(&allocating_threads); } -void uds_memory_exit(void) +void vdo_memory_exit(void) { ASSERT_LOG_ONLY(memory_stats.kmalloc_bytes == 0, "kmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", @@ -396,7 +394,7 @@ void uds_memory_exit(void) uds_log_debug("peak usage %zd bytes", memory_stats.peak_bytes); } -void uds_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used) +void vdo_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used) { unsigned long flags; @@ -410,7 +408,7 @@ void uds_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used) * Report stats on any allocated memory that we're tracking. Not all allocation types are * guaranteed to be tracked in bytes (e.g., bios). */ -void uds_report_memory_usage(void) +void vdo_report_memory_usage(void) { unsigned long flags; u64 kmalloc_blocks; diff --git a/drivers/md/dm-vdo/memory-alloc.h b/drivers/md/dm-vdo/memory-alloc.h index d72d597f98cf..3f27dd722a2d 100644 --- a/drivers/md/dm-vdo/memory-alloc.h +++ b/drivers/md/dm-vdo/memory-alloc.h @@ -3,8 +3,8 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_MEMORY_ALLOC_H -#define UDS_MEMORY_ALLOC_H +#ifndef VDO_MEMORY_ALLOC_H +#define VDO_MEMORY_ALLOC_H #include #include /* for PAGE_SIZE */ @@ -12,8 +12,8 @@ #include "permassert.h" #include "thread-registry.h" -/* Custom memory allocation function for UDS that tracks memory usage */ -int __must_check uds_allocate_memory(size_t size, size_t align, const char *what, void *ptr); +/* Custom memory allocation function that tracks memory usage */ +int __must_check vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr); /* * Allocate storage based on element counts, sizes, and alignment. @@ -37,7 +37,7 @@ int __must_check uds_allocate_memory(size_t size, size_t align, const char *what * * Return: UDS_SUCCESS or an error code */ -static inline int uds_do_allocation(size_t count, size_t size, size_t extra, +static inline int vdo_do_allocation(size_t count, size_t size, size_t extra, size_t align, const char *what, void *ptr) { size_t total_size = count * size + extra; @@ -53,7 +53,7 @@ static inline int uds_do_allocation(size_t count, size_t size, size_t extra, total_size = SIZE_MAX; } - return uds_allocate_memory(total_size, align, what, ptr); + return vdo_allocate_memory(total_size, align, what, ptr); } /* @@ -67,8 +67,8 @@ static inline int uds_do_allocation(size_t count, size_t size, size_t extra, * * Return: UDS_SUCCESS or an error code */ -#define uds_allocate(COUNT, TYPE, WHAT, PTR) \ - uds_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) +#define vdo_allocate(COUNT, TYPE, WHAT, PTR) \ + vdo_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) /* * Allocate one object of an indicated type, followed by one or more elements of a second type, @@ -83,12 +83,12 @@ static inline int uds_do_allocation(size_t count, size_t size, size_t extra, * * Return: UDS_SUCCESS or an error code */ -#define uds_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ +#define vdo_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ __extension__({ \ int _result; \ TYPE1 **_ptr = (PTR); \ BUILD_BUG_ON(__alignof__(TYPE1) < __alignof__(TYPE2)); \ - _result = uds_do_allocation(COUNT, \ + _result = vdo_do_allocation(COUNT, \ sizeof(TYPE2), \ sizeof(TYPE1), \ __alignof__(TYPE1), \ @@ -107,9 +107,9 @@ static inline int uds_do_allocation(size_t count, size_t size, size_t extra, * * Return: UDS_SUCCESS or an error code */ -static inline int __must_check uds_allocate_cache_aligned(size_t size, const char *what, void *ptr) +static inline int __must_check vdo_allocate_cache_aligned(size_t size, const char *what, void *ptr) { - return uds_allocate_memory(size, L1_CACHE_BYTES, what, ptr); + return vdo_allocate_memory(size, L1_CACHE_BYTES, what, ptr); } /* @@ -121,18 +121,18 @@ static inline int __must_check uds_allocate_cache_aligned(size_t size, const cha * * Return: pointer to the memory, or NULL if the memory is not available. */ -void *__must_check uds_allocate_memory_nowait(size_t size, const char *what); +void *__must_check vdo_allocate_memory_nowait(size_t size, const char *what); -int __must_check uds_reallocate_memory(void *ptr, size_t old_size, size_t size, +int __must_check vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what, void *new_ptr); -int __must_check uds_duplicate_string(const char *string, const char *what, +int __must_check vdo_duplicate_string(const char *string, const char *what, char **new_string); -/* Free memory allocated with uds_allocate(). */ -void uds_free(void *ptr); +/* Free memory allocated with vdo_allocate(). */ +void vdo_free(void *ptr); -static inline void *__uds_forget(void **ptr_ptr) +static inline void *__vdo_forget(void **ptr_ptr) { void *ptr = *ptr_ptr; @@ -144,19 +144,19 @@ static inline void *__uds_forget(void **ptr_ptr) * Null out a pointer and return a copy to it. This macro should be used when passing a pointer to * a function for which it is not safe to access the pointer once the function returns. */ -#define uds_forget(ptr) __uds_forget((void **) &(ptr)) +#define vdo_forget(ptr) __vdo_forget((void **) &(ptr)) -void uds_memory_init(void); +void vdo_memory_init(void); -void uds_memory_exit(void); +void vdo_memory_exit(void); -void uds_register_allocating_thread(struct registered_thread *new_thread, +void vdo_register_allocating_thread(struct registered_thread *new_thread, const bool *flag_ptr); -void uds_unregister_allocating_thread(void); +void vdo_unregister_allocating_thread(void); -void uds_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used); +void vdo_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used); -void uds_report_memory_usage(void); +void vdo_report_memory_usage(void); -#endif /* UDS_MEMORY_ALLOC_H */ +#endif /* VDO_MEMORY_ALLOC_H */ diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index cac7232f467b..ec24fff2a21b 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -419,7 +419,7 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) struct vdo_statistics *stats; int result; - result = uds_allocate(1, struct vdo_statistics, __func__, &stats); + result = vdo_allocate(1, struct vdo_statistics, __func__, &stats); if (result != UDS_SUCCESS) { uds_log_error("Cannot allocate memory to write VDO statistics"); return result; @@ -427,6 +427,6 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) vdo_fetch_statistics(vdo, stats); write_vdo_statistics(NULL, stats, NULL, &buf, &maxlen); - uds_free(stats); + vdo_free(stats); return VDO_SUCCESS; } diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c index 3283c8d56c59..5774d8fd5c5a 100644 --- a/drivers/md/dm-vdo/packer.c +++ b/drivers/md/dm-vdo/packer.c @@ -120,7 +120,7 @@ static int __must_check make_bin(struct packer *packer) struct packer_bin *bin; int result; - result = uds_allocate_extended(struct packer_bin, VDO_MAX_COMPRESSION_SLOTS, + result = vdo_allocate_extended(struct packer_bin, VDO_MAX_COMPRESSION_SLOTS, struct vio *, __func__, &bin); if (result != VDO_SUCCESS) return result; @@ -146,7 +146,7 @@ int vdo_make_packer(struct vdo *vdo, block_count_t bin_count, struct packer **pa block_count_t i; int result; - result = uds_allocate(1, struct packer, __func__, &packer); + result = vdo_allocate(1, struct packer, __func__, &packer); if (result != VDO_SUCCESS) return result; @@ -168,7 +168,7 @@ int vdo_make_packer(struct vdo *vdo, block_count_t bin_count, struct packer **pa * bin must have a canceler for which it is waiting, and any canceler will only have * canceled one lock holder at a time. */ - result = uds_allocate_extended(struct packer_bin, MAXIMUM_VDO_USER_VIOS / 2, + result = vdo_allocate_extended(struct packer_bin, MAXIMUM_VDO_USER_VIOS / 2, struct vio *, __func__, &packer->canceled_bin); if (result != VDO_SUCCESS) { vdo_free_packer(packer); @@ -198,11 +198,11 @@ void vdo_free_packer(struct packer *packer) list_for_each_entry_safe(bin, tmp, &packer->bins, list) { list_del_init(&bin->list); - uds_free(bin); + vdo_free(bin); } - uds_free(uds_forget(packer->canceled_bin)); - uds_free(packer); + vdo_free(vdo_forget(packer->canceled_bin)); + vdo_free(packer); } /** @@ -669,7 +669,7 @@ void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion) assert_data_vio_in_packer_zone(data_vio); - lock_holder = uds_forget(data_vio->compression.lock_holder); + lock_holder = vdo_forget(data_vio->compression.lock_holder); bin = lock_holder->compression.bin; ASSERT_LOG_ONLY((bin != NULL), "data_vio in packer has a bin"); diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index 62d142b28282..fadcea23288e 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -239,7 +239,7 @@ static int make_pbn_lock_pool(size_t capacity, struct pbn_lock_pool **pool_ptr) struct pbn_lock_pool *pool; int result; - result = uds_allocate_extended(struct pbn_lock_pool, capacity, idle_pbn_lock, + result = vdo_allocate_extended(struct pbn_lock_pool, capacity, idle_pbn_lock, __func__, &pool); if (result != VDO_SUCCESS) return result; @@ -270,7 +270,7 @@ static void free_pbn_lock_pool(struct pbn_lock_pool *pool) ASSERT_LOG_ONLY(pool->borrowed == 0, "All PBN locks must be returned to the pool before it is freed, but %zu locks are still on loan", pool->borrowed); - uds_free(pool); + vdo_free(pool); } /** @@ -344,7 +344,7 @@ static int initialize_zone(struct vdo *vdo, struct physical_zones *zones) zone->next = &zones->zones[(zone_number + 1) % vdo->thread_config.physical_zone_count]; result = vdo_make_default_thread(vdo, zone->thread_id); if (result != VDO_SUCCESS) { - free_pbn_lock_pool(uds_forget(zone->lock_pool)); + free_pbn_lock_pool(vdo_forget(zone->lock_pool)); vdo_int_map_free(zone->pbn_operations); return result; } @@ -367,7 +367,7 @@ int vdo_make_physical_zones(struct vdo *vdo, struct physical_zones **zones_ptr) if (zone_count == 0) return VDO_SUCCESS; - result = uds_allocate_extended(struct physical_zones, zone_count, + result = vdo_allocate_extended(struct physical_zones, zone_count, struct physical_zone, __func__, &zones); if (result != VDO_SUCCESS) return result; @@ -398,11 +398,11 @@ void vdo_free_physical_zones(struct physical_zones *zones) for (index = 0; index < zones->zone_count; index++) { struct physical_zone *zone = &zones->zones[index]; - free_pbn_lock_pool(uds_forget(zone->lock_pool)); - vdo_int_map_free(uds_forget(zone->pbn_operations)); + free_pbn_lock_pool(vdo_forget(zone->lock_pool)); + vdo_int_map_free(vdo_forget(zone->pbn_operations)); } - uds_free(zones); + vdo_free(zones); } /** @@ -460,7 +460,7 @@ int vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, if (lock != NULL) { /* The lock is already held, so we don't need the borrowed one. */ - return_pbn_lock_to_pool(zone->lock_pool, uds_forget(new_lock)); + return_pbn_lock_to_pool(zone->lock_pool, vdo_forget(new_lock)); result = ASSERT(lock->holder_count > 0, "physical block %llu lock held", (unsigned long long) pbn); if (result != VDO_SUCCESS) diff --git a/drivers/md/dm-vdo/pool-sysfs.c b/drivers/md/dm-vdo/pool-sysfs.c index f2be0f2bbd68..6769c5711cbc 100644 --- a/drivers/md/dm-vdo/pool-sysfs.c +++ b/drivers/md/dm-vdo/pool-sysfs.c @@ -110,7 +110,7 @@ static ssize_t pool_requests_maximum_show(struct vdo *vdo, char *buf) static void vdo_pool_release(struct kobject *directory) { - uds_free(container_of(directory, struct vdo, vdo_directory)); + vdo_free(container_of(directory, struct vdo, vdo_directory)); } static struct pool_attribute vdo_pool_compressing_attr = { diff --git a/drivers/md/dm-vdo/priority-table.c b/drivers/md/dm-vdo/priority-table.c index a59e9d40ca90..fc99268d2437 100644 --- a/drivers/md/dm-vdo/priority-table.c +++ b/drivers/md/dm-vdo/priority-table.c @@ -60,7 +60,7 @@ int vdo_make_priority_table(unsigned int max_priority, struct priority_table **t if (max_priority > MAX_PRIORITY) return UDS_INVALID_ARGUMENT; - result = uds_allocate_extended(struct priority_table, max_priority + 1, + result = vdo_allocate_extended(struct priority_table, max_priority + 1, struct bucket, __func__, &table); if (result != VDO_SUCCESS) return result; @@ -96,7 +96,7 @@ void vdo_free_priority_table(struct priority_table *table) */ vdo_reset_priority_table(table); - uds_free(table); + vdo_free(table); } /** diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index c1d355346bcf..615755697e60 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -591,31 +591,31 @@ static int __must_check initialize_lock_counter(struct recovery_journal *journal struct thread_config *config = &vdo->thread_config; struct lock_counter *counter = &journal->lock_counter; - result = uds_allocate(journal->size, u16, __func__, &counter->journal_counters); + result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters); if (result != VDO_SUCCESS) return result; - result = uds_allocate(journal->size, atomic_t, __func__, + result = vdo_allocate(journal->size, atomic_t, __func__, &counter->journal_decrement_counts); if (result != VDO_SUCCESS) return result; - result = uds_allocate(journal->size * config->logical_zone_count, u16, __func__, + result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__, &counter->logical_counters); if (result != VDO_SUCCESS) return result; - result = uds_allocate(journal->size, atomic_t, __func__, + result = vdo_allocate(journal->size, atomic_t, __func__, &counter->logical_zone_counts); if (result != VDO_SUCCESS) return result; - result = uds_allocate(journal->size * config->physical_zone_count, u16, __func__, + result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__, &counter->physical_counters); if (result != VDO_SUCCESS) return result; - result = uds_allocate(journal->size, atomic_t, __func__, + result = vdo_allocate(journal->size, atomic_t, __func__, &counter->physical_zone_counts); if (result != VDO_SUCCESS) return result; @@ -670,14 +670,14 @@ static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *j * Allocate a full block for the journal block even though not all of the space is used * since the VIO needs to write a full disk block. */ - result = uds_allocate(VDO_BLOCK_SIZE, char, __func__, &data); + result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &data); if (result != VDO_SUCCESS) return result; result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL, VIO_PRIORITY_HIGH, block, 1, data, &block->vio); if (result != VDO_SUCCESS) { - uds_free(data); + vdo_free(data); return result; } @@ -709,7 +709,7 @@ int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t struct recovery_journal *journal; int result; - result = uds_allocate_extended(struct recovery_journal, + result = vdo_allocate_extended(struct recovery_journal, RECOVERY_JOURNAL_RESERVED_BLOCKS, struct recovery_journal_block, __func__, &journal); @@ -787,13 +787,13 @@ void vdo_free_recovery_journal(struct recovery_journal *journal) if (journal == NULL) return; - uds_free(uds_forget(journal->lock_counter.logical_zone_counts)); - uds_free(uds_forget(journal->lock_counter.physical_zone_counts)); - uds_free(uds_forget(journal->lock_counter.journal_counters)); - uds_free(uds_forget(journal->lock_counter.journal_decrement_counts)); - uds_free(uds_forget(journal->lock_counter.logical_counters)); - uds_free(uds_forget(journal->lock_counter.physical_counters)); - free_vio(uds_forget(journal->flush_vio)); + vdo_free(vdo_forget(journal->lock_counter.logical_zone_counts)); + vdo_free(vdo_forget(journal->lock_counter.physical_zone_counts)); + vdo_free(vdo_forget(journal->lock_counter.journal_counters)); + vdo_free(vdo_forget(journal->lock_counter.journal_decrement_counts)); + vdo_free(vdo_forget(journal->lock_counter.logical_counters)); + vdo_free(vdo_forget(journal->lock_counter.physical_counters)); + free_vio(vdo_forget(journal->flush_vio)); /* * FIXME: eventually, the journal should be constructed in a quiescent state which @@ -810,11 +810,11 @@ void vdo_free_recovery_journal(struct recovery_journal *journal) for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) { struct recovery_journal_block *block = &journal->blocks[i]; - uds_free(uds_forget(block->vio.data)); + vdo_free(vdo_forget(block->vio.data)); free_vio_components(&block->vio); } - uds_free(journal); + vdo_free(journal); } /** diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index 847aca9fbe47..bfcdedeedb86 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -226,7 +226,7 @@ static void uninitialize_vios(struct repair_completion *repair) while (repair->vio_count > 0) free_vio_components(&repair->vios[--repair->vio_count]); - uds_free(uds_forget(repair->vios)); + vdo_free(vdo_forget(repair->vios)); } static void free_repair_completion(struct repair_completion *repair) @@ -241,9 +241,9 @@ static void free_repair_completion(struct repair_completion *repair) repair->completion.vdo->block_map->zones[0].page_cache.rebuilding = false; uninitialize_vios(repair); - uds_free(uds_forget(repair->journal_data)); - uds_free(uds_forget(repair->entries)); - uds_free(repair); + vdo_free(vdo_forget(repair->journal_data)); + vdo_free(vdo_forget(repair->entries)); + vdo_free(repair); } static void finish_repair(struct vdo_completion *completion) @@ -262,7 +262,7 @@ static void finish_repair(struct vdo_completion *completion) repair->highest_tail, repair->logical_blocks_used, repair->block_map_data_blocks); - free_repair_completion(uds_forget(repair)); + free_repair_completion(vdo_forget(repair)); if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { uds_log_info("Read-only rebuild complete"); @@ -295,7 +295,7 @@ static void abort_repair(struct vdo_completion *completion) else uds_log_warning("Recovery aborted"); - free_repair_completion(uds_forget(repair)); + free_repair_completion(vdo_forget(repair)); vdo_continue_completion(parent, result); } @@ -1108,7 +1108,7 @@ static void recover_block_map(struct vdo_completion *completion) if (repair->block_map_entry_count == 0) { uds_log_info("Replaying 0 recovery entries into block map"); - uds_free(uds_forget(repair->journal_data)); + vdo_free(vdo_forget(repair->journal_data)); launch_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); return; } @@ -1418,7 +1418,7 @@ static int parse_journal_for_rebuild(struct repair_completion *repair) * packed_recovery_journal_entry from every valid journal block. */ count = ((repair->highest_tail - repair->block_map_head + 1) * entries_per_block); - result = uds_allocate(count, struct numbered_block_mapping, __func__, + result = vdo_allocate(count, struct numbered_block_mapping, __func__, &repair->entries); if (result != VDO_SUCCESS) return result; @@ -1464,7 +1464,7 @@ static int extract_new_mappings(struct repair_completion *repair) * Allocate an array of numbered_block_mapping structs just large enough to transcribe * every packed_recovery_journal_entry from every valid journal block. */ - result = uds_allocate(repair->entry_count, struct numbered_block_mapping, + result = vdo_allocate(repair->entry_count, struct numbered_block_mapping, __func__, &repair->entries); if (result != VDO_SUCCESS) return result; @@ -1709,7 +1709,7 @@ void vdo_repair(struct vdo_completion *parent) uds_log_warning("Device was dirty, rebuilding reference counts"); } - result = uds_allocate_extended(struct repair_completion, page_count, + result = vdo_allocate_extended(struct repair_completion, page_count, struct vdo_page_completion, __func__, &repair); if (result != VDO_SUCCESS) { @@ -1723,12 +1723,12 @@ void vdo_repair(struct vdo_completion *parent) prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); repair->page_count = page_count; - result = uds_allocate(remaining * VDO_BLOCK_SIZE, char, __func__, + result = vdo_allocate(remaining * VDO_BLOCK_SIZE, char, __func__, &repair->journal_data); if (abort_on_error(result, repair)) return; - result = uds_allocate(vio_count, struct vio, __func__, &repair->vios); + result = vdo_allocate(vio_count, struct vio, __func__, &repair->vios); if (abort_on_error(result, repair)) return; diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 8c6376e79a23..2d2cccf89edb 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -415,7 +415,7 @@ static void complete_reaping(struct vdo_completion *completion) struct slab_journal *journal = completion->parent; return_vio_to_pool(journal->slab->allocator->vio_pool, - vio_as_pooled_vio(as_vio(uds_forget(completion)))); + vio_as_pooled_vio(as_vio(vdo_forget(completion)))); finish_reaping(journal); reap_slab_journal(journal); } @@ -698,7 +698,7 @@ static void complete_write(struct vdo_completion *completion) sequence_number_t committed = get_committing_sequence_number(pooled); list_del_init(&pooled->list_entry); - return_vio_to_pool(journal->slab->allocator->vio_pool, uds_forget(pooled)); + return_vio_to_pool(journal->slab->allocator->vio_pool, vdo_forget(pooled)); if (result != VDO_SUCCESS) { vio_record_metadata_io_error(as_vio(completion)); @@ -777,7 +777,7 @@ static void write_slab_journal_block(struct vdo_waiter *waiter, void *context) * The slab summary update does a flush which is sufficient to protect us from corruption * due to out of order slab journal, reference block, or block map writes. */ - vdo_submit_metadata_vio(uds_forget(vio), block_number, write_slab_journal_endio, + vdo_submit_metadata_vio(vdo_forget(vio), block_number, write_slab_journal_endio, complete_write, REQ_OP_WRITE); /* Since the write is submitted, the tail block structure can be reused. */ @@ -2367,7 +2367,7 @@ static int allocate_slab_counters(struct vdo_slab *slab) if (result != VDO_SUCCESS) return result; - result = uds_allocate(slab->reference_block_count, struct reference_block, + result = vdo_allocate(slab->reference_block_count, struct reference_block, __func__, &slab->reference_blocks); if (result != VDO_SUCCESS) return result; @@ -2377,10 +2377,10 @@ static int allocate_slab_counters(struct vdo_slab *slab) * so we can word-search even at the very end. */ bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD); - result = uds_allocate(bytes, vdo_refcount_t, "ref counts array", + result = vdo_allocate(bytes, vdo_refcount_t, "ref counts array", &slab->counters); if (result != UDS_SUCCESS) { - uds_free(uds_forget(slab->reference_blocks)); + vdo_free(vdo_forget(slab->reference_blocks)); return result; } @@ -2658,7 +2658,7 @@ static inline bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubbe */ static void uninitialize_scrubber_vio(struct slab_scrubber *scrubber) { - uds_free(uds_forget(scrubber->vio.data)); + vdo_free(vdo_forget(scrubber->vio.data)); free_vio_components(&scrubber->vio); } @@ -2679,7 +2679,7 @@ static void finish_scrubbing(struct slab_scrubber *scrubber, int result) if (scrubber->high_priority_only) { scrubber->high_priority_only = false; - vdo_fail_completion(uds_forget(scrubber->vio.completion.parent), result); + vdo_fail_completion(vdo_forget(scrubber->vio.completion.parent), result); } else if (done && (atomic_add_return(-1, &allocator->depot->zones_to_scrub) == 0)) { /* All of our slabs were scrubbed, and we're the last allocator to finish. */ enum vdo_state prior_state = @@ -3382,7 +3382,7 @@ static void finish_loading_allocator(struct vdo_completion *completion) vdo_get_admin_state_code(&allocator->state); if (allocator->eraser != NULL) - dm_kcopyd_client_destroy(uds_forget(allocator->eraser)); + dm_kcopyd_client_destroy(vdo_forget(allocator->eraser)); if (operation == VDO_ADMIN_STATE_LOADING_FOR_RECOVERY) { void *context = @@ -3485,7 +3485,7 @@ static int get_slab_statuses(struct block_allocator *allocator, struct slab_status *statuses; struct slab_iterator iterator = get_slab_iterator(allocator); - result = uds_allocate(allocator->slab_count, struct slab_status, __func__, + result = vdo_allocate(allocator->slab_count, struct slab_status, __func__, &statuses); if (result != VDO_SUCCESS) return result; @@ -3552,7 +3552,7 @@ static int __must_check vdo_prepare_slabs_for_allocation(struct block_allocator register_slab_for_scrubbing(slab, high_priority); } - uds_free(slab_statuses); + vdo_free(slab_statuses); return VDO_SUCCESS; } @@ -3648,11 +3648,11 @@ static void free_slab(struct vdo_slab *slab) return; list_del(&slab->allocq_entry); - uds_free(uds_forget(slab->journal.block)); - uds_free(uds_forget(slab->journal.locks)); - uds_free(uds_forget(slab->counters)); - uds_free(uds_forget(slab->reference_blocks)); - uds_free(slab); + vdo_free(vdo_forget(slab->journal.block)); + vdo_free(vdo_forget(slab->journal.locks)); + vdo_free(vdo_forget(slab->counters)); + vdo_free(vdo_forget(slab->reference_blocks)); + vdo_free(slab); } static int initialize_slab_journal(struct vdo_slab *slab) @@ -3661,12 +3661,12 @@ static int initialize_slab_journal(struct vdo_slab *slab) const struct slab_config *slab_config = &slab->allocator->depot->slab_config; int result; - result = uds_allocate(slab_config->slab_journal_blocks, struct journal_lock, + result = vdo_allocate(slab_config->slab_journal_blocks, struct journal_lock, __func__, &journal->locks); if (result != VDO_SUCCESS) return result; - result = uds_allocate(VDO_BLOCK_SIZE, char, "struct packed_slab_journal_block", + result = vdo_allocate(VDO_BLOCK_SIZE, char, "struct packed_slab_journal_block", (char **) &journal->block); if (result != VDO_SUCCESS) return result; @@ -3722,7 +3722,7 @@ static int __must_check make_slab(physical_block_number_t slab_origin, struct vdo_slab *slab; int result; - result = uds_allocate(1, struct vdo_slab, __func__, &slab); + result = vdo_allocate(1, struct vdo_slab, __func__, &slab); if (result != VDO_SUCCESS) return result; @@ -3779,7 +3779,7 @@ static int allocate_slabs(struct slab_depot *depot, slab_count_t slab_count) physical_block_number_t slab_origin; int result; - result = uds_allocate(slab_count, struct vdo_slab *, + result = vdo_allocate(slab_count, struct vdo_slab *, "slab pointer array", &depot->new_slabs); if (result != VDO_SUCCESS) return result; @@ -3821,10 +3821,10 @@ void vdo_abandon_new_slabs(struct slab_depot *depot) return; for (i = depot->slab_count; i < depot->new_slab_count; i++) - free_slab(uds_forget(depot->new_slabs[i])); + free_slab(vdo_forget(depot->new_slabs[i])); depot->new_slab_count = 0; depot->new_size = 0; - uds_free(uds_forget(depot->new_slabs)); + vdo_free(vdo_forget(depot->new_slabs)); } /** @@ -3934,7 +3934,7 @@ static int initialize_slab_scrubber(struct block_allocator *allocator) char *journal_data; int result; - result = uds_allocate(VDO_BLOCK_SIZE * slab_journal_size, + result = vdo_allocate(VDO_BLOCK_SIZE * slab_journal_size, char, __func__, &journal_data); if (result != VDO_SUCCESS) return result; @@ -3945,7 +3945,7 @@ static int initialize_slab_scrubber(struct block_allocator *allocator) allocator, slab_journal_size, journal_data, &scrubber->vio); if (result != VDO_SUCCESS) { - uds_free(journal_data); + vdo_free(journal_data); return result; } @@ -3968,7 +3968,7 @@ static int __must_check initialize_slab_summary_block(struct block_allocator *al struct slab_summary_block *block = &allocator->summary_blocks[index]; int result; - result = uds_allocate(VDO_BLOCK_SIZE, char, __func__, &block->outgoing_entries); + result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &block->outgoing_entries); if (result != VDO_SUCCESS) return result; @@ -4024,7 +4024,7 @@ static int __must_check initialize_block_allocator(struct slab_depot *depot, if (result != VDO_SUCCESS) return result; - result = uds_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE, + result = vdo_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE, struct slab_summary_block, __func__, &allocator->summary_blocks); if (result != VDO_SUCCESS) @@ -4084,7 +4084,7 @@ static int allocate_components(struct slab_depot *depot, depot->summary_origin = summary_partition->offset; depot->hint_shift = vdo_get_slab_summary_hint_shift(depot->slab_size_shift); - result = uds_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES, + result = vdo_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES, struct slab_summary_entry, __func__, &depot->summary_entries); if (result != VDO_SUCCESS) @@ -4172,7 +4172,7 @@ int vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo, } slab_size_shift = ilog2(slab_size); - result = uds_allocate_extended(struct slab_depot, + result = vdo_allocate_extended(struct slab_depot, vdo->thread_config.physical_zone_count, struct block_allocator, __func__, &depot); if (result != VDO_SUCCESS) @@ -4205,10 +4205,10 @@ static void uninitialize_allocator_summary(struct block_allocator *allocator) for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { free_vio_components(&allocator->summary_blocks[i].vio); - uds_free(uds_forget(allocator->summary_blocks[i].outgoing_entries)); + vdo_free(vdo_forget(allocator->summary_blocks[i].outgoing_entries)); } - uds_free(uds_forget(allocator->summary_blocks)); + vdo_free(vdo_forget(allocator->summary_blocks)); } /** @@ -4228,25 +4228,25 @@ void vdo_free_slab_depot(struct slab_depot *depot) struct block_allocator *allocator = &depot->allocators[zone]; if (allocator->eraser != NULL) - dm_kcopyd_client_destroy(uds_forget(allocator->eraser)); + dm_kcopyd_client_destroy(vdo_forget(allocator->eraser)); uninitialize_allocator_summary(allocator); uninitialize_scrubber_vio(&allocator->scrubber); - free_vio_pool(uds_forget(allocator->vio_pool)); - vdo_free_priority_table(uds_forget(allocator->prioritized_slabs)); + free_vio_pool(vdo_forget(allocator->vio_pool)); + vdo_free_priority_table(vdo_forget(allocator->prioritized_slabs)); } if (depot->slabs != NULL) { slab_count_t i; for (i = 0; i < depot->slab_count; i++) - free_slab(uds_forget(depot->slabs[i])); + free_slab(vdo_forget(depot->slabs[i])); } - uds_free(uds_forget(depot->slabs)); - uds_free(uds_forget(depot->action_manager)); - uds_free(uds_forget(depot->summary_entries)); - uds_free(depot); + vdo_free(vdo_forget(depot->slabs)); + vdo_free(vdo_forget(depot->action_manager)); + vdo_free(vdo_forget(depot->summary_entries)); + vdo_free(depot); } /** @@ -4447,7 +4447,7 @@ static void finish_combining_zones(struct vdo_completion *completion) int result = completion->result; struct vdo_completion *parent = completion->parent; - free_vio(as_vio(uds_forget(completion))); + free_vio(as_vio(vdo_forget(completion))); vdo_fail_completion(parent, result); } @@ -4708,7 +4708,7 @@ static int finish_registration(void *context) struct slab_depot *depot = context; WRITE_ONCE(depot->slab_count, depot->new_slab_count); - uds_free(depot->slabs); + vdo_free(depot->slabs); depot->slabs = depot->new_slabs; depot->new_slabs = NULL; depot->new_slab_count = 0; diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h index fba293f9713e..f234853501ca 100644 --- a/drivers/md/dm-vdo/slab-depot.h +++ b/drivers/md/dm-vdo/slab-depot.h @@ -241,7 +241,7 @@ struct vdo_slab { /* The number of free blocks */ u32 free_blocks; /* The array of reference counts */ - vdo_refcount_t *counters; /* use uds_allocate() to align data ptr */ + vdo_refcount_t *counters; /* use vdo_allocate() to align data ptr */ /* The saved block pointer and array indexes for the free block search */ struct search_cursor search_cursor; diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index 244abc6ad848..ad7682784459 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -66,9 +66,9 @@ static int thread_starter(void *arg) mutex_lock(&thread_mutex); hlist_add_head(&thread->thread_links, &thread_list); mutex_unlock(&thread_mutex); - uds_register_allocating_thread(&allocating_thread, NULL); + vdo_register_allocating_thread(&allocating_thread, NULL); thread->thread_function(thread->thread_data); - uds_unregister_allocating_thread(); + vdo_unregister_allocating_thread(); complete(&thread->thread_done); return 0; } @@ -82,7 +82,7 @@ int vdo_create_thread(void (*thread_function)(void *), void *thread_data, struct thread *thread; int result; - result = uds_allocate(1, struct thread, __func__, &thread); + result = vdo_allocate(1, struct thread, __func__, &thread); if (result != UDS_SUCCESS) { uds_log_warning("Error allocating memory for %s", name); return result; @@ -114,7 +114,7 @@ int vdo_create_thread(void (*thread_function)(void *), void *thread_data, } if (IS_ERR(task)) { - uds_free(thread); + vdo_free(thread); return PTR_ERR(task); } @@ -130,5 +130,5 @@ void vdo_join_threads(struct thread *thread) mutex_lock(&thread_mutex); hlist_del(&thread->thread_links); mutex_unlock(&thread_mutex); - uds_free(thread); + vdo_free(thread); } diff --git a/drivers/md/dm-vdo/uds-sysfs.c b/drivers/md/dm-vdo/uds-sysfs.c index 2c4fb277ba38..d1d5a30b3717 100644 --- a/drivers/md/dm-vdo/uds-sysfs.c +++ b/drivers/md/dm-vdo/uds-sysfs.c @@ -35,7 +35,7 @@ static char *buffer_to_string(const char *buf, size_t length) { char *string; - if (uds_allocate(length + 1, char, __func__, &string) != UDS_SUCCESS) + if (vdo_allocate(length + 1, char, __func__, &string) != UDS_SUCCESS) return NULL; memcpy(string, buf, length); @@ -118,7 +118,7 @@ static ssize_t parameter_store(struct kobject *kobj, struct attribute *attr, return -ENOMEM; pa->store_string(string); - uds_free(string); + vdo_free(string); return length; } diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index 6baf319d79c6..ae62f260c5ec 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -134,13 +134,13 @@ static void start_vdo_request_queue(void *ptr) { struct vdo_thread *thread = vdo_get_work_queue_owner(vdo_get_current_work_queue()); - uds_register_allocating_thread(&thread->allocating_thread, + vdo_register_allocating_thread(&thread->allocating_thread, &thread->vdo->allocations_allowed); } static void finish_vdo_request_queue(void *ptr) { - uds_unregister_allocating_thread(); + vdo_unregister_allocating_thread(); } #ifdef MODULE @@ -172,10 +172,10 @@ static const struct vdo_work_queue_type cpu_q_type = { static void uninitialize_thread_config(struct thread_config *config) { - uds_free(uds_forget(config->logical_threads)); - uds_free(uds_forget(config->physical_threads)); - uds_free(uds_forget(config->hash_zone_threads)); - uds_free(uds_forget(config->bio_threads)); + vdo_free(vdo_forget(config->logical_threads)); + vdo_free(vdo_forget(config->physical_threads)); + vdo_free(vdo_forget(config->hash_zone_threads)); + vdo_free(vdo_forget(config->bio_threads)); memset(config, 0, sizeof(struct thread_config)); } @@ -214,28 +214,28 @@ static int __must_check initialize_thread_config(struct thread_count_config coun config->hash_zone_count = counts.hash_zones; } - result = uds_allocate(config->logical_zone_count, thread_id_t, + result = vdo_allocate(config->logical_zone_count, thread_id_t, "logical thread array", &config->logical_threads); if (result != VDO_SUCCESS) { uninitialize_thread_config(config); return result; } - result = uds_allocate(config->physical_zone_count, thread_id_t, + result = vdo_allocate(config->physical_zone_count, thread_id_t, "physical thread array", &config->physical_threads); if (result != VDO_SUCCESS) { uninitialize_thread_config(config); return result; } - result = uds_allocate(config->hash_zone_count, thread_id_t, + result = vdo_allocate(config->hash_zone_count, thread_id_t, "hash thread array", &config->hash_zone_threads); if (result != VDO_SUCCESS) { uninitialize_thread_config(config); return result; } - result = uds_allocate(config->bio_thread_count, thread_id_t, + result = vdo_allocate(config->bio_thread_count, thread_id_t, "bio thread array", &config->bio_threads); if (result != VDO_SUCCESS) { uninitialize_thread_config(config); @@ -276,14 +276,14 @@ static int __must_check read_geometry_block(struct vdo *vdo) char *block; int result; - result = uds_allocate(VDO_BLOCK_SIZE, u8, __func__, &block); + result = vdo_allocate(VDO_BLOCK_SIZE, u8, __func__, &block); if (result != VDO_SUCCESS) return result; result = create_metadata_vio(vdo, VIO_TYPE_GEOMETRY, VIO_PRIORITY_HIGH, NULL, block, &vio); if (result != VDO_SUCCESS) { - uds_free(block); + vdo_free(block); return result; } @@ -295,23 +295,23 @@ static int __must_check read_geometry_block(struct vdo *vdo) result = vio_reset_bio(vio, block, NULL, REQ_OP_READ, VDO_GEOMETRY_BLOCK_LOCATION); if (result != VDO_SUCCESS) { - free_vio(uds_forget(vio)); - uds_free(block); + free_vio(vdo_forget(vio)); + vdo_free(block); return result; } bio_set_dev(vio->bio, vdo_get_backing_device(vdo)); submit_bio_wait(vio->bio); result = blk_status_to_errno(vio->bio->bi_status); - free_vio(uds_forget(vio)); + free_vio(vdo_forget(vio)); if (result != 0) { uds_log_error_strerror(result, "synchronous read failed"); - uds_free(block); + vdo_free(block); return -EIO; } result = vdo_parse_geometry_block((u8 *) block, &vdo->geometry); - uds_free(block); + vdo_free(block); return result; } @@ -500,7 +500,7 @@ static int initialize_vdo(struct vdo *vdo, struct device_config *config, config->thread_counts.hash_zones, vdo->thread_config.thread_count); /* Compression context storage */ - result = uds_allocate(config->thread_counts.cpu_threads, char *, "LZ4 context", + result = vdo_allocate(config->thread_counts.cpu_threads, char *, "LZ4 context", &vdo->compression_context); if (result != VDO_SUCCESS) { *reason = "cannot allocate LZ4 context"; @@ -508,7 +508,7 @@ static int initialize_vdo(struct vdo *vdo, struct device_config *config, } for (i = 0; i < config->thread_counts.cpu_threads; i++) { - result = uds_allocate(LZ4_MEM_COMPRESS, char, "LZ4 context", + result = vdo_allocate(LZ4_MEM_COMPRESS, char, "LZ4 context", &vdo->compression_context[i]); if (result != VDO_SUCCESS) { *reason = "cannot allocate LZ4 context"; @@ -544,7 +544,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason, /* Initialize with a generic failure reason to prevent returning garbage. */ *reason = "Unspecified error"; - result = uds_allocate(1, struct vdo, __func__, &vdo); + result = vdo_allocate(1, struct vdo, __func__, &vdo); if (result != UDS_SUCCESS) { *reason = "Cannot allocate VDO"; return result; @@ -562,7 +562,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason, snprintf(vdo->thread_name_prefix, sizeof(vdo->thread_name_prefix), "%s%u", MODULE_NAME, instance); BUG_ON(vdo->thread_name_prefix[0] == '\0'); - result = uds_allocate(vdo->thread_config.thread_count, + result = vdo_allocate(vdo->thread_config.thread_count, struct vdo_thread, __func__, &vdo->threads); if (result != VDO_SUCCESS) { *reason = "Cannot allocate thread structures"; @@ -650,16 +650,16 @@ static void free_listeners(struct vdo_thread *thread) { struct read_only_listener *listener, *next; - for (listener = uds_forget(thread->listeners); listener != NULL; listener = next) { - next = uds_forget(listener->next); - uds_free(listener); + for (listener = vdo_forget(thread->listeners); listener != NULL; listener = next) { + next = vdo_forget(listener->next); + vdo_free(listener); } } static void uninitialize_super_block(struct vdo_super_block *super_block) { free_vio_components(&super_block->vio); - uds_free(super_block->buffer); + vdo_free(super_block->buffer); } /** @@ -701,36 +701,36 @@ void vdo_destroy(struct vdo *vdo) finish_vdo(vdo); unregister_vdo(vdo); free_data_vio_pool(vdo->data_vio_pool); - vdo_free_io_submitter(uds_forget(vdo->io_submitter)); - vdo_free_flusher(uds_forget(vdo->flusher)); - vdo_free_packer(uds_forget(vdo->packer)); - vdo_free_recovery_journal(uds_forget(vdo->recovery_journal)); - vdo_free_slab_depot(uds_forget(vdo->depot)); + vdo_free_io_submitter(vdo_forget(vdo->io_submitter)); + vdo_free_flusher(vdo_forget(vdo->flusher)); + vdo_free_packer(vdo_forget(vdo->packer)); + vdo_free_recovery_journal(vdo_forget(vdo->recovery_journal)); + vdo_free_slab_depot(vdo_forget(vdo->depot)); vdo_uninitialize_layout(&vdo->layout); vdo_uninitialize_layout(&vdo->next_layout); if (vdo->partition_copier) - dm_kcopyd_client_destroy(uds_forget(vdo->partition_copier)); + dm_kcopyd_client_destroy(vdo_forget(vdo->partition_copier)); uninitialize_super_block(&vdo->super_block); - vdo_free_block_map(uds_forget(vdo->block_map)); - vdo_free_hash_zones(uds_forget(vdo->hash_zones)); - vdo_free_physical_zones(uds_forget(vdo->physical_zones)); - vdo_free_logical_zones(uds_forget(vdo->logical_zones)); + vdo_free_block_map(vdo_forget(vdo->block_map)); + vdo_free_hash_zones(vdo_forget(vdo->hash_zones)); + vdo_free_physical_zones(vdo_forget(vdo->physical_zones)); + vdo_free_logical_zones(vdo_forget(vdo->logical_zones)); if (vdo->threads != NULL) { for (i = 0; i < vdo->thread_config.thread_count; i++) { free_listeners(&vdo->threads[i]); - vdo_free_work_queue(uds_forget(vdo->threads[i].queue)); + vdo_free_work_queue(vdo_forget(vdo->threads[i].queue)); } - uds_free(uds_forget(vdo->threads)); + vdo_free(vdo_forget(vdo->threads)); } uninitialize_thread_config(&vdo->thread_config); if (vdo->compression_context != NULL) { for (i = 0; i < vdo->device_config->thread_counts.cpu_threads; i++) - uds_free(uds_forget(vdo->compression_context[i])); + vdo_free(vdo_forget(vdo->compression_context[i])); - uds_free(uds_forget(vdo->compression_context)); + vdo_free(vdo_forget(vdo->compression_context)); } /* @@ -738,7 +738,7 @@ void vdo_destroy(struct vdo *vdo) * the count goes to zero the VDO object will be freed as a side effect. */ if (!vdo->sysfs_added) - uds_free(vdo); + vdo_free(vdo); else kobject_put(&vdo->vdo_directory); } @@ -747,7 +747,7 @@ static int initialize_super_block(struct vdo *vdo, struct vdo_super_block *super { int result; - result = uds_allocate(VDO_BLOCK_SIZE, char, "encoded super block", + result = vdo_allocate(VDO_BLOCK_SIZE, char, "encoded super block", (char **) &vdo->super_block.buffer); if (result != VDO_SUCCESS) return result; @@ -769,7 +769,7 @@ static void finish_reading_super_block(struct vdo_completion *completion) struct vdo_super_block *super_block = container_of(as_vio(completion), struct vdo_super_block, vio); - vdo_continue_completion(uds_forget(completion->parent), + vdo_continue_completion(vdo_forget(completion->parent), vdo_decode_super_block(super_block->buffer)); } @@ -965,7 +965,7 @@ static void record_vdo(struct vdo *vdo) */ static void continue_super_block_parent(struct vdo_completion *completion) { - vdo_continue_completion(uds_forget(completion->parent), completion->result); + vdo_continue_completion(vdo_forget(completion->parent), completion->result); } /** @@ -1055,7 +1055,7 @@ int vdo_register_read_only_listener(struct vdo *vdo, void *listener, if (result != VDO_SUCCESS) return result; - result = uds_allocate(1, struct read_only_listener, __func__, + result = vdo_allocate(1, struct read_only_listener, __func__, &read_only_listener); if (result != VDO_SUCCESS) return result; @@ -1184,7 +1184,7 @@ static void finish_entering_read_only_mode(struct vdo_completion *completion) spin_unlock(¬ifier->lock); if (notifier->waiter != NULL) - vdo_continue_completion(uds_forget(notifier->waiter), + vdo_continue_completion(vdo_forget(notifier->waiter), completion->result); } @@ -1621,7 +1621,7 @@ static void get_vdo_statistics(const struct vdo *vdo, struct vdo_statistics *sta copy_bio_stat(&stats->bios_acknowledged_partial, &vdo->stats.bios_acknowledged_partial); stats->bios_in_progress = subtract_bio_stats(stats->bios_in, stats->bios_acknowledged); - uds_get_memory_stats(&stats->memory_usage.bytes_used, + vdo_get_memory_stats(&stats->memory_usage.bytes_used, &stats->memory_usage.peak_bytes_used); } diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index 4832ea46551f..83c36f7590de 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -52,7 +52,7 @@ static int create_multi_block_bio(block_count_t size, struct bio **bio_ptr) struct bio *bio = NULL; int result; - result = uds_allocate_extended(struct bio, size + 1, struct bio_vec, + result = vdo_allocate_extended(struct bio, size + 1, struct bio_vec, "bio", &bio); if (result != VDO_SUCCESS) return result; @@ -72,7 +72,7 @@ void vdo_free_bio(struct bio *bio) return; bio_uninit(bio); - uds_free(uds_forget(bio)); + vdo_free(vdo_forget(bio)); } int allocate_vio_components(struct vdo *vdo, enum vio_type vio_type, @@ -129,7 +129,7 @@ int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, * Metadata vios should use direct allocation and not use the buffer pool, which is * reserved for submissions from the linux block layer. */ - result = uds_allocate(1, struct vio, __func__, &vio); + result = vdo_allocate(1, struct vio, __func__, &vio); if (result != VDO_SUCCESS) { uds_log_error("metadata vio allocation failure %d", result); return result; @@ -138,7 +138,7 @@ int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, result = allocate_vio_components(vdo, vio_type, priority, parent, block_count, data, vio); if (result != VDO_SUCCESS) { - uds_free(vio); + vdo_free(vio); return result; } @@ -156,7 +156,7 @@ void free_vio_components(struct vio *vio) return; BUG_ON(is_data_vio(vio)); - vdo_free_bio(uds_forget(vio->bio)); + vdo_free_bio(vdo_forget(vio->bio)); } /** @@ -166,7 +166,7 @@ void free_vio_components(struct vio *vio) void free_vio(struct vio *vio) { free_vio_components(vio); - uds_free(vio); + vdo_free(vio); } /* Set bio properties for a VDO read or write. */ @@ -316,7 +316,7 @@ int make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t thread_id, char *ptr; int result; - result = uds_allocate_extended(struct vio_pool, pool_size, struct pooled_vio, + result = vdo_allocate_extended(struct vio_pool, pool_size, struct pooled_vio, __func__, &pool); if (result != VDO_SUCCESS) return result; @@ -325,7 +325,7 @@ int make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t thread_id, INIT_LIST_HEAD(&pool->available); INIT_LIST_HEAD(&pool->busy); - result = uds_allocate(pool_size * VDO_BLOCK_SIZE, char, + result = vdo_allocate(pool_size * VDO_BLOCK_SIZE, char, "VIO pool buffer", &pool->buffer); if (result != VDO_SUCCESS) { free_vio_pool(pool); @@ -380,8 +380,8 @@ void free_vio_pool(struct vio_pool *pool) ASSERT_LOG_ONLY(pool->size == 0, "VIO pool must not have missing entries when being freed"); - uds_free(uds_forget(pool->buffer)); - uds_free(pool); + vdo_free(vdo_forget(pool->buffer)); + vdo_free(pool); } /** From 8f89115efc1e8f34dd3182e2f3f7064225b52762 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 11:17:00 -0600 Subject: [PATCH 0942/1038] dm vdo memory-alloc: rename vdo_do_allocation to __vdo_do_allocation __vdo_do_allocation shouldn't be used outside of memory-alloc.h, so add hidden prefix. Also, tabify the vdo_allocate_extended macro. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/memory-alloc.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/md/dm-vdo/memory-alloc.h b/drivers/md/dm-vdo/memory-alloc.h index 3f27dd722a2d..1401843db5e0 100644 --- a/drivers/md/dm-vdo/memory-alloc.h +++ b/drivers/md/dm-vdo/memory-alloc.h @@ -37,8 +37,8 @@ int __must_check vdo_allocate_memory(size_t size, size_t align, const char *what * * Return: UDS_SUCCESS or an error code */ -static inline int vdo_do_allocation(size_t count, size_t size, size_t extra, - size_t align, const char *what, void *ptr) +static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra, + size_t align, const char *what, void *ptr) { size_t total_size = count * size + extra; @@ -68,7 +68,7 @@ static inline int vdo_do_allocation(size_t count, size_t size, size_t extra, * Return: UDS_SUCCESS or an error code */ #define vdo_allocate(COUNT, TYPE, WHAT, PTR) \ - vdo_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) + __vdo_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) /* * Allocate one object of an indicated type, followed by one or more elements of a second type, @@ -83,18 +83,18 @@ static inline int vdo_do_allocation(size_t count, size_t size, size_t extra, * * Return: UDS_SUCCESS or an error code */ -#define vdo_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ - __extension__({ \ - int _result; \ - TYPE1 **_ptr = (PTR); \ - BUILD_BUG_ON(__alignof__(TYPE1) < __alignof__(TYPE2)); \ - _result = vdo_do_allocation(COUNT, \ - sizeof(TYPE2), \ - sizeof(TYPE1), \ - __alignof__(TYPE1), \ - WHAT, \ - _ptr); \ - _result; \ +#define vdo_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ + __extension__({ \ + int _result; \ + TYPE1 **_ptr = (PTR); \ + BUILD_BUG_ON(__alignof__(TYPE1) < __alignof__(TYPE2)); \ + _result = __vdo_do_allocation(COUNT, \ + sizeof(TYPE2), \ + sizeof(TYPE1), \ + __alignof__(TYPE1), \ + WHAT, \ + _ptr); \ + _result; \ }) /* From ee8f6ec1b16044510a82eb9a3f7059a35d4fb09a Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Tue, 27 Feb 2024 19:22:59 -0500 Subject: [PATCH 0943/1038] dm vdo errors: remove unused error codes Also define VDO_SUCCESS in a more central location, and rename error block constants for clarity. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/errors.c | 4 ---- drivers/md/dm-vdo/errors.h | 13 +++---------- drivers/md/dm-vdo/status-codes.c | 10 ---------- drivers/md/dm-vdo/status-codes.h | 32 +++++--------------------------- 4 files changed, 8 insertions(+), 51 deletions(-) diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c index dc1f0533bd7a..df2498553312 100644 --- a/drivers/md/dm-vdo/errors.c +++ b/drivers/md/dm-vdo/errors.c @@ -58,13 +58,9 @@ static const struct error_info error_list[] = { { "UDS_DUPLICATE_NAME", "Attempt to enter the same name into a delta index twice" }, { "UDS_ASSERTION_FAILED", "Assertion failed" }, { "UDS_QUEUED", "Request queued" }, - { "UDS_BUFFER_ERROR", "Buffer error" }, - { "UDS_NO_DIRECTORY", "Expected directory is missing" }, { "UDS_ALREADY_REGISTERED", "Error range already registered" }, { "UDS_OUT_OF_RANGE", "Cannot access data outside specified limits" }, - { "UDS_EMODULE_LOAD", "Could not load modules" }, { "UDS_DISABLED", "UDS library context is disabled" }, - { "UDS_UNKNOWN_ERROR", "Unknown error" }, { "UDS_UNSUPPORTED_VERSION", "Unsupported version" }, { "UDS_CORRUPT_DATA", "Some index structure is corrupt" }, { "UDS_NO_INDEX", "No index found" }, diff --git a/drivers/md/dm-vdo/errors.h b/drivers/md/dm-vdo/errors.h index cf15d7243204..c6c085b26a0e 100644 --- a/drivers/md/dm-vdo/errors.h +++ b/drivers/md/dm-vdo/errors.h @@ -9,12 +9,13 @@ #include #include -/* Custom error codes and error-related utilities for UDS */ +/* Custom error codes and error-related utilities */ +#define VDO_SUCCESS 0 /* Valid status codes for internal UDS functions. */ enum uds_status_codes { /* Successful return */ - UDS_SUCCESS = 0, + UDS_SUCCESS = VDO_SUCCESS, /* Used as a base value for reporting internal errors */ UDS_ERROR_CODE_BASE = 1024, /* Index overflow */ @@ -29,20 +30,12 @@ enum uds_status_codes { UDS_ASSERTION_FAILED, /* A request has been queued for later processing (not an error) */ UDS_QUEUED, - /* A problem has occurred with a buffer */ - UDS_BUFFER_ERROR, - /* No directory was found where one was expected */ - UDS_NO_DIRECTORY, /* This error range has already been registered */ UDS_ALREADY_REGISTERED, /* Attempt to read or write data outside the valid range */ UDS_OUT_OF_RANGE, - /* Could not load modules */ - UDS_EMODULE_LOAD, /* The index session is disabled */ UDS_DISABLED, - /* Unknown error */ - UDS_UNKNOWN_ERROR, /* The index configuration or volume format is no longer supported */ UDS_UNSUPPORTED_VERSION, /* Some index structure is corrupt */ diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index efba1ead0aca..92c42b8bbb8b 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -15,22 +15,16 @@ const struct error_info vdo_status_list[] = { { "VDO_OUT_OF_RANGE", "Out of range" }, { "VDO_REF_COUNT_INVALID", "Reference count would become invalid" }, { "VDO_NO_SPACE", "Out of space" }, - { "VDO_UNEXPECTED_EOF", "Unexpected EOF on block read" }, { "VDO_BAD_CONFIGURATION", "Bad configuration option" }, - { "VDO_SOCKET_ERROR", "Socket error" }, - { "VDO_BAD_ALIGNMENT", "Mis-aligned block reference" }, { "VDO_COMPONENT_BUSY", "Prior operation still in progress" }, { "VDO_BAD_PAGE", "Corrupt or incorrect page" }, { "VDO_UNSUPPORTED_VERSION", "Unsupported component version" }, { "VDO_INCORRECT_COMPONENT", "Component id mismatch in decoder" }, { "VDO_PARAMETER_MISMATCH", "Parameters have conflicting values" }, - { "VDO_BLOCK_SIZE_TOO_SMALL", "The block size is too small" }, { "VDO_UNKNOWN_PARTITION", "No partition exists with a given id" }, { "VDO_PARTITION_EXISTS", "A partition already exists with a given id" }, - { "VDO_NOT_READ_ONLY", "The device is not in read-only mode" }, { "VDO_INCREMENT_TOO_SMALL", "Physical block growth of too few blocks" }, { "VDO_CHECKSUM_MISMATCH", "Incorrect checksum" }, - { "VDO_RECOVERY_JOURNAL_FULL", "The recovery journal is full" }, { "VDO_LOCK_ERROR", "A lock is held incorrectly" }, { "VDO_READ_ONLY", "The device is in read-only mode" }, { "VDO_SHUTTING_DOWN", "The device is shutting down" }, @@ -38,11 +32,7 @@ const struct error_info vdo_status_list[] = { { "VDO_TOO_MANY_SLABS", "Exceeds maximum number of slabs supported" }, { "VDO_INVALID_FRAGMENT", "Compressed block fragment is invalid" }, { "VDO_RETRY_AFTER_REBUILD", "Retry operation after rebuilding finishes" }, - { "VDO_UNKNOWN_COMMAND", "The extended command is not known" }, - { "VDO_COMMAND_ERROR", "Bad extended command parameters" }, - { "VDO_CANNOT_DETERMINE_SIZE", "Cannot determine config sizes to fit" }, { "VDO_BAD_MAPPING", "Invalid page mapping" }, - { "VDO_READ_CACHE_BUSY", "Read cache has no free slots" }, { "VDO_BIO_CREATION_FAILED", "Bio creation failed" }, { "VDO_BAD_MAGIC", "Bad magic number" }, { "VDO_BAD_NONCE", "Bad nonce" }, diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h index 5d1e8bbe54b4..eb847def8eb4 100644 --- a/drivers/md/dm-vdo/status-codes.h +++ b/drivers/md/dm-vdo/status-codes.h @@ -9,17 +9,15 @@ #include "errors.h" enum { - UDS_BLOCK_SIZE = UDS_ERROR_CODE_BLOCK_END - UDS_ERROR_CODE_BASE, - VDO_BLOCK_START = UDS_ERROR_CODE_BLOCK_END, - VDO_BLOCK_END = VDO_BLOCK_START + UDS_BLOCK_SIZE, + UDS_ERRORS_BLOCK_SIZE = UDS_ERROR_CODE_BLOCK_END - UDS_ERROR_CODE_BASE, + VDO_ERRORS_BLOCK_START = UDS_ERROR_CODE_BLOCK_END, + VDO_ERRORS_BLOCK_END = VDO_ERRORS_BLOCK_START + UDS_ERRORS_BLOCK_SIZE, }; /* VDO-specific status codes. */ enum vdo_status_codes { - /* successful result */ - VDO_SUCCESS = UDS_SUCCESS, /* base of all VDO errors */ - VDO_STATUS_CODE_BASE = VDO_BLOCK_START, + VDO_STATUS_CODE_BASE = VDO_ERRORS_BLOCK_START, /* we haven't written this yet */ VDO_NOT_IMPLEMENTED = VDO_STATUS_CODE_BASE, /* input out of range */ @@ -28,14 +26,8 @@ enum vdo_status_codes { VDO_REF_COUNT_INVALID, /* a free block could not be allocated */ VDO_NO_SPACE, - /* unexpected EOF on block read */ - VDO_UNEXPECTED_EOF, /* improper or missing configuration option */ VDO_BAD_CONFIGURATION, - /* socket opening or binding problem */ - VDO_SOCKET_ERROR, - /* read or write on non-aligned offset */ - VDO_BAD_ALIGNMENT, /* prior operation still in progress */ VDO_COMPONENT_BUSY, /* page contents incorrect or corrupt data */ @@ -46,20 +38,14 @@ enum vdo_status_codes { VDO_INCORRECT_COMPONENT, /* parameters have conflicting values */ VDO_PARAMETER_MISMATCH, - /* the block size is too small */ - VDO_BLOCK_SIZE_TOO_SMALL, /* no partition exists with a given id */ VDO_UNKNOWN_PARTITION, /* a partition already exists with a given id */ VDO_PARTITION_EXISTS, - /* the VDO is not in read-only mode */ - VDO_NOT_READ_ONLY, /* physical block growth of too few blocks */ VDO_INCREMENT_TOO_SMALL, /* incorrect checksum */ VDO_CHECKSUM_MISMATCH, - /* the recovery journal is full */ - VDO_RECOVERY_JOURNAL_FULL, /* a lock is held incorrectly */ VDO_LOCK_ERROR, /* the VDO is in read-only mode */ @@ -74,16 +60,8 @@ enum vdo_status_codes { VDO_INVALID_FRAGMENT, /* action is unsupported while rebuilding */ VDO_RETRY_AFTER_REBUILD, - /* the extended command is not known */ - VDO_UNKNOWN_COMMAND, - /* bad extended command parameters */ - VDO_COMMAND_ERROR, - /* cannot determine sizes to fit */ - VDO_CANNOT_DETERMINE_SIZE, /* a block map entry is invalid */ VDO_BAD_MAPPING, - /* read cache has no free slots */ - VDO_READ_CACHE_BUSY, /* bio_add_page failed */ VDO_BIO_CREATION_FAILED, /* bad magic number */ @@ -98,7 +76,7 @@ enum vdo_status_codes { VDO_CANT_ADD_SYSFS_NODE, /* one more than last error code */ VDO_STATUS_CODE_LAST, - VDO_STATUS_CODE_BLOCK_END = VDO_BLOCK_END + VDO_STATUS_CODE_BLOCK_END = VDO_ERRORS_BLOCK_END }; extern const struct error_info vdo_status_list[]; From 97d3380396b4cbf302912a0e818fa074afdc0db5 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 11:37:05 -0600 Subject: [PATCH 0944/1038] dm vdo memory-alloc: return VDO_SUCCESS on success Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/memory-alloc.c | 24 ++++++++++++------------ drivers/md/dm-vdo/memory-alloc.h | 8 ++++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index 8d5df3e45a24..dd5acc582fb3 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -194,7 +194,7 @@ static inline bool use_kmalloc(size_t size) * @what: What is being allocated (for error logging) * @ptr: A pointer to hold the allocated memory * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code */ int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) { @@ -216,12 +216,12 @@ int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) unsigned long start_time; void *p = NULL; - if (ptr == NULL) - return UDS_INVALID_ARGUMENT; + if (unlikely(ptr == NULL)) + return -EINVAL; if (size == 0) { *((void **) ptr) = NULL; - return UDS_SUCCESS; + return VDO_SUCCESS; } if (allocations_restricted) @@ -245,7 +245,7 @@ int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) } else { struct vmalloc_block_info *block; - if (vdo_allocate(1, struct vmalloc_block_info, __func__, &block) == UDS_SUCCESS) { + if (vdo_allocate(1, struct vmalloc_block_info, __func__, &block) == VDO_SUCCESS) { /* * It is possible for __vmalloc to fail to allocate memory because there * are no pages available. A short sleep may allow the page reclaimer @@ -290,7 +290,7 @@ int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) } *((void **) ptr) = p; - return UDS_SUCCESS; + return VDO_SUCCESS; } /* @@ -335,7 +335,7 @@ void vdo_free(void *ptr) * @what: What is being allocated (for error logging) * @new_ptr: A pointer to hold the reallocated pointer * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code */ int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what, void *new_ptr) @@ -345,11 +345,11 @@ int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *w if (size == 0) { vdo_free(ptr); *(void **) new_ptr = NULL; - return UDS_SUCCESS; + return VDO_SUCCESS; } result = vdo_allocate(size, char, what, new_ptr); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; if (ptr != NULL) { @@ -360,7 +360,7 @@ int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *w vdo_free(ptr); } - return UDS_SUCCESS; + return VDO_SUCCESS; } int vdo_duplicate_string(const char *string, const char *what, char **new_string) @@ -369,12 +369,12 @@ int vdo_duplicate_string(const char *string, const char *what, char **new_string u8 *dup; result = vdo_allocate(strlen(string) + 1, u8, what, &dup); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; memcpy(dup, string, strlen(string) + 1); *new_string = dup; - return UDS_SUCCESS; + return VDO_SUCCESS; } void vdo_memory_init(void) diff --git a/drivers/md/dm-vdo/memory-alloc.h b/drivers/md/dm-vdo/memory-alloc.h index 1401843db5e0..0093d9f940d9 100644 --- a/drivers/md/dm-vdo/memory-alloc.h +++ b/drivers/md/dm-vdo/memory-alloc.h @@ -35,7 +35,7 @@ int __must_check vdo_allocate_memory(size_t size, size_t align, const char *what * @what: What is being allocated (for error logging) * @ptr: A pointer to hold the allocated memory * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code */ static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra, size_t align, const char *what, void *ptr) @@ -65,7 +65,7 @@ static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra, * @WHAT: What is being allocated (for error logging) * @PTR: A pointer to hold the allocated memory * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code */ #define vdo_allocate(COUNT, TYPE, WHAT, PTR) \ __vdo_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) @@ -81,7 +81,7 @@ static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra, * @WHAT: What is being allocated (for error logging) * @PTR: A pointer to hold the allocated memory * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code */ #define vdo_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ __extension__({ \ @@ -105,7 +105,7 @@ static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra, * @what: What is being allocated (for error logging) * @ptr: A pointer to hold the allocated memory * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code */ static inline int __must_check vdo_allocate_cache_aligned(size_t size, const char *what, void *ptr) { From 2de70388b3751e8cd6727441330978e69a578e0c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 12:06:53 -0600 Subject: [PATCH 0945/1038] dm vdo: check for VDO_SUCCESS return value from memory-alloc functions VDO_SUCCESS and UDS_SUCCESS were used interchangably, update all callers of VDO's memory-alloc functions to consistently check for VDO_SUCCESS. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 6 +++--- drivers/md/dm-vdo/data-vio.c | 2 +- drivers/md/dm-vdo/dm-vdo-target.c | 18 +++++++++--------- drivers/md/dm-vdo/encodings.c | 2 +- drivers/md/dm-vdo/funnel-queue.c | 2 +- drivers/md/dm-vdo/funnel-workqueue.c | 6 +++--- drivers/md/dm-vdo/indexer/chapter-index.c | 2 +- drivers/md/dm-vdo/indexer/config.c | 2 +- drivers/md/dm-vdo/indexer/delta-index.c | 10 +++++----- .../md/dm-vdo/indexer/funnel-requestqueue.c | 2 +- drivers/md/dm-vdo/indexer/geometry.c | 2 +- drivers/md/dm-vdo/indexer/index-layout.c | 18 +++++++++--------- drivers/md/dm-vdo/indexer/index-page-map.c | 8 ++++---- drivers/md/dm-vdo/indexer/index-session.c | 2 +- drivers/md/dm-vdo/indexer/index.c | 12 ++++++------ drivers/md/dm-vdo/indexer/io-factory.c | 6 +++--- drivers/md/dm-vdo/indexer/open-chapter.c | 4 ++-- drivers/md/dm-vdo/indexer/radix-sort.c | 2 +- drivers/md/dm-vdo/indexer/sparse-cache.c | 8 ++++---- drivers/md/dm-vdo/indexer/volume-index.c | 6 +++--- drivers/md/dm-vdo/indexer/volume.c | 14 +++++++------- drivers/md/dm-vdo/int-map.c | 18 +++++++++--------- drivers/md/dm-vdo/io-submitter.c | 2 +- drivers/md/dm-vdo/message-stats.c | 2 +- drivers/md/dm-vdo/slab-depot.c | 2 +- drivers/md/dm-vdo/thread-utils.c | 2 +- drivers/md/dm-vdo/uds-sysfs.c | 2 +- drivers/md/dm-vdo/vdo.c | 2 +- 28 files changed, 82 insertions(+), 82 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index b09974ad41d2..c4719fb30f86 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -223,11 +223,11 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache) result = vdo_allocate(cache->page_count, struct page_info, "page infos", &cache->infos); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_int_map_create(cache->page_count, &cache->page_map); @@ -2874,7 +2874,7 @@ int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical result = vdo_allocate_extended(struct block_map, vdo->thread_config.logical_zone_count, struct block_map_zone, __func__, &map); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; map->vdo = vdo; diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index dcdd767e40e5..3d5054e61330 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -847,7 +847,7 @@ int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, result = vdo_allocate_extended(struct data_vio_pool, pool_size, struct data_vio, __func__, &pool); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; ASSERT_LOG_ONLY((discard_limit <= pool_size), diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 86c30fbd75ca..90ba379f8d3e 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -280,7 +280,7 @@ static int split_string(const char *string, char separator, char ***substring_ar result = vdo_allocate(substring_count + 1, char *, "string-splitting array", &substrings); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; for (s = string; *s != 0; s++) { @@ -289,7 +289,7 @@ static int split_string(const char *string, char separator, char ***substring_ar result = vdo_allocate(length + 1, char, "split string", &substrings[current_substring]); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { free_string_array(substrings); return result; } @@ -310,7 +310,7 @@ static int split_string(const char *string, char separator, char ***substring_ar result = vdo_allocate(length + 1, char, "split string", &substrings[current_substring]); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { free_string_array(substrings); return result; } @@ -1527,7 +1527,7 @@ static size_t get_bit_array_size(unsigned int bit_count) * Since the array is initially NULL, this also initializes the array the first time we allocate an * instance number. * - * Return: UDS_SUCCESS or an error code from the allocation + * Return: VDO_SUCCESS or an error code from the allocation */ static int grow_bit_array(void) { @@ -1540,19 +1540,19 @@ static int grow_bit_array(void) get_bit_array_size(instances.bit_count), get_bit_array_size(new_count), "instance number bit array", &new_words); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; instances.bit_count = new_count; instances.words = new_words; - return UDS_SUCCESS; + return VDO_SUCCESS; } /** * allocate_instance() - Allocate an instance number. * @instance_ptr: A point to hold the instance number * - * Return: UDS_SUCCESS or an error code + * Return: VDO_SUCCESS or an error code * * This function must be called while holding the instances lock. */ @@ -1564,7 +1564,7 @@ static int allocate_instance(unsigned int *instance_ptr) /* If there are no unallocated instances, grow the bit array. */ if (instances.count >= instances.bit_count) { result = grow_bit_array(); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; } @@ -1587,7 +1587,7 @@ static int allocate_instance(unsigned int *instance_ptr) instances.count++; instances.next = instance + 1; *instance_ptr = instance; - return UDS_SUCCESS; + return VDO_SUCCESS; } static int construct_new_vdo_registered(struct dm_target *ti, unsigned int argc, diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index 56d94339d2af..a97771fe0a43 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -800,7 +800,7 @@ static int allocate_partition(struct layout *layout, u8 id, int result; result = vdo_allocate(1, struct partition, __func__, &partition); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; partition->id = id; diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c index 67f7b52ecc86..ce0e801fd955 100644 --- a/drivers/md/dm-vdo/funnel-queue.c +++ b/drivers/md/dm-vdo/funnel-queue.c @@ -15,7 +15,7 @@ int uds_make_funnel_queue(struct funnel_queue **queue_ptr) struct funnel_queue *queue; result = vdo_allocate(1, struct funnel_queue, "funnel queue", &queue); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; /* diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c index ebf8dce67086..a88f5c93eae5 100644 --- a/drivers/md/dm-vdo/funnel-workqueue.c +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -324,7 +324,7 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na VDO_WORK_Q_MAX_PRIORITY); result = vdo_allocate(1, struct simple_work_queue, "simple work queue", &queue); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; queue->private = private; @@ -401,12 +401,12 @@ int vdo_make_work_queue(const char *thread_name_prefix, const char *name, result = vdo_allocate(1, struct round_robin_work_queue, "round-robin work queue", &queue); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_allocate(thread_count, struct simple_work_queue *, "subordinate work queues", &queue->service_queues); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { vdo_free(queue); return result; } diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c index 9477150362ae..68d86028dbb7 100644 --- a/drivers/md/dm-vdo/indexer/chapter-index.c +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -21,7 +21,7 @@ int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, struct open_chapter_index *index; result = vdo_allocate(1, struct open_chapter_index, "open chapter index", &index); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; /* diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c index cd20ee8b9a02..5da39043b9ae 100644 --- a/drivers/md/dm-vdo/indexer/config.c +++ b/drivers/md/dm-vdo/indexer/config.c @@ -326,7 +326,7 @@ int uds_make_configuration(const struct uds_parameters *params, return result; result = vdo_allocate(1, struct uds_configuration, __func__, &config); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = uds_make_index_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter, diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c index 11f7b85b6710..5bba9a48c5a0 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -312,18 +312,18 @@ static int initialize_delta_zone(struct delta_zone *delta_zone, size_t size, int result; result = vdo_allocate(size, u8, "delta list", &delta_zone->memory); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_allocate(list_count + 2, u64, "delta list temp", &delta_zone->new_offsets); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; /* Allocate the delta lists. */ result = vdo_allocate(list_count + 2, struct delta_list, "delta lists", &delta_zone->delta_lists); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; compute_coding_constants(mean_delta, &delta_zone->min_bits, @@ -354,7 +354,7 @@ int uds_initialize_delta_index(struct delta_index *delta_index, unsigned int zon result = vdo_allocate(zone_count, struct delta_zone, "Delta Index Zones", &delta_index->delta_zones); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; delta_index->zone_count = zone_count; @@ -1048,7 +1048,7 @@ int uds_finish_restoring_delta_index(struct delta_index *delta_index, u8 *data; result = vdo_allocate(DELTA_LIST_MAX_BYTE_COUNT, u8, __func__, &data); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; for (z = 0; z < reader_count; z++) { diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c index 95a402ec31c9..eee7b980960f 100644 --- a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c +++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c @@ -199,7 +199,7 @@ int uds_make_request_queue(const char *queue_name, struct uds_request_queue *queue; result = vdo_allocate(1, struct uds_request_queue, __func__, &queue); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; queue->processor = processor; diff --git a/drivers/md/dm-vdo/indexer/geometry.c b/drivers/md/dm-vdo/indexer/geometry.c index c735e6cb4425..c0575612e820 100644 --- a/drivers/md/dm-vdo/indexer/geometry.c +++ b/drivers/md/dm-vdo/indexer/geometry.c @@ -62,7 +62,7 @@ int uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, struct index_geometry *geometry; result = vdo_allocate(1, struct index_geometry, "geometry", &geometry); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; geometry->bytes_per_page = bytes_per_page; diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index c1bcff03cc55..01e0db4184aa 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -487,7 +487,7 @@ static int __must_check make_index_save_region_table(struct index_save_layout *i result = vdo_allocate_extended(struct region_table, region_count, struct layout_region, "layout region table for ISL", &table); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; lr = &table->regions[0]; @@ -546,7 +546,7 @@ static int __must_check write_index_save_header(struct index_save_layout *isl, size_t offset = 0; result = vdo_allocate(table->encoded_size, u8, "index save data", &buffer); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; encode_region_table(buffer, &offset, table); @@ -670,7 +670,7 @@ static int __must_check make_layout_region_table(struct index_layout *layout, result = vdo_allocate_extended(struct region_table, region_count, struct layout_region, "layout region table", &table); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; lr = &table->regions[0]; @@ -716,7 +716,7 @@ static int __must_check write_layout_header(struct index_layout *layout, size_t offset = 0; result = vdo_allocate(table->encoded_size, u8, "layout data", &buffer); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; encode_region_table(buffer, &offset, table); @@ -807,7 +807,7 @@ static int create_index_layout(struct index_layout *layout, struct uds_configura result = vdo_allocate(sizes.save_count, struct index_save_layout, __func__, &layout->index.saves); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; initialize_layout(layout, &sizes); @@ -1165,7 +1165,7 @@ static int __must_check load_region_table(struct buffered_reader *reader, result = vdo_allocate_extended(struct region_table, header.region_count, struct layout_region, "single file layout region table", &table); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; table->header = header; @@ -1202,7 +1202,7 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, size_t offset = 0; result = vdo_allocate(saved_size, u8, "super block data", &buffer); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = uds_read_from_buffered_reader(reader, buffer, saved_size); @@ -1337,7 +1337,7 @@ static int __must_check reconstitute_layout(struct index_layout *layout, result = vdo_allocate(layout->super.max_saves, struct index_save_layout, __func__, &layout->index.saves); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; layout->total_blocks = table->header.region_blocks; @@ -1696,7 +1696,7 @@ int uds_make_index_layout(struct uds_configuration *config, bool new_layout, return result; result = vdo_allocate(1, struct index_layout, __func__, &layout); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = create_layout_factory(layout, config); diff --git a/drivers/md/dm-vdo/indexer/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c index ddb6d843cbd9..c5d1b9995846 100644 --- a/drivers/md/dm-vdo/indexer/index-page-map.c +++ b/drivers/md/dm-vdo/indexer/index-page-map.c @@ -39,14 +39,14 @@ int uds_make_index_page_map(const struct index_geometry *geometry, struct index_page_map *map; result = vdo_allocate(1, struct index_page_map, "page map", &map); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; map->geometry = geometry; map->entries_per_chapter = geometry->index_pages_per_chapter - 1; result = vdo_allocate(get_entry_count(geometry), u16, "Index Page Map Entries", &map->entries); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_index_page_map(map); return result; } @@ -119,7 +119,7 @@ int uds_write_index_page_map(struct index_page_map *map, struct buffered_writer u32 i; result = vdo_allocate(saved_size, u8, "page map data", &buffer); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; memcpy(buffer, PAGE_MAP_MAGIC, PAGE_MAP_MAGIC_LENGTH); @@ -146,7 +146,7 @@ int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader * u32 i; result = vdo_allocate(saved_size, u8, "page map data", &buffer); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = uds_read_from_buffered_reader(reader, buffer, saved_size); diff --git a/drivers/md/dm-vdo/indexer/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c index 0f920a583021..22445dcb3fe0 100644 --- a/drivers/md/dm-vdo/indexer/index-session.c +++ b/drivers/md/dm-vdo/indexer/index-session.c @@ -222,7 +222,7 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde struct uds_index_session *session; result = vdo_allocate(1, struct uds_index_session, __func__, &session); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; mutex_init(&session->request_mutex); diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c index c576033b8a53..243a9deab4de 100644 --- a/drivers/md/dm-vdo/indexer/index.c +++ b/drivers/md/dm-vdo/indexer/index.c @@ -89,7 +89,7 @@ static int launch_zone_message(struct uds_zone_message message, unsigned int zon struct uds_request *request; result = vdo_allocate(1, struct uds_request, __func__, &request); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; request->index = index; @@ -770,7 +770,7 @@ static int make_chapter_writer(struct uds_index *index, result = vdo_allocate_extended(struct chapter_writer, index->zone_count, struct open_chapter_zone *, "Chapter Writer", &writer); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; writer->index = index; @@ -779,7 +779,7 @@ static int make_chapter_writer(struct uds_index *index, result = vdo_allocate_cache_aligned(collated_records_size, "collated records", &writer->collated_records); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { free_chapter_writer(writer); return result; } @@ -1127,7 +1127,7 @@ static int make_index_zone(struct uds_index *index, unsigned int zone_number) struct index_zone *zone; result = vdo_allocate(1, struct index_zone, "index zone", &zone); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = uds_make_open_chapter(index->volume->geometry, index->zone_count, @@ -1165,7 +1165,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op result = vdo_allocate_extended(struct uds_index, config->zone_count, struct uds_request_queue *, "index", &index); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; index->zone_count = config->zone_count; @@ -1178,7 +1178,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op result = vdo_allocate(index->zone_count, struct index_zone *, "zones", &index->zones); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_index(index); return result; } diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c index 749c950c0189..0dcf6d596653 100644 --- a/drivers/md/dm-vdo/indexer/io-factory.c +++ b/drivers/md/dm-vdo/indexer/io-factory.c @@ -65,7 +65,7 @@ int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_p struct io_factory *factory; result = vdo_allocate(1, struct io_factory, __func__, &factory); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; factory->bdev = bdev; @@ -145,7 +145,7 @@ int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block return result; result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { dm_bufio_client_destroy(client); return result; } @@ -283,7 +283,7 @@ int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block return result; result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { dm_bufio_client_destroy(client); return result; } diff --git a/drivers/md/dm-vdo/indexer/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c index 4a4dc94915dd..46b7bc1ac324 100644 --- a/drivers/md/dm-vdo/indexer/open-chapter.c +++ b/drivers/md/dm-vdo/indexer/open-chapter.c @@ -71,14 +71,14 @@ int uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zo result = vdo_allocate_extended(struct open_chapter_zone, slot_count, struct open_chapter_zone_slot, "open chapter", &open_chapter); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; open_chapter->slot_count = slot_count; open_chapter->capacity = capacity; result = vdo_allocate_cache_aligned(records_size(open_chapter), "record pages", &open_chapter->records); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_open_chapter(open_chapter); return result; } diff --git a/drivers/md/dm-vdo/indexer/radix-sort.c b/drivers/md/dm-vdo/indexer/radix-sort.c index 74ea18b8e9be..66b8c706a1ef 100644 --- a/drivers/md/dm-vdo/indexer/radix-sort.c +++ b/drivers/md/dm-vdo/indexer/radix-sort.c @@ -213,7 +213,7 @@ int uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter) result = vdo_allocate_extended(struct radix_sorter, stack_size, struct task, __func__, &radix_sorter); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; radix_sorter->count = count; diff --git a/drivers/md/dm-vdo/indexer/sparse-cache.c b/drivers/md/dm-vdo/indexer/sparse-cache.c index e297ba2d6ceb..28920167827c 100644 --- a/drivers/md/dm-vdo/indexer/sparse-cache.c +++ b/drivers/md/dm-vdo/indexer/sparse-cache.c @@ -224,7 +224,7 @@ static int __must_check initialize_cached_chapter_index(struct cached_chapter_in result = vdo_allocate(chapter->index_pages_count, struct delta_index_page, __func__, &chapter->index_pages); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; return vdo_allocate(chapter->index_pages_count, struct dm_buffer *, @@ -242,7 +242,7 @@ static int __must_check make_search_list(struct sparse_cache *cache, bytes = (sizeof(struct search_list) + (cache->capacity * sizeof(struct cached_chapter_index *))); result = vdo_allocate_cache_aligned(bytes, "search list", &list); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; list->capacity = cache->capacity; @@ -265,7 +265,7 @@ int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int ca bytes = (sizeof(struct sparse_cache) + (capacity * sizeof(struct cached_chapter_index))); result = vdo_allocate_cache_aligned(bytes, "sparse cache", &cache); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; cache->geometry = geometry; @@ -296,7 +296,7 @@ int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int ca /* purge_search_list() needs some temporary lists for sorting. */ result = vdo_allocate(capacity * 2, struct cached_chapter_index *, "scratch entries", &cache->scratch_entries); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) goto out; *cache_ptr = cache; diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c index 1a762e6dd709..1cc9ac4fe510 100644 --- a/drivers/md/dm-vdo/indexer/volume-index.c +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -1213,7 +1213,7 @@ static int initialize_volume_sub_index(const struct uds_configuration *config, /* The following arrays are initialized to all zeros. */ result = vdo_allocate(params.list_count, u64, "first chapter to flush", &sub_index->flush_chapters); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; return vdo_allocate(zone_count, struct volume_sub_index_zone, @@ -1229,7 +1229,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non int result; result = vdo_allocate(1, struct volume_index, "volume index", &volume_index); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; volume_index->zone_count = config->zone_count; @@ -1251,7 +1251,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non result = vdo_allocate(config->zone_count, struct volume_index_zone, "volume index zones", &volume_index->zones); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_volume_index(volume_index); return result; } diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c index 2d8901732f5d..959dd82ef665 100644 --- a/drivers/md/dm-vdo/indexer/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -1509,22 +1509,22 @@ static int __must_check initialize_page_cache(struct page_cache *cache, result = vdo_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read, "volume read queue", &cache->read_queue); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_allocate(cache->zone_count, struct search_pending_counter, "Volume Cache Zones", &cache->search_pending_counters); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_allocate(cache->indexable_pages, u16, "page cache index", &cache->index); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; result = vdo_allocate(cache->cache_slots, struct cached_page, "page cache cache", &cache->cache); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; /* Initialize index values to invalid values. */ @@ -1547,7 +1547,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout int result; result = vdo_allocate(1, struct volume, "volume", &volume); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; volume->nonce = uds_get_volume_nonce(layout); @@ -1586,7 +1586,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout result = vdo_allocate(geometry->records_per_page, const struct uds_volume_record *, "record pointers", &volume->record_pointers); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_volume(volume); return result; } @@ -1626,7 +1626,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout result = vdo_allocate(config->read_threads, struct thread *, "reader threads", &volume->reader_threads); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_volume(volume); return result; } diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index 0bd742ecbe2e..9849d12f2a36 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -152,7 +152,7 @@ static u64 hash_key(u64 key) * @map: The map to initialize. * @capacity: The initial capacity of the map. * - * Return: UDS_SUCCESS or an error code. + * Return: VDO_SUCCESS or an error code. */ static int allocate_buckets(struct int_map *map, size_t capacity) { @@ -174,7 +174,7 @@ static int allocate_buckets(struct int_map *map, size_t capacity) * tells the map to use its own small default). * @map_ptr: Output, a pointer to hold the new int_map. * - * Return: UDS_SUCCESS or an error code. + * Return: VDO_SUCCESS or an error code. */ int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) { @@ -183,7 +183,7 @@ int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) size_t capacity; result = vdo_allocate(1, struct int_map, "struct int_map", &map); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; /* Use the default capacity if the caller did not specify one. */ @@ -196,13 +196,13 @@ int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) capacity = capacity * 100 / DEFAULT_LOAD; result = allocate_buckets(map, capacity); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { vdo_int_map_free(vdo_forget(map)); return result; } *map_ptr = map; - return UDS_SUCCESS; + return VDO_SUCCESS; } /** @@ -368,7 +368,7 @@ void *vdo_int_map_get(struct int_map *map, u64 key) * * Resizes and rehashes all the existing entries, storing them in the new buckets. * - * Return: UDS_SUCCESS or an error code. + * Return: VDO_SUCCESS or an error code. */ static int resize_buckets(struct int_map *map) { @@ -384,7 +384,7 @@ static int resize_buckets(struct int_map *map) uds_log_info("%s: attempting resize from %zu to %zu, current size=%zu", __func__, map->capacity, new_capacity, map->size); result = allocate_buckets(map, new_capacity); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { *map = old_map; return result; } @@ -407,7 +407,7 @@ static int resize_buckets(struct int_map *map) /* Destroy the old bucket array. */ vdo_free(vdo_forget(old_map.buckets)); - return UDS_SUCCESS; + return VDO_SUCCESS; } /** @@ -647,7 +647,7 @@ int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, * large maps). */ result = resize_buckets(map); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; /* diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 23549b7e9e6d..b0f1ba810cd0 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -383,7 +383,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter result = vdo_allocate_extended(struct io_submitter, thread_count, struct bio_queue_data, "bio submission data", &io_submitter); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; io_submitter->bio_queue_rotation_interval = rotation_interval; diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index ec24fff2a21b..18c9d2af8aed 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -420,7 +420,7 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) int result; result = vdo_allocate(1, struct vdo_statistics, __func__, &stats); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_log_error("Cannot allocate memory to write VDO statistics"); return result; } diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 2d2cccf89edb..97208c9e0062 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -2379,7 +2379,7 @@ static int allocate_slab_counters(struct vdo_slab *slab) bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD); result = vdo_allocate(bytes, vdo_refcount_t, "ref counts array", &slab->counters); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { vdo_free(vdo_forget(slab->reference_blocks)); return result; } diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index ad7682784459..b4aa71fffdbf 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -83,7 +83,7 @@ int vdo_create_thread(void (*thread_function)(void *), void *thread_data, int result; result = vdo_allocate(1, struct thread, __func__, &thread); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_log_warning("Error allocating memory for %s", name); return result; } diff --git a/drivers/md/dm-vdo/uds-sysfs.c b/drivers/md/dm-vdo/uds-sysfs.c index d1d5a30b3717..753d81d6f207 100644 --- a/drivers/md/dm-vdo/uds-sysfs.c +++ b/drivers/md/dm-vdo/uds-sysfs.c @@ -35,7 +35,7 @@ static char *buffer_to_string(const char *buf, size_t length) { char *string; - if (vdo_allocate(length + 1, char, __func__, &string) != UDS_SUCCESS) + if (vdo_allocate(length + 1, char, __func__, &string) != VDO_SUCCESS) return NULL; memcpy(string, buf, length); diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index ae62f260c5ec..b4dd0634a5cb 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -545,7 +545,7 @@ int vdo_make(unsigned int instance, struct device_config *config, char **reason, *reason = "Unspecified error"; result = vdo_allocate(1, struct vdo, __func__, &vdo); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { *reason = "Cannot allocate VDO"; return result; } From 6c43cf24882e3a8ed8f918e92d5f5bed935a580b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 12:17:53 -0600 Subject: [PATCH 0946/1038] dm vdo int-map: return VDO_SUCCESS on success Update all callers to check for VDO_SUCCESS (most already did). Also fix whitespace for update_mapping() parameters. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/block-map.c | 4 ++-- drivers/md/dm-vdo/int-map.c | 20 ++++++++------------ drivers/md/dm-vdo/io-submitter.c | 4 ++-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index c4719fb30f86..320e76527e2b 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -231,7 +231,7 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache) return result; result = vdo_int_map_create(cache->page_count, &cache->page_map); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; return initialize_info(cache); @@ -390,7 +390,7 @@ static int __must_check set_info_pbn(struct page_info *info, physical_block_numb if (pbn != NO_PAGE) { result = vdo_int_map_put(cache->page_map, pbn, info, true, NULL); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; } return VDO_SUCCESS; diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index 9849d12f2a36..a909a11204c1 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -397,7 +397,7 @@ static int resize_buckets(struct int_map *map) continue; result = vdo_int_map_put(map, entry->key, entry->value, true, NULL); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { /* Destroy the new partial map and restore the map from the stack. */ vdo_free(vdo_forget(map->buckets)); *map = old_map; @@ -525,12 +525,8 @@ static struct bucket *move_empty_bucket(struct int_map *map __always_unused, * * Return: true if the map contains a mapping for the key, false if it does not. */ -static bool update_mapping(struct int_map *map, - struct bucket *neighborhood, - u64 key, - void *new_value, - bool update, - void **old_value_ptr) +static bool update_mapping(struct int_map *map, struct bucket *neighborhood, + u64 key, void *new_value, bool update, void **old_value_ptr) { struct bucket *bucket = search_hop_list(map, neighborhood, key, NULL); @@ -609,15 +605,15 @@ static struct bucket *find_or_make_vacancy(struct int_map *map, * update is true. In either case the old value is returned. If the map does not already contain a * value for the specified key, the new value is added regardless of the value of update. * - * Return: UDS_SUCCESS or an error code. + * Return: VDO_SUCCESS or an error code. */ int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, void **old_value_ptr) { struct bucket *neighborhood, *bucket; - if (new_value == NULL) - return UDS_INVALID_ARGUMENT; + if (unlikely(new_value == NULL)) + return -EINVAL; /* * Select the bucket at the start of the neighborhood that must contain any entry for the @@ -630,7 +626,7 @@ int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, * optionally update it, returning the old value. */ if (update_mapping(map, neighborhood, key, new_value, update, old_value_ptr)) - return UDS_SUCCESS; + return VDO_SUCCESS; /* * Find an empty bucket in the desired neighborhood for the new entry or re-arrange entries @@ -666,7 +662,7 @@ int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, /* There was no existing entry, so there was no old value to be returned. */ if (old_value_ptr != NULL) *old_value_ptr = NULL; - return UDS_SUCCESS; + return VDO_SUCCESS; } /** diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index b0f1ba810cd0..e82b4a8c6fc4 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -300,7 +300,7 @@ static bool try_bio_map_merge(struct vio *vio) mutex_unlock(&bio_queue_data->lock); /* We don't care about failure of int_map_put in this case. */ - ASSERT_LOG_ONLY(result == UDS_SUCCESS, "bio map insertion succeeds"); + ASSERT_LOG_ONLY(result == VDO_SUCCESS, "bio map insertion succeeds"); return merged; } @@ -403,7 +403,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter */ result = vdo_int_map_create(max_requests_active * 2, &bio_queue_data->map); - if (result != 0) { + if (result != VDO_SUCCESS) { /* * Clean up the partially initialized bio-queue entirely and indicate that * initialization failed. From 34edf9e28c917cfb16522bf4adc630efed8629c5 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 13:18:35 -0600 Subject: [PATCH 0947/1038] dm vdo thread-utils: return VDO_SUCCESS on vdo_create_thread success Update all callers to check for VDO_SUCCESS. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/indexer/funnel-requestqueue.c | 2 +- drivers/md/dm-vdo/indexer/index.c | 2 +- drivers/md/dm-vdo/indexer/volume.c | 2 +- drivers/md/dm-vdo/thread-utils.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c index eee7b980960f..84c7c1ae1333 100644 --- a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c +++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c @@ -221,7 +221,7 @@ int uds_make_request_queue(const char *queue_name, result = vdo_create_thread(request_queue_worker, queue, queue_name, &queue->thread); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_request_queue_finish(queue); return result; } diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c index 243a9deab4de..226713221105 100644 --- a/drivers/md/dm-vdo/indexer/index.c +++ b/drivers/md/dm-vdo/indexer/index.c @@ -798,7 +798,7 @@ static int make_chapter_writer(struct uds_index *index, writer->open_chapter_index->memory_size); result = vdo_create_thread(close_chapters, writer, "writer", &writer->thread); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { free_chapter_writer(writer); return result; } diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c index 959dd82ef665..0a4beef8ac8d 100644 --- a/drivers/md/dm-vdo/indexer/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -1634,7 +1634,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout for (i = 0; i < config->read_threads; i++) { result = vdo_create_thread(read_thread_function, (void *) volume, "reader", &volume->reader_threads[i]); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_free_volume(volume); return result; } diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index b4aa71fffdbf..c822df86f731 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -119,7 +119,7 @@ int vdo_create_thread(void (*thread_function)(void *), void *thread_data, } *new_thread = thread; - return UDS_SUCCESS; + return VDO_SUCCESS; } void vdo_join_threads(struct thread *thread) From a958c53af7a5b9297ec0dcaf3689172c34485e35 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 13:50:41 -0600 Subject: [PATCH 0948/1038] dm-vdo funnel-workqueue: return VDO_SUCCESS from make_simple_work_queue Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/funnel-workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c index a88f5c93eae5..a923432f0a37 100644 --- a/drivers/md/dm-vdo/funnel-workqueue.c +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -367,7 +367,7 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na wait_for_completion(&started); *queue_ptr = queue; - return UDS_SUCCESS; + return VDO_SUCCESS; } /** From 6a79248b425dcddc749ecbe0a2e1017afb5fdcd2 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 14:57:33 -0600 Subject: [PATCH 0949/1038] dm vdo permassert: audit all of ASSERT to test for VDO_SUCCESS Also rename ASSERT to VDO_ASSERT and ASSERT_LOG_ONLY to VDO_ASSERT_LOG_ONLY. But re-introduce ASSERT and ASSERT_LOG_ONLY as a placeholder for the benefit of dm-vdo/indexer. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/action-manager.c | 8 +- drivers/md/dm-vdo/block-map.c | 118 +++++++++---------- drivers/md/dm-vdo/completion.c | 10 +- drivers/md/dm-vdo/completion.h | 6 +- drivers/md/dm-vdo/data-vio.c | 108 +++++++++--------- drivers/md/dm-vdo/data-vio.h | 68 +++++------ drivers/md/dm-vdo/dedupe.c | 161 +++++++++++++------------- drivers/md/dm-vdo/dm-vdo-target.c | 38 +++---- drivers/md/dm-vdo/encodings.c | 164 +++++++++++++-------------- drivers/md/dm-vdo/errors.c | 5 +- drivers/md/dm-vdo/flush.c | 22 ++-- drivers/md/dm-vdo/funnel-workqueue.c | 22 ++-- drivers/md/dm-vdo/io-submitter.c | 8 +- drivers/md/dm-vdo/logical-zone.c | 22 ++-- drivers/md/dm-vdo/memory-alloc.c | 12 +- drivers/md/dm-vdo/packer.c | 12 +- drivers/md/dm-vdo/permassert.h | 15 ++- drivers/md/dm-vdo/physical-zone.c | 48 ++++---- drivers/md/dm-vdo/priority-table.c | 4 +- drivers/md/dm-vdo/recovery-journal.c | 60 +++++----- drivers/md/dm-vdo/repair.c | 12 +- drivers/md/dm-vdo/slab-depot.c | 116 +++++++++---------- drivers/md/dm-vdo/thread-registry.c | 4 +- drivers/md/dm-vdo/vdo.c | 32 +++--- drivers/md/dm-vdo/vio.c | 40 +++---- drivers/md/dm-vdo/vio.h | 8 +- 26 files changed, 563 insertions(+), 560 deletions(-) diff --git a/drivers/md/dm-vdo/action-manager.c b/drivers/md/dm-vdo/action-manager.c index 709be4c17d27..a0e5e7077d13 100644 --- a/drivers/md/dm-vdo/action-manager.c +++ b/drivers/md/dm-vdo/action-manager.c @@ -177,8 +177,8 @@ static void apply_to_zone(struct vdo_completion *completion) zone_count_t zone; struct action_manager *manager = as_action_manager(completion); - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == get_acting_zone_thread_id(manager)), - "%s() called on acting zones's thread", __func__); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == get_acting_zone_thread_id(manager)), + "%s() called on acting zones's thread", __func__); zone = manager->acting_zone++; if (manager->acting_zone == manager->zones) { @@ -357,8 +357,8 @@ bool vdo_schedule_operation_with_context(struct action_manager *manager, { struct action *current_action; - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == manager->initiator_thread_id), - "action initiated from correct thread"); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == manager->initiator_thread_id), + "action initiated from correct thread"); if (!manager->current_action->in_use) { current_action = manager->current_action; } else if (!manager->current_action->next->in_use) { diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index 320e76527e2b..b70294d8bb61 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -246,16 +246,16 @@ static inline void assert_on_cache_thread(struct vdo_page_cache *cache, { thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((thread_id == cache->zone->thread_id), - "%s() must only be called on cache thread %d, not thread %d", - function_name, cache->zone->thread_id, thread_id); + VDO_ASSERT_LOG_ONLY((thread_id == cache->zone->thread_id), + "%s() must only be called on cache thread %d, not thread %d", + function_name, cache->zone->thread_id, thread_id); } /** assert_io_allowed() - Assert that a page cache may issue I/O. */ static inline void assert_io_allowed(struct vdo_page_cache *cache) { - ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&cache->zone->state), - "VDO page cache may issue I/O"); + VDO_ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&cache->zone->state), + "VDO page cache may issue I/O"); } /** report_cache_pressure() - Log and, if enabled, report cache pressure. */ @@ -287,9 +287,9 @@ static const char * __must_check get_page_state_name(enum vdo_page_buffer_state BUILD_BUG_ON(ARRAY_SIZE(state_names) != PAGE_STATE_COUNT); - result = ASSERT(state < ARRAY_SIZE(state_names), - "Unknown page_state value %d", state); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(state < ARRAY_SIZE(state_names), + "Unknown page_state value %d", state); + if (result != VDO_SUCCESS) return "[UNKNOWN PAGE STATE]"; return state_names[state]; @@ -378,8 +378,8 @@ static int __must_check set_info_pbn(struct page_info *info, physical_block_numb struct vdo_page_cache *cache = info->cache; /* Either the new or the old page number must be NO_PAGE. */ - int result = ASSERT((pbn == NO_PAGE) || (info->pbn == NO_PAGE), - "Must free a page before reusing it."); + int result = VDO_ASSERT((pbn == NO_PAGE) || (info->pbn == NO_PAGE), + "Must free a page before reusing it."); if (result != VDO_SUCCESS) return result; @@ -401,13 +401,13 @@ static int reset_page_info(struct page_info *info) { int result; - result = ASSERT(info->busy == 0, "VDO Page must not be busy"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(info->busy == 0, "VDO Page must not be busy"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(!vdo_waitq_has_waiters(&info->waiting), - "VDO Page must not have waiters"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(!vdo_waitq_has_waiters(&info->waiting), + "VDO Page must not have waiters"); + if (result != VDO_SUCCESS) return result; result = set_info_pbn(info, NO_PAGE); @@ -592,29 +592,29 @@ static int __must_check validate_completed_page(struct vdo_page_completion *comp { int result; - result = ASSERT(completion->ready, "VDO Page completion not ready"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(completion->ready, "VDO Page completion not ready"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(completion->info != NULL, - "VDO Page Completion must be complete"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(completion->info != NULL, + "VDO Page Completion must be complete"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(completion->info->pbn == completion->pbn, - "VDO Page Completion pbn must be consistent"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(completion->info->pbn == completion->pbn, + "VDO Page Completion pbn must be consistent"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(is_valid(completion->info), - "VDO Page Completion page must be valid"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(is_valid(completion->info), + "VDO Page Completion page must be valid"); + if (result != VDO_SUCCESS) return result; if (writable) { - result = ASSERT(completion->writable, - "VDO Page Completion must be writable"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(completion->writable, + "VDO Page Completion must be writable"); + if (result != VDO_SUCCESS) return result; } @@ -776,7 +776,7 @@ static int __must_check launch_page_load(struct page_info *info, if (result != VDO_SUCCESS) return result; - result = ASSERT((info->busy == 0), "Page is not busy before loading."); + result = VDO_ASSERT((info->busy == 0), "Page is not busy before loading."); if (result != VDO_SUCCESS) return result; @@ -949,8 +949,8 @@ static void discard_a_page(struct vdo_page_cache *cache) return; } - ASSERT_LOG_ONLY(!is_in_flight(info), - "page selected for discard is not in flight"); + VDO_ASSERT_LOG_ONLY(!is_in_flight(info), + "page selected for discard is not in flight"); cache->discard_count++; info->write_status = WRITE_STATUS_DISCARD; @@ -1153,8 +1153,8 @@ void vdo_release_page_completion(struct vdo_completion *completion) discard_info = page_completion->info; } - ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), - "Page being released after leaving all queues"); + VDO_ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), + "Page being released after leaving all queues"); page_completion->info = NULL; cache = page_completion->cache; @@ -1217,8 +1217,8 @@ void vdo_get_page(struct vdo_page_completion *page_completion, struct page_info *info; assert_on_cache_thread(cache, __func__); - ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), - "New page completion was not already on a wait queue"); + VDO_ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), + "New page completion was not already on a wait queue"); *page_completion = (struct vdo_page_completion) { .pbn = pbn, @@ -1265,7 +1265,7 @@ void vdo_get_page(struct vdo_page_completion *page_completion, } /* Something horrible has gone wrong. */ - ASSERT_LOG_ONLY(false, "Info found in a usable state."); + VDO_ASSERT_LOG_ONLY(false, "Info found in a usable state."); } /* The page must be fetched. */ @@ -1334,7 +1334,7 @@ int vdo_invalidate_page_cache(struct vdo_page_cache *cache) /* Make sure we don't throw away any dirty pages. */ for (info = cache->infos; info < cache->infos + cache->page_count; info++) { - int result = ASSERT(!is_dirty(info), "cache must have no dirty pages"); + int result = VDO_ASSERT(!is_dirty(info), "cache must have no dirty pages"); if (result != VDO_SUCCESS) return result; @@ -1440,10 +1440,10 @@ static bool __must_check is_not_older(struct block_map_zone *zone, u8 a, u8 b) { int result; - result = ASSERT((in_cyclic_range(zone->oldest_generation, a, zone->generation, 1 << 8) && - in_cyclic_range(zone->oldest_generation, b, zone->generation, 1 << 8)), - "generation(s) %u, %u are out of range [%u, %u]", - a, b, zone->oldest_generation, zone->generation); + result = VDO_ASSERT((in_cyclic_range(zone->oldest_generation, a, zone->generation, 1 << 8) && + in_cyclic_range(zone->oldest_generation, b, zone->generation, 1 << 8)), + "generation(s) %u, %u are out of range [%u, %u]", + a, b, zone->oldest_generation, zone->generation); if (result != VDO_SUCCESS) { enter_zone_read_only_mode(zone, result); return true; @@ -1456,8 +1456,8 @@ static void release_generation(struct block_map_zone *zone, u8 generation) { int result; - result = ASSERT((zone->dirty_page_counts[generation] > 0), - "dirty page count underflow for generation %u", generation); + result = VDO_ASSERT((zone->dirty_page_counts[generation] > 0), + "dirty page count underflow for generation %u", generation); if (result != VDO_SUCCESS) { enter_zone_read_only_mode(zone, result); return; @@ -1482,8 +1482,8 @@ static void set_generation(struct block_map_zone *zone, struct tree_page *page, page->generation = new_generation; new_count = ++zone->dirty_page_counts[new_generation]; - result = ASSERT((new_count != 0), "dirty page count overflow for generation %u", - new_generation); + result = VDO_ASSERT((new_count != 0), "dirty page count overflow for generation %u", + new_generation); if (result != VDO_SUCCESS) { enter_zone_read_only_mode(zone, result); return; @@ -1698,15 +1698,15 @@ static void release_page_lock(struct data_vio *data_vio, char *what) struct tree_lock *lock_holder; struct tree_lock *lock = &data_vio->tree_lock; - ASSERT_LOG_ONLY(lock->locked, - "release of unlocked block map page %s for key %llu in tree %u", - what, (unsigned long long) lock->key, lock->root_index); + VDO_ASSERT_LOG_ONLY(lock->locked, + "release of unlocked block map page %s for key %llu in tree %u", + what, (unsigned long long) lock->key, lock->root_index); zone = data_vio->logical.zone->block_map_zone; lock_holder = vdo_int_map_remove(zone->loading_pages, lock->key); - ASSERT_LOG_ONLY((lock_holder == lock), - "block map page %s mismatch for key %llu in tree %u", - what, (unsigned long long) lock->key, lock->root_index); + VDO_ASSERT_LOG_ONLY((lock_holder == lock), + "block map page %s mismatch for key %llu in tree %u", + what, (unsigned long long) lock->key, lock->root_index); lock->locked = false; } @@ -2008,8 +2008,8 @@ static void write_expired_elements(struct block_map_zone *zone) list_del_init(&page->entry); - result = ASSERT(!vdo_waiter_is_waiting(&page->waiter), - "Newly expired page not already waiting to write"); + result = VDO_ASSERT(!vdo_waiter_is_waiting(&page->waiter), + "Newly expired page not already waiting to write"); if (result != VDO_SUCCESS) { enter_zone_read_only_mode(zone, result); continue; @@ -2867,8 +2867,8 @@ int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical BUILD_BUG_ON(VDO_BLOCK_MAP_ENTRIES_PER_PAGE != ((VDO_BLOCK_SIZE - sizeof(struct block_map_page)) / sizeof(struct block_map_entry))); - result = ASSERT(cache_size > 0, "block map cache size is specified"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(cache_size > 0, "block map cache size is specified"); + if (result != VDO_SUCCESS) return result; result = vdo_allocate_extended(struct block_map, @@ -2937,7 +2937,7 @@ void vdo_initialize_block_map_from_journal(struct block_map *map, for (z = 0; z < map->zone_count; z++) { struct dirty_lists *dirty_lists = map->zones[z].dirty_lists; - ASSERT_LOG_ONLY(dirty_lists->next_period == 0, "current period not set"); + VDO_ASSERT_LOG_ONLY(dirty_lists->next_period == 0, "current period not set"); dirty_lists->oldest_period = map->current_era_point; dirty_lists->next_period = map->current_era_point + 1; dirty_lists->offset = map->current_era_point % dirty_lists->maximum_age; @@ -2971,8 +2971,8 @@ static void initiate_drain(struct admin_state *state) { struct block_map_zone *zone = container_of(state, struct block_map_zone, state); - ASSERT_LOG_ONLY((zone->active_lookups == 0), - "%s() called with no active lookups", __func__); + VDO_ASSERT_LOG_ONLY((zone->active_lookups == 0), + "%s() called with no active lookups", __func__); if (!vdo_is_state_suspending(state)) { while (zone->dirty_lists->oldest_period < zone->dirty_lists->next_period) diff --git a/drivers/md/dm-vdo/completion.c b/drivers/md/dm-vdo/completion.c index 9e2381dc3683..5ad85334632d 100644 --- a/drivers/md/dm-vdo/completion.c +++ b/drivers/md/dm-vdo/completion.c @@ -60,7 +60,7 @@ void vdo_initialize_completion(struct vdo_completion *completion, static inline void assert_incomplete(struct vdo_completion *completion) { - ASSERT_LOG_ONLY(!completion->complete, "completion is not complete"); + VDO_ASSERT_LOG_ONLY(!completion->complete, "completion is not complete"); } /** @@ -111,10 +111,10 @@ void vdo_enqueue_completion(struct vdo_completion *completion, struct vdo *vdo = completion->vdo; thread_id_t thread_id = completion->callback_thread_id; - if (ASSERT(thread_id < vdo->thread_config.thread_count, - "thread_id %u (completion type %d) is less than thread count %u", - thread_id, completion->type, - vdo->thread_config.thread_count) != UDS_SUCCESS) + if (VDO_ASSERT(thread_id < vdo->thread_config.thread_count, + "thread_id %u (completion type %d) is less than thread count %u", + thread_id, completion->type, + vdo->thread_config.thread_count) != VDO_SUCCESS) BUG(); completion->requeue = false; diff --git a/drivers/md/dm-vdo/completion.h b/drivers/md/dm-vdo/completion.h index aa145d73a686..3407f34ce58c 100644 --- a/drivers/md/dm-vdo/completion.h +++ b/drivers/md/dm-vdo/completion.h @@ -85,9 +85,9 @@ static inline void vdo_fail_completion(struct vdo_completion *completion, int re static inline int vdo_assert_completion_type(struct vdo_completion *completion, enum vdo_completion_type expected) { - return ASSERT(expected == completion->type, - "completion type should be %u, not %u", expected, - completion->type); + return VDO_ASSERT(expected == completion->type, + "completion type should be %u, not %u", expected, + completion->type); } static inline void vdo_set_completion_callback(struct vdo_completion *completion, diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 3d5054e61330..51c49fad1b8b 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -232,8 +232,8 @@ static bool check_for_drain_complete_locked(struct data_vio_pool *pool) if (pool->limiter.busy > 0) return false; - ASSERT_LOG_ONLY((pool->discard_limiter.busy == 0), - "no outstanding discard permits"); + VDO_ASSERT_LOG_ONLY((pool->discard_limiter.busy == 0), + "no outstanding discard permits"); return (bio_list_empty(&pool->limiter.new_waiters) && bio_list_empty(&pool->discard_limiter.new_waiters)); @@ -277,9 +277,9 @@ static void acknowledge_data_vio(struct data_vio *data_vio) if (bio == NULL) return; - ASSERT_LOG_ONLY((data_vio->remaining_discard <= - (u32) (VDO_BLOCK_SIZE - data_vio->offset)), - "data_vio to acknowledge is not an incomplete discard"); + VDO_ASSERT_LOG_ONLY((data_vio->remaining_discard <= + (u32) (VDO_BLOCK_SIZE - data_vio->offset)), + "data_vio to acknowledge is not an incomplete discard"); data_vio->user_bio = NULL; vdo_count_bios(&vdo->stats.bios_acknowledged, bio); @@ -443,7 +443,7 @@ static void attempt_logical_block_lock(struct vdo_completion *completion) return; } - result = ASSERT(lock_holder->logical.locked, "logical block lock held"); + result = VDO_ASSERT(lock_holder->logical.locked, "logical block lock held"); if (result != VDO_SUCCESS) { continue_data_vio_with_error(data_vio, result); return; @@ -627,9 +627,9 @@ static void update_limiter(struct limiter *limiter) struct bio_list *waiters = &limiter->waiters; data_vio_count_t available = limiter->limit - limiter->busy; - ASSERT_LOG_ONLY((limiter->release_count <= limiter->busy), - "Release count %u is not more than busy count %u", - limiter->release_count, limiter->busy); + VDO_ASSERT_LOG_ONLY((limiter->release_count <= limiter->busy), + "Release count %u is not more than busy count %u", + limiter->release_count, limiter->busy); get_waiters(limiter); for (; (limiter->release_count > 0) && !bio_list_empty(waiters); limiter->release_count--) @@ -850,8 +850,8 @@ int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, if (result != VDO_SUCCESS) return result; - ASSERT_LOG_ONLY((discard_limit <= pool_size), - "discard limit does not exceed pool size"); + VDO_ASSERT_LOG_ONLY((discard_limit <= pool_size), + "discard limit does not exceed pool size"); initialize_limiter(&pool->discard_limiter, pool, assign_discard_permit, discard_limit); pool->discard_limiter.permitted_waiters = &pool->permitted_discards; @@ -908,15 +908,15 @@ void free_data_vio_pool(struct data_vio_pool *pool) BUG_ON(atomic_read(&pool->processing)); spin_lock(&pool->lock); - ASSERT_LOG_ONLY((pool->limiter.busy == 0), - "data_vio pool must not have %u busy entries when being freed", - pool->limiter.busy); - ASSERT_LOG_ONLY((bio_list_empty(&pool->limiter.waiters) && - bio_list_empty(&pool->limiter.new_waiters)), - "data_vio pool must not have threads waiting to read or write when being freed"); - ASSERT_LOG_ONLY((bio_list_empty(&pool->discard_limiter.waiters) && - bio_list_empty(&pool->discard_limiter.new_waiters)), - "data_vio pool must not have threads waiting to discard when being freed"); + VDO_ASSERT_LOG_ONLY((pool->limiter.busy == 0), + "data_vio pool must not have %u busy entries when being freed", + pool->limiter.busy); + VDO_ASSERT_LOG_ONLY((bio_list_empty(&pool->limiter.waiters) && + bio_list_empty(&pool->limiter.new_waiters)), + "data_vio pool must not have threads waiting to read or write when being freed"); + VDO_ASSERT_LOG_ONLY((bio_list_empty(&pool->discard_limiter.waiters) && + bio_list_empty(&pool->discard_limiter.new_waiters)), + "data_vio pool must not have threads waiting to discard when being freed"); spin_unlock(&pool->lock); list_for_each_entry_safe(data_vio, tmp, &pool->available, pool_entry) { @@ -961,8 +961,8 @@ void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio) { struct data_vio *data_vio; - ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&pool->state), - "data_vio_pool not quiescent on acquire"); + VDO_ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&pool->state), + "data_vio_pool not quiescent on acquire"); bio->bi_private = (void *) jiffies; spin_lock(&pool->lock); @@ -998,8 +998,8 @@ static void initiate_drain(struct admin_state *state) static void assert_on_vdo_cpu_thread(const struct vdo *vdo, const char *name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.cpu_thread), - "%s called on cpu thread", name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.cpu_thread), + "%s called on cpu thread", name); } /** @@ -1173,17 +1173,17 @@ static void release_lock(struct data_vio *data_vio, struct lbn_lock *lock) /* The lock is not locked, so it had better not be registered in the lock map. */ struct data_vio *lock_holder = vdo_int_map_get(lock_map, lock->lbn); - ASSERT_LOG_ONLY((data_vio != lock_holder), - "no logical block lock held for block %llu", - (unsigned long long) lock->lbn); + VDO_ASSERT_LOG_ONLY((data_vio != lock_holder), + "no logical block lock held for block %llu", + (unsigned long long) lock->lbn); return; } /* Release the lock by removing the lock from the map. */ lock_holder = vdo_int_map_remove(lock_map, lock->lbn); - ASSERT_LOG_ONLY((data_vio == lock_holder), - "logical block lock mismatch for block %llu", - (unsigned long long) lock->lbn); + VDO_ASSERT_LOG_ONLY((data_vio == lock_holder), + "logical block lock mismatch for block %llu", + (unsigned long long) lock->lbn); lock->locked = false; } @@ -1193,7 +1193,7 @@ static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) struct data_vio *lock_holder, *next_lock_holder; int result; - ASSERT_LOG_ONLY(lock->locked, "lbn_lock with waiters is not locked"); + VDO_ASSERT_LOG_ONLY(lock->locked, "lbn_lock with waiters is not locked"); /* Another data_vio is waiting for the lock, transfer it in a single lock map operation. */ next_lock_holder = @@ -1210,9 +1210,9 @@ static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) return; } - ASSERT_LOG_ONLY((lock_holder == data_vio), - "logical block lock mismatch for block %llu", - (unsigned long long) lock->lbn); + VDO_ASSERT_LOG_ONLY((lock_holder == data_vio), + "logical block lock mismatch for block %llu", + (unsigned long long) lock->lbn); lock->locked = false; /* @@ -1275,10 +1275,10 @@ static void finish_cleanup(struct data_vio *data_vio) { struct vdo_completion *completion = &data_vio->vio.completion; - ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL, - "complete data_vio has no allocation lock"); - ASSERT_LOG_ONLY(data_vio->hash_lock == NULL, - "complete data_vio has no hash lock"); + VDO_ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL, + "complete data_vio has no allocation lock"); + VDO_ASSERT_LOG_ONLY(data_vio->hash_lock == NULL, + "complete data_vio has no hash lock"); if ((data_vio->remaining_discard <= VDO_BLOCK_SIZE) || (completion->result != VDO_SUCCESS)) { struct data_vio_pool *pool = completion->vdo->data_vio_pool; @@ -1404,8 +1404,8 @@ void data_vio_allocate_data_block(struct data_vio *data_vio, { struct allocation *allocation = &data_vio->allocation; - ASSERT_LOG_ONLY((allocation->pbn == VDO_ZERO_BLOCK), - "data_vio does not have an allocation"); + VDO_ASSERT_LOG_ONLY((allocation->pbn == VDO_ZERO_BLOCK), + "data_vio does not have an allocation"); allocation->write_lock_type = write_lock_type; allocation->zone = vdo_get_next_allocation_zone(data_vio->logical.zone); allocation->first_allocation_zone = allocation->zone->zone_number; @@ -1796,11 +1796,11 @@ static void compress_data_vio(struct vdo_completion *completion) */ void launch_compress_data_vio(struct data_vio *data_vio) { - ASSERT_LOG_ONLY(!data_vio->is_duplicate, "compressing a non-duplicate block"); - ASSERT_LOG_ONLY(data_vio->hash_lock != NULL, - "data_vio to compress has a hash_lock"); - ASSERT_LOG_ONLY(data_vio_has_allocation(data_vio), - "data_vio to compress has an allocation"); + VDO_ASSERT_LOG_ONLY(!data_vio->is_duplicate, "compressing a non-duplicate block"); + VDO_ASSERT_LOG_ONLY(data_vio->hash_lock != NULL, + "data_vio to compress has a hash_lock"); + VDO_ASSERT_LOG_ONLY(data_vio_has_allocation(data_vio), + "data_vio to compress has an allocation"); /* * There are 4 reasons why a data_vio which has reached this point will not be eligible for @@ -1841,7 +1841,7 @@ static void hash_data_vio(struct vdo_completion *completion) struct data_vio *data_vio = as_data_vio(completion); assert_data_vio_on_cpu_thread(data_vio); - ASSERT_LOG_ONLY(!data_vio->is_zero, "zero blocks should not be hashed"); + VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "zero blocks should not be hashed"); murmurhash3_128(data_vio->vio.data, VDO_BLOCK_SIZE, 0x62ea60be, &data_vio->record_name); @@ -1856,7 +1856,7 @@ static void hash_data_vio(struct vdo_completion *completion) static void prepare_for_dedupe(struct data_vio *data_vio) { /* We don't care what thread we are on. */ - ASSERT_LOG_ONLY(!data_vio->is_zero, "must not prepare to dedupe zero blocks"); + VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "must not prepare to dedupe zero blocks"); /* * Before we can dedupe, we need to know the record name, so the first @@ -1929,11 +1929,11 @@ static void acknowledge_write_callback(struct vdo_completion *completion) struct data_vio *data_vio = as_data_vio(completion); struct vdo *vdo = completion->vdo; - ASSERT_LOG_ONLY((!vdo_uses_bio_ack_queue(vdo) || - (vdo_get_callback_thread_id() == vdo->thread_config.bio_ack_thread)), - "%s() called on bio ack queue", __func__); - ASSERT_LOG_ONLY(data_vio_has_flush_generation_lock(data_vio), - "write VIO to be acknowledged has a flush generation lock"); + VDO_ASSERT_LOG_ONLY((!vdo_uses_bio_ack_queue(vdo) || + (vdo_get_callback_thread_id() == vdo->thread_config.bio_ack_thread)), + "%s() called on bio ack queue", __func__); + VDO_ASSERT_LOG_ONLY(data_vio_has_flush_generation_lock(data_vio), + "write VIO to be acknowledged has a flush generation lock"); acknowledge_data_vio(data_vio); if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) { /* This is a zero write or discard */ @@ -1998,8 +1998,8 @@ static void handle_allocation_error(struct vdo_completion *completion) static int assert_is_discard(struct data_vio *data_vio) { - int result = ASSERT(data_vio->is_discard, - "data_vio with no block map page is a discard"); + int result = VDO_ASSERT(data_vio->is_discard, + "data_vio with no block map page is a discard"); return ((result == VDO_SUCCESS) ? result : VDO_READ_ONLY); } diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h index 44fd0d8ccb76..25926b6cd98b 100644 --- a/drivers/md/dm-vdo/data-vio.h +++ b/drivers/md/dm-vdo/data-vio.h @@ -280,7 +280,7 @@ struct data_vio { static inline struct data_vio *vio_as_data_vio(struct vio *vio) { - ASSERT_LOG_ONLY((vio->type == VIO_TYPE_DATA), "vio is a data_vio"); + VDO_ASSERT_LOG_ONLY((vio->type == VIO_TYPE_DATA), "vio is a data_vio"); return container_of(vio, struct data_vio, vio); } @@ -374,9 +374,9 @@ static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio) * It's odd to use the LBN, but converting the record name to hex is a bit clunky for an * inline, and the LBN better than nothing as an identifier. */ - ASSERT_LOG_ONLY((expected == thread_id), - "data_vio for logical block %llu on thread %u, should be on hash zone thread %u", - (unsigned long long) data_vio->logical.lbn, thread_id, expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for logical block %llu on thread %u, should be on hash zone thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, expected); } static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio, @@ -402,9 +402,9 @@ static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio) thread_id_t expected = data_vio->logical.zone->thread_id; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((expected == thread_id), - "data_vio for logical block %llu on thread %u, should be on thread %u", - (unsigned long long) data_vio->logical.lbn, thread_id, expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for logical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, expected); } static inline void set_data_vio_logical_callback(struct data_vio *data_vio, @@ -430,10 +430,10 @@ static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio) thread_id_t expected = data_vio->allocation.zone->thread_id; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((expected == thread_id), - "struct data_vio for allocated physical block %llu on thread %u, should be on thread %u", - (unsigned long long) data_vio->allocation.pbn, thread_id, - expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "struct data_vio for allocated physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->allocation.pbn, thread_id, + expected); } static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio, @@ -460,10 +460,10 @@ static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio) thread_id_t expected = data_vio->duplicate.zone->thread_id; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((expected == thread_id), - "data_vio for duplicate physical block %llu on thread %u, should be on thread %u", - (unsigned long long) data_vio->duplicate.pbn, thread_id, - expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for duplicate physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->duplicate.pbn, thread_id, + expected); } static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio, @@ -490,9 +490,9 @@ static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio) thread_id_t expected = data_vio->mapped.zone->thread_id; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((expected == thread_id), - "data_vio for mapped physical block %llu on thread %u, should be on thread %u", - (unsigned long long) data_vio->mapped.pbn, thread_id, expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for mapped physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->mapped.pbn, thread_id, expected); } static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio, @@ -507,10 +507,10 @@ static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio) thread_id_t expected = data_vio->new_mapped.zone->thread_id; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((expected == thread_id), - "data_vio for new_mapped physical block %llu on thread %u, should be on thread %u", - (unsigned long long) data_vio->new_mapped.pbn, thread_id, - expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for new_mapped physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->new_mapped.pbn, thread_id, + expected); } static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio, @@ -525,10 +525,10 @@ static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio) thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((journal_thread == thread_id), - "data_vio for logical block %llu on thread %u, should be on journal thread %u", - (unsigned long long) data_vio->logical.lbn, thread_id, - journal_thread); + VDO_ASSERT_LOG_ONLY((journal_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on journal thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + journal_thread); } static inline void set_data_vio_journal_callback(struct data_vio *data_vio, @@ -555,10 +555,10 @@ static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio) thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((packer_thread == thread_id), - "data_vio for logical block %llu on thread %u, should be on packer thread %u", - (unsigned long long) data_vio->logical.lbn, thread_id, - packer_thread); + VDO_ASSERT_LOG_ONLY((packer_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on packer thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + packer_thread); } static inline void set_data_vio_packer_callback(struct data_vio *data_vio, @@ -585,10 +585,10 @@ static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio) thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((cpu_thread == thread_id), - "data_vio for logical block %llu on thread %u, should be on cpu thread %u", - (unsigned long long) data_vio->logical.lbn, thread_id, - cpu_thread); + VDO_ASSERT_LOG_ONLY((cpu_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on cpu thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + cpu_thread); } static inline void set_data_vio_cpu_callback(struct data_vio *data_vio, diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 7cdbe825116f..52bdf657db64 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -327,8 +327,8 @@ static inline struct hash_zones *as_hash_zones(struct vdo_completion *completion static inline void assert_in_hash_zone(struct hash_zone *zone, const char *name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), - "%s called on hash zone thread", name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), + "%s called on hash zone thread", name); } static inline bool change_context_state(struct dedupe_context *context, int old, int new) @@ -404,8 +404,8 @@ static void assert_hash_lock_agent(struct data_vio *data_vio, const char *where) { /* Not safe to access the agent field except from the hash zone. */ assert_data_vio_in_hash_zone(data_vio); - ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent, - "%s must be for the hash lock agent", where); + VDO_ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent, + "%s must be for the hash lock agent", where); } /** @@ -416,9 +416,8 @@ static void assert_hash_lock_agent(struct data_vio *data_vio, const char *where) */ static void set_duplicate_lock(struct hash_lock *hash_lock, struct pbn_lock *pbn_lock) { - ASSERT_LOG_ONLY((hash_lock->duplicate_lock == NULL), - "hash lock must not already hold a duplicate lock"); - + VDO_ASSERT_LOG_ONLY((hash_lock->duplicate_lock == NULL), + "hash lock must not already hold a duplicate lock"); pbn_lock->holder_count += 1; hash_lock->duplicate_lock = pbn_lock; } @@ -446,12 +445,12 @@ static void set_hash_lock(struct data_vio *data_vio, struct hash_lock *new_lock) struct hash_lock *old_lock = data_vio->hash_lock; if (old_lock != NULL) { - ASSERT_LOG_ONLY(data_vio->hash_zone != NULL, - "must have a hash zone when holding a hash lock"); - ASSERT_LOG_ONLY(!list_empty(&data_vio->hash_lock_entry), - "must be on a hash lock ring when holding a hash lock"); - ASSERT_LOG_ONLY(old_lock->reference_count > 0, - "hash lock reference must be counted"); + VDO_ASSERT_LOG_ONLY(data_vio->hash_zone != NULL, + "must have a hash zone when holding a hash lock"); + VDO_ASSERT_LOG_ONLY(!list_empty(&data_vio->hash_lock_entry), + "must be on a hash lock ring when holding a hash lock"); + VDO_ASSERT_LOG_ONLY(old_lock->reference_count > 0, + "hash lock reference must be counted"); if ((old_lock->state != VDO_HASH_LOCK_BYPASSING) && (old_lock->state != VDO_HASH_LOCK_UNLOCKING)) { @@ -459,9 +458,9 @@ static void set_hash_lock(struct data_vio *data_vio, struct hash_lock *new_lock) * If the reference count goes to zero in a non-terminal state, we're most * likely leaking this lock. */ - ASSERT_LOG_ONLY(old_lock->reference_count > 1, - "hash locks should only become unreferenced in a terminal state, not state %s", - get_hash_lock_state_name(old_lock->state)); + VDO_ASSERT_LOG_ONLY(old_lock->reference_count > 1, + "hash locks should only become unreferenced in a terminal state, not state %s", + get_hash_lock_state_name(old_lock->state)); } list_del_init(&data_vio->hash_lock_entry); @@ -641,8 +640,8 @@ static void finish_unlocking(struct vdo_completion *completion) assert_hash_lock_agent(agent, __func__); - ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, - "must have released the duplicate lock for the hash lock"); + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must have released the duplicate lock for the hash lock"); if (!lock->verified) { /* @@ -696,8 +695,8 @@ static void unlock_duplicate_pbn(struct vdo_completion *completion) struct hash_lock *lock = agent->hash_lock; assert_data_vio_in_duplicate_zone(agent); - ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, - "must have a duplicate lock to release"); + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, + "must have a duplicate lock to release"); vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, agent->duplicate.pbn, vdo_forget(lock->duplicate_lock)); @@ -799,8 +798,8 @@ static void start_updating(struct hash_lock *lock, struct data_vio *agent) { lock->state = VDO_HASH_LOCK_UPDATING; - ASSERT_LOG_ONLY(lock->verified, "new advice should have been verified"); - ASSERT_LOG_ONLY(lock->update_advice, "should only update advice if needed"); + VDO_ASSERT_LOG_ONLY(lock->verified, "new advice should have been verified"); + VDO_ASSERT_LOG_ONLY(lock->update_advice, "should only update advice if needed"); agent->last_async_operation = VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX; set_data_vio_hash_zone_callback(agent, finish_updating); @@ -822,9 +821,9 @@ static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio) { struct data_vio *agent = data_vio; - ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING"); - ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), - "shouldn't have any lock waiters in DEDUPING"); + VDO_ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING"); + VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), + "shouldn't have any lock waiters in DEDUPING"); /* Just release the lock reference if other data_vios are still deduping. */ if (lock->reference_count > 1) { @@ -879,8 +878,8 @@ static int __must_check acquire_lock(struct hash_zone *zone, * Borrow and prepare a lock from the pool so we don't have to do two int_map accesses * in the common case of no lock contention. */ - result = ASSERT(!list_empty(&zone->lock_pool), - "never need to wait for a free hash lock"); + result = VDO_ASSERT(!list_empty(&zone->lock_pool), + "never need to wait for a free hash lock"); if (result != VDO_SUCCESS) return result; @@ -902,11 +901,11 @@ static int __must_check acquire_lock(struct hash_zone *zone, if (replace_lock != NULL) { /* On mismatch put the old lock back and return a severe error */ - ASSERT_LOG_ONLY(lock == replace_lock, - "old lock must have been in the lock map"); + VDO_ASSERT_LOG_ONLY(lock == replace_lock, + "old lock must have been in the lock map"); /* TODO: Check earlier and bail out? */ - ASSERT_LOG_ONLY(replace_lock->registered, - "old lock must have been marked registered"); + VDO_ASSERT_LOG_ONLY(replace_lock->registered, + "old lock must have been marked registered"); replace_lock->registered = false; } @@ -1018,15 +1017,15 @@ static void start_deduping(struct hash_lock *lock, struct data_vio *agent, * deduplicate against it. */ if (lock->duplicate_lock == NULL) { - ASSERT_LOG_ONLY(!vdo_is_state_compressed(agent->new_mapped.state), - "compression must have shared a lock"); - ASSERT_LOG_ONLY(agent_is_done, - "agent must have written the new duplicate"); + VDO_ASSERT_LOG_ONLY(!vdo_is_state_compressed(agent->new_mapped.state), + "compression must have shared a lock"); + VDO_ASSERT_LOG_ONLY(agent_is_done, + "agent must have written the new duplicate"); transfer_allocation_lock(agent); } - ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(lock->duplicate_lock), - "duplicate_lock must be a PBN read lock"); + VDO_ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(lock->duplicate_lock), + "duplicate_lock must be a PBN read lock"); /* * This state is not like any of the other states. There is no designated agent--the agent @@ -1204,7 +1203,7 @@ static void start_verifying(struct hash_lock *lock, struct data_vio *agent) agent->scratch_block); lock->state = VDO_HASH_LOCK_VERIFYING; - ASSERT_LOG_ONLY(!lock->verified, "hash lock only verifies advice once"); + VDO_ASSERT_LOG_ONLY(!lock->verified, "hash lock only verifies advice once"); agent->last_async_operation = VIO_ASYNC_OP_VERIFY_DUPLICATION; result = vio_reset_bio(vio, buffer, verify_endio, REQ_OP_READ, @@ -1234,8 +1233,8 @@ static void finish_locking(struct vdo_completion *completion) assert_hash_lock_agent(agent, __func__); if (!agent->is_duplicate) { - ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, - "must not hold duplicate_lock if not flagged as a duplicate"); + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must not hold duplicate_lock if not flagged as a duplicate"); /* * LOCKING -> WRITING transition: The advice block is being modified or has no * available references, so try to write or compress the data, remembering to @@ -1247,8 +1246,8 @@ static void finish_locking(struct vdo_completion *completion) return; } - ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, - "must hold duplicate_lock if flagged as a duplicate"); + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, + "must hold duplicate_lock if flagged as a duplicate"); if (!lock->verified) { /* @@ -1418,8 +1417,8 @@ static void lock_duplicate_pbn(struct vdo_completion *completion) */ static void start_locking(struct hash_lock *lock, struct data_vio *agent) { - ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, - "must not acquire a duplicate lock when already holding it"); + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must not acquire a duplicate lock when already holding it"); lock->state = VDO_HASH_LOCK_LOCKING; @@ -1725,8 +1724,8 @@ static void start_querying(struct hash_lock *lock, struct data_vio *data_vio) */ static void report_bogus_lock_state(struct hash_lock *lock, struct data_vio *data_vio) { - ASSERT_LOG_ONLY(false, "hash lock must not be in unimplemented state %s", - get_hash_lock_state_name(lock->state)); + VDO_ASSERT_LOG_ONLY(false, "hash lock must not be in unimplemented state %s", + get_hash_lock_state_name(lock->state)); continue_data_vio_with_error(data_vio, VDO_LOCK_ERROR); } @@ -1748,8 +1747,8 @@ void vdo_continue_hash_lock(struct vdo_completion *completion) switch (lock->state) { case VDO_HASH_LOCK_WRITING: - ASSERT_LOG_ONLY(data_vio == lock->agent, - "only the lock agent may continue the lock"); + VDO_ASSERT_LOG_ONLY(data_vio == lock->agent, + "only the lock agent may continue the lock"); finish_writing(lock, data_vio); break; @@ -1815,18 +1814,18 @@ static inline int assert_hash_lock_preconditions(const struct data_vio *data_vio int result; /* FIXME: BUG_ON() and/or enter read-only mode? */ - result = ASSERT(data_vio->hash_lock == NULL, - "must not already hold a hash lock"); + result = VDO_ASSERT(data_vio->hash_lock == NULL, + "must not already hold a hash lock"); if (result != VDO_SUCCESS) return result; - result = ASSERT(list_empty(&data_vio->hash_lock_entry), - "must not already be a member of a hash lock ring"); + result = VDO_ASSERT(list_empty(&data_vio->hash_lock_entry), + "must not already be a member of a hash lock ring"); if (result != VDO_SUCCESS) return result; - return ASSERT(data_vio->recovery_sequence_number == 0, - "must not hold a recovery lock when getting a hash lock"); + return VDO_ASSERT(data_vio->recovery_sequence_number == 0, + "must not hold a recovery lock when getting a hash lock"); } /** @@ -1933,24 +1932,24 @@ void vdo_release_hash_lock(struct data_vio *data_vio) struct hash_lock *removed; removed = vdo_int_map_remove(zone->hash_lock_map, lock_key); - ASSERT_LOG_ONLY(lock == removed, - "hash lock being released must have been mapped"); + VDO_ASSERT_LOG_ONLY(lock == removed, + "hash lock being released must have been mapped"); } else { - ASSERT_LOG_ONLY(lock != vdo_int_map_get(zone->hash_lock_map, lock_key), - "unregistered hash lock must not be in the lock map"); + VDO_ASSERT_LOG_ONLY(lock != vdo_int_map_get(zone->hash_lock_map, lock_key), + "unregistered hash lock must not be in the lock map"); } - ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), - "hash lock returned to zone must have no waiters"); - ASSERT_LOG_ONLY((lock->duplicate_lock == NULL), - "hash lock returned to zone must not reference a PBN lock"); - ASSERT_LOG_ONLY((lock->state == VDO_HASH_LOCK_BYPASSING), - "returned hash lock must not be in use with state %s", - get_hash_lock_state_name(lock->state)); - ASSERT_LOG_ONLY(list_empty(&lock->pool_node), - "hash lock returned to zone must not be in a pool ring"); - ASSERT_LOG_ONLY(list_empty(&lock->duplicate_ring), - "hash lock returned to zone must not reference DataVIOs"); + VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), + "hash lock returned to zone must have no waiters"); + VDO_ASSERT_LOG_ONLY((lock->duplicate_lock == NULL), + "hash lock returned to zone must not reference a PBN lock"); + VDO_ASSERT_LOG_ONLY((lock->state == VDO_HASH_LOCK_BYPASSING), + "returned hash lock must not be in use with state %s", + get_hash_lock_state_name(lock->state)); + VDO_ASSERT_LOG_ONLY(list_empty(&lock->pool_node), + "hash lock returned to zone must not be in a pool ring"); + VDO_ASSERT_LOG_ONLY(list_empty(&lock->duplicate_ring), + "hash lock returned to zone must not reference DataVIOs"); return_hash_lock_to_pool(zone, lock); } @@ -1965,13 +1964,13 @@ static void transfer_allocation_lock(struct data_vio *data_vio) struct allocation *allocation = &data_vio->allocation; struct hash_lock *hash_lock = data_vio->hash_lock; - ASSERT_LOG_ONLY(data_vio->new_mapped.pbn == allocation->pbn, - "transferred lock must be for the block written"); + VDO_ASSERT_LOG_ONLY(data_vio->new_mapped.pbn == allocation->pbn, + "transferred lock must be for the block written"); allocation->pbn = VDO_ZERO_BLOCK; - ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(allocation->lock), - "must have downgraded the allocation lock before transfer"); + VDO_ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(allocation->lock), + "must have downgraded the allocation lock before transfer"); hash_lock->duplicate = data_vio->new_mapped; data_vio->duplicate = data_vio->new_mapped; @@ -1997,10 +1996,10 @@ void vdo_share_compressed_write_lock(struct data_vio *data_vio, { bool claimed; - ASSERT_LOG_ONLY(vdo_get_duplicate_lock(data_vio) == NULL, - "a duplicate PBN lock should not exist when writing"); - ASSERT_LOG_ONLY(vdo_is_state_compressed(data_vio->new_mapped.state), - "lock transfer must be for a compressed write"); + VDO_ASSERT_LOG_ONLY(vdo_get_duplicate_lock(data_vio) == NULL, + "a duplicate PBN lock should not exist when writing"); + VDO_ASSERT_LOG_ONLY(vdo_is_state_compressed(data_vio->new_mapped.state), + "lock transfer must be for a compressed write"); assert_data_vio_in_new_mapped_zone(data_vio); /* First sharer downgrades the lock. */ @@ -2020,7 +2019,7 @@ void vdo_share_compressed_write_lock(struct data_vio *data_vio, * deduplicating against it before our incRef. */ claimed = vdo_claim_pbn_lock_increment(pbn_lock); - ASSERT_LOG_ONLY(claimed, "impossible to fail to claim an initial increment"); + VDO_ASSERT_LOG_ONLY(claimed, "impossible to fail to claim an initial increment"); } static void dedupe_kobj_release(struct kobject *directory) @@ -2296,8 +2295,8 @@ static void finish_index_operation(struct uds_request *request) */ if (!change_context_state(context, DEDUPE_CONTEXT_TIMED_OUT, DEDUPE_CONTEXT_TIMED_OUT_COMPLETE)) { - ASSERT_LOG_ONLY(false, "uds request was timed out (state %d)", - atomic_read(&context->state)); + VDO_ASSERT_LOG_ONLY(false, "uds request was timed out (state %d)", + atomic_read(&context->state)); } uds_funnel_queue_put(context->zone->timed_out_complete, &context->queue_entry); @@ -2341,7 +2340,7 @@ static void check_for_drain_complete(struct hash_zone *zone) if (recycled > 0) WRITE_ONCE(zone->active, zone->active - recycled); - ASSERT_LOG_ONLY(READ_ONCE(zone->active) == 0, "all contexts inactive"); + VDO_ASSERT_LOG_ONLY(READ_ONCE(zone->active) == 0, "all contexts inactive"); vdo_finish_draining(&zone->state); } diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 90ba379f8d3e..e493b2fec90b 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -904,8 +904,8 @@ static int vdo_map_bio(struct dm_target *ti, struct bio *bio) struct vdo_work_queue *current_work_queue; const struct admin_state_code *code = vdo_get_admin_state_code(&vdo->admin.state); - ASSERT_LOG_ONLY(code->normal, "vdo should not receive bios while in state %s", - code->name); + VDO_ASSERT_LOG_ONLY(code->normal, "vdo should not receive bios while in state %s", + code->name); /* Count all incoming bios. */ vdo_count_bios(&vdo->stats.bios_in, bio); @@ -1244,9 +1244,9 @@ static int perform_admin_operation(struct vdo *vdo, u32 starting_phase, /* Assert that we are operating on the correct thread for the current phase. */ static void assert_admin_phase_thread(struct vdo *vdo, const char *what) { - ASSERT_LOG_ONLY(vdo_get_callback_thread_id() == get_thread_id_for_phase(vdo), - "%s on correct thread for %s", what, - ADMIN_PHASE_NAMES[vdo->admin.phase]); + VDO_ASSERT_LOG_ONLY(vdo_get_callback_thread_id() == get_thread_id_for_phase(vdo), + "%s on correct thread for %s", what, + ADMIN_PHASE_NAMES[vdo->admin.phase]); } /** @@ -1424,11 +1424,11 @@ static void release_instance(unsigned int instance) { mutex_lock(&instances_lock); if (instance >= instances.bit_count) { - ASSERT_LOG_ONLY(false, - "instance number %u must be less than bit count %u", - instance, instances.bit_count); + VDO_ASSERT_LOG_ONLY(false, + "instance number %u must be less than bit count %u", + instance, instances.bit_count); } else if (test_bit(instance, instances.words) == 0) { - ASSERT_LOG_ONLY(false, "instance number %u must be allocated", instance); + VDO_ASSERT_LOG_ONLY(false, "instance number %u must be allocated", instance); } else { __clear_bit(instance, instances.words); instances.count -= 1; @@ -1577,9 +1577,9 @@ static int allocate_instance(unsigned int *instance_ptr) if (instance >= instances.bit_count) { /* Nothing free after next, so wrap around to instance zero. */ instance = find_first_zero_bit(instances.words, instances.bit_count); - result = ASSERT(instance < instances.bit_count, - "impossibly, no zero bit found"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(instance < instances.bit_count, + "impossibly, no zero bit found"); + if (result != VDO_SUCCESS) return result; } @@ -1729,8 +1729,8 @@ static int prepare_to_grow_physical(struct vdo *vdo, block_count_t new_physical_ uds_log_info("Preparing to resize physical to %llu", (unsigned long long) new_physical_blocks); - ASSERT_LOG_ONLY((new_physical_blocks > current_physical_blocks), - "New physical size is larger than current physical size"); + VDO_ASSERT_LOG_ONLY((new_physical_blocks > current_physical_blocks), + "New physical size is larger than current physical size"); result = perform_admin_operation(vdo, PREPARE_GROW_PHYSICAL_PHASE_START, check_may_grow_physical, finish_operation_callback, @@ -1829,8 +1829,8 @@ static int prepare_to_modify(struct dm_target *ti, struct device_config *config, uds_log_info("Preparing to resize logical to %llu", (unsigned long long) config->logical_blocks); - ASSERT_LOG_ONLY((config->logical_blocks > logical_blocks), - "New logical size is larger than current size"); + VDO_ASSERT_LOG_ONLY((config->logical_blocks > logical_blocks), + "New logical size is larger than current size"); result = vdo_prepare_to_grow_block_map(vdo->block_map, config->logical_blocks); @@ -2890,9 +2890,9 @@ static void vdo_module_destroy(void) if (dm_registered) dm_unregister_target(&vdo_target_bio); - ASSERT_LOG_ONLY(instances.count == 0, - "should have no instance numbers still in use, but have %u", - instances.count); + VDO_ASSERT_LOG_ONLY(instances.count == 0, + "should have no instance numbers still in use, but have %u", + instances.count); vdo_free(instances.words); memset(&instances, 0, sizeof(struct instance_tracker)); diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index a97771fe0a43..e24c31bc3524 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -320,8 +320,8 @@ int __must_check vdo_parse_geometry_block(u8 *block, struct volume_geometry *geo decode_volume_geometry(block, &offset, geometry, header.version.major_version); - result = ASSERT(header.size == offset + sizeof(u32), - "should have decoded up to the geometry checksum"); + result = VDO_ASSERT(header.size == offset + sizeof(u32), + "should have decoded up to the geometry checksum"); if (result != VDO_SUCCESS) return result; @@ -380,25 +380,25 @@ static int decode_block_map_state_2_0(u8 *buffer, size_t *offset, initial_offset = *offset; decode_u64_le(buffer, offset, &flat_page_origin); - result = ASSERT(flat_page_origin == VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, - "Flat page origin must be %u (recorded as %llu)", - VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, - (unsigned long long) state->flat_page_origin); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(flat_page_origin == VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + "Flat page origin must be %u (recorded as %llu)", + VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + (unsigned long long) state->flat_page_origin); + if (result != VDO_SUCCESS) return result; decode_u64_le(buffer, offset, &flat_page_count); - result = ASSERT(flat_page_count == 0, - "Flat page count must be 0 (recorded as %llu)", - (unsigned long long) state->flat_page_count); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(flat_page_count == 0, + "Flat page count must be 0 (recorded as %llu)", + (unsigned long long) state->flat_page_count); + if (result != VDO_SUCCESS) return result; decode_u64_le(buffer, offset, &root_origin); decode_u64_le(buffer, offset, &root_count); - result = ASSERT(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, - "decoded block map component size must match header size"); + result = VDO_ASSERT(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, + "decoded block map component size must match header size"); if (result != VDO_SUCCESS) return result; @@ -425,8 +425,8 @@ static void encode_block_map_state_2_0(u8 *buffer, size_t *offset, encode_u64_le(buffer, offset, state.root_origin); encode_u64_le(buffer, offset, state.root_count); - ASSERT_LOG_ONLY(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, - "encoded block map component size must match header size"); + VDO_ASSERT_LOG_ONLY(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, + "encoded block map component size must match header size"); } /** @@ -477,8 +477,8 @@ static void encode_recovery_journal_state_7_0(u8 *buffer, size_t *offset, encode_u64_le(buffer, offset, state.logical_blocks_used); encode_u64_le(buffer, offset, state.block_map_data_blocks); - ASSERT_LOG_ONLY(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, - "encoded recovery journal component size must match header size"); + VDO_ASSERT_LOG_ONLY(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, + "encoded recovery journal component size must match header size"); } /** @@ -508,9 +508,9 @@ static int __must_check decode_recovery_journal_state_7_0(u8 *buffer, size_t *of decode_u64_le(buffer, offset, &logical_blocks_used); decode_u64_le(buffer, offset, &block_map_data_blocks); - result = ASSERT(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, - "decoded recovery journal component size must match header size"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, + "decoded recovery journal component size must match header size"); + if (result != VDO_SUCCESS) return result; *state = (struct recovery_journal_state_7_0) { @@ -566,8 +566,8 @@ static void encode_slab_depot_state_2_0(u8 *buffer, size_t *offset, encode_u64_le(buffer, offset, state.last_block); buffer[(*offset)++] = state.zone_count; - ASSERT_LOG_ONLY(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, - "encoded block map component size must match header size"); + VDO_ASSERT_LOG_ONLY(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, + "encoded block map component size must match header size"); } /** @@ -618,9 +618,9 @@ static int decode_slab_depot_state_2_0(u8 *buffer, size_t *offset, decode_u64_le(buffer, offset, &last_block); zone_count = buffer[(*offset)++]; - result = ASSERT(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, - "decoded slab depot component size must match header size"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, + "decoded slab depot component size must match header size"); + if (result != VDO_SUCCESS) return result; *state = (struct slab_depot_state_2_0) { @@ -970,7 +970,7 @@ struct partition *vdo_get_known_partition(struct layout *layout, enum partition_ struct partition *partition; int result = vdo_get_partition(layout, id, &partition); - ASSERT_LOG_ONLY(result == VDO_SUCCESS, "layout has expected partition: %u", id); + VDO_ASSERT_LOG_ONLY(result == VDO_SUCCESS, "layout has expected partition: %u", id); return partition; } @@ -982,8 +982,8 @@ static void encode_layout(u8 *buffer, size_t *offset, const struct layout *layou struct header header = VDO_LAYOUT_HEADER_3_0; BUILD_BUG_ON(sizeof(enum partition_id) != sizeof(u8)); - ASSERT_LOG_ONLY(layout->num_partitions <= U8_MAX, - "layout partition count must fit in a byte"); + VDO_ASSERT_LOG_ONLY(layout->num_partitions <= U8_MAX, + "layout partition count must fit in a byte"); vdo_encode_header(buffer, offset, &header); @@ -992,8 +992,8 @@ static void encode_layout(u8 *buffer, size_t *offset, const struct layout *layou encode_u64_le(buffer, offset, layout->last_free); buffer[(*offset)++] = layout->num_partitions; - ASSERT_LOG_ONLY(sizeof(struct layout_3_0) == *offset - initial_offset, - "encoded size of a layout header must match structure"); + VDO_ASSERT_LOG_ONLY(sizeof(struct layout_3_0) == *offset - initial_offset, + "encoded size of a layout header must match structure"); for (partition = layout->head; partition != NULL; partition = partition->next) { buffer[(*offset)++] = partition->id; @@ -1003,8 +1003,8 @@ static void encode_layout(u8 *buffer, size_t *offset, const struct layout *layou encode_u64_le(buffer, offset, partition->count); } - ASSERT_LOG_ONLY(header.size == *offset - initial_offset, - "encoded size of a layout must match header size"); + VDO_ASSERT_LOG_ONLY(header.size == *offset - initial_offset, + "encoded size of a layout must match header size"); } static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t start, @@ -1035,8 +1035,8 @@ static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t sta .partition_count = partition_count, }; - result = ASSERT(sizeof(struct layout_3_0) == *offset - initial_offset, - "decoded size of a layout header must match structure"); + result = VDO_ASSERT(sizeof(struct layout_3_0) == *offset - initial_offset, + "decoded size of a layout header must match structure"); if (result != VDO_SUCCESS) return result; @@ -1208,29 +1208,29 @@ int vdo_validate_config(const struct vdo_config *config, struct slab_config slab_config; int result; - result = ASSERT(config->slab_size > 0, "slab size unspecified"); - if (result != UDS_SUCCESS) - return result; - - result = ASSERT(is_power_of_2(config->slab_size), - "slab size must be a power of two"); - if (result != UDS_SUCCESS) - return result; - - result = ASSERT(config->slab_size <= (1 << MAX_VDO_SLAB_BITS), - "slab size must be less than or equal to 2^%d", - MAX_VDO_SLAB_BITS); + result = VDO_ASSERT(config->slab_size > 0, "slab size unspecified"); if (result != VDO_SUCCESS) return result; - result = ASSERT(config->slab_journal_blocks >= MINIMUM_VDO_SLAB_JOURNAL_BLOCKS, - "slab journal size meets minimum size"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(is_power_of_2(config->slab_size), + "slab size must be a power of two"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(config->slab_journal_blocks <= config->slab_size, - "slab journal size is within expected bound"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(config->slab_size <= (1 << MAX_VDO_SLAB_BITS), + "slab size must be less than or equal to 2^%d", + MAX_VDO_SLAB_BITS); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->slab_journal_blocks >= MINIMUM_VDO_SLAB_JOURNAL_BLOCKS, + "slab journal size meets minimum size"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->slab_journal_blocks <= config->slab_size, + "slab journal size is within expected bound"); + if (result != VDO_SUCCESS) return result; result = vdo_configure_slab(config->slab_size, config->slab_journal_blocks, @@ -1238,20 +1238,20 @@ int vdo_validate_config(const struct vdo_config *config, if (result != VDO_SUCCESS) return result; - result = ASSERT((slab_config.data_blocks >= 1), - "slab must be able to hold at least one block"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((slab_config.data_blocks >= 1), + "slab must be able to hold at least one block"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(config->physical_blocks > 0, "physical blocks unspecified"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(config->physical_blocks > 0, "physical blocks unspecified"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(config->physical_blocks <= MAXIMUM_VDO_PHYSICAL_BLOCKS, - "physical block count %llu exceeds maximum %llu", - (unsigned long long) config->physical_blocks, - (unsigned long long) MAXIMUM_VDO_PHYSICAL_BLOCKS); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(config->physical_blocks <= MAXIMUM_VDO_PHYSICAL_BLOCKS, + "physical block count %llu exceeds maximum %llu", + (unsigned long long) config->physical_blocks, + (unsigned long long) MAXIMUM_VDO_PHYSICAL_BLOCKS); + if (result != VDO_SUCCESS) return VDO_OUT_OF_RANGE; if (physical_block_count != config->physical_blocks) { @@ -1262,9 +1262,9 @@ int vdo_validate_config(const struct vdo_config *config, } if (logical_block_count > 0) { - result = ASSERT((config->logical_blocks > 0), - "logical blocks unspecified"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((config->logical_blocks > 0), + "logical blocks unspecified"); + if (result != VDO_SUCCESS) return result; if (logical_block_count != config->logical_blocks) { @@ -1275,19 +1275,19 @@ int vdo_validate_config(const struct vdo_config *config, } } - result = ASSERT(config->logical_blocks <= MAXIMUM_VDO_LOGICAL_BLOCKS, - "logical blocks too large"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(config->logical_blocks <= MAXIMUM_VDO_LOGICAL_BLOCKS, + "logical blocks too large"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(config->recovery_journal_size > 0, - "recovery journal size unspecified"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(config->recovery_journal_size > 0, + "recovery journal size unspecified"); + if (result != VDO_SUCCESS) return result; - result = ASSERT(is_power_of_2(config->recovery_journal_size), - "recovery journal size must be a power of two"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(is_power_of_2(config->recovery_journal_size), + "recovery journal size must be a power of two"); + if (result != VDO_SUCCESS) return result; return result; @@ -1341,8 +1341,8 @@ static int __must_check decode_components(u8 *buffer, size_t *offset, if (result != VDO_SUCCESS) return result; - ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, - "All decoded component data was used"); + VDO_ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, + "All decoded component data was used"); return VDO_SUCCESS; } @@ -1416,8 +1416,8 @@ static void vdo_encode_component_states(u8 *buffer, size_t *offset, encode_slab_depot_state_2_0(buffer, offset, states->slab_depot); encode_block_map_state_2_0(buffer, offset, states->block_map); - ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, - "All super block component data was encoded"); + VDO_ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, + "All super block component data was encoded"); } /** @@ -1440,8 +1440,8 @@ void vdo_encode_super_block(u8 *buffer, struct vdo_component_states *states) * Even though the buffer is a full block, to avoid the potential corruption from a torn * write, the entire encoding must fit in the first sector. */ - ASSERT_LOG_ONLY(offset <= VDO_SECTOR_SIZE, - "entire superblock must fit in one sector"); + VDO_ASSERT_LOG_ONLY(offset <= VDO_SECTOR_SIZE, + "entire superblock must fit in one sector"); } /** @@ -1476,8 +1476,8 @@ int vdo_decode_super_block(u8 *buffer) checksum = vdo_crc32(buffer, offset); decode_u32_le(buffer, &offset, &saved_checksum); - result = ASSERT(offset == VDO_SUPER_BLOCK_FIXED_SIZE + VDO_COMPONENT_DATA_SIZE, - "must have decoded entire superblock payload"); + result = VDO_ASSERT(offset == VDO_SUPER_BLOCK_FIXED_SIZE + VDO_COMPONENT_DATA_SIZE, + "must have decoded entire superblock payload"); if (result != VDO_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c index df2498553312..3b5fddad8ddf 100644 --- a/drivers/md/dm-vdo/errors.c +++ b/drivers/md/dm-vdo/errors.c @@ -281,8 +281,9 @@ int uds_register_error_block(const char *block_name, int first_error, .infos = infos, }; - result = ASSERT(first_error < next_free_error, "well-defined error block range"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(first_error < next_free_error, + "well-defined error block range"); + if (result != VDO_SUCCESS) return result; if (registered_errors.count == registered_errors.allocated) { diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index 8d8d9cf4a24c..e03679e4d1ba 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -59,8 +59,8 @@ struct flusher { */ static inline void assert_on_flusher_thread(struct flusher *flusher, const char *caller) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == flusher->thread_id), - "%s() called from flusher thread", caller); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == flusher->thread_id), + "%s() called from flusher thread", caller); } /** @@ -272,8 +272,8 @@ static void flush_vdo(struct vdo_completion *completion) int result; assert_on_flusher_thread(flusher, __func__); - result = ASSERT(vdo_is_state_normal(&flusher->state), - "flusher is in normal operation"); + result = VDO_ASSERT(vdo_is_state_normal(&flusher->state), + "flusher is in normal operation"); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(flusher->vdo, result); vdo_complete_flush(flush); @@ -330,11 +330,11 @@ void vdo_complete_flushes(struct flusher *flusher) if (flush->flush_generation >= oldest_active_generation) return; - ASSERT_LOG_ONLY((flush->flush_generation == - flusher->first_unacknowledged_generation), - "acknowledged next expected flush, %llu, was: %llu", - (unsigned long long) flusher->first_unacknowledged_generation, - (unsigned long long) flush->flush_generation); + VDO_ASSERT_LOG_ONLY((flush->flush_generation == + flusher->first_unacknowledged_generation), + "acknowledged next expected flush, %llu, was: %llu", + (unsigned long long) flusher->first_unacknowledged_generation, + (unsigned long long) flush->flush_generation); vdo_waitq_dequeue_waiter(&flusher->pending_flushes); vdo_complete_flush(flush); flusher->first_unacknowledged_generation++; @@ -400,8 +400,8 @@ void vdo_launch_flush(struct vdo *vdo, struct bio *bio) struct flusher *flusher = vdo->flusher; const struct admin_state_code *code = vdo_get_admin_state_code(&flusher->state); - ASSERT_LOG_ONLY(!code->quiescent, "Flushing not allowed in state %s", - code->name); + VDO_ASSERT_LOG_ONLY(!code->quiescent, "Flushing not allowed in state %s", + code->name); spin_lock(&flusher->lock); diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c index a923432f0a37..03296e7fec12 100644 --- a/drivers/md/dm-vdo/funnel-workqueue.c +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -110,14 +110,14 @@ static struct vdo_completion *poll_for_completion(struct simple_work_queue *queu static void enqueue_work_queue_completion(struct simple_work_queue *queue, struct vdo_completion *completion) { - ASSERT_LOG_ONLY(completion->my_queue == NULL, - "completion %px (fn %px) to enqueue (%px) is not already queued (%px)", - completion, completion->callback, queue, completion->my_queue); + VDO_ASSERT_LOG_ONLY(completion->my_queue == NULL, + "completion %px (fn %px) to enqueue (%px) is not already queued (%px)", + completion, completion->callback, queue, completion->my_queue); if (completion->priority == VDO_WORK_Q_DEFAULT_PRIORITY) completion->priority = queue->common.type->default_priority; - if (ASSERT(completion->priority <= queue->common.type->max_priority, - "priority is in range for queue") != VDO_SUCCESS) + if (VDO_ASSERT(completion->priority <= queue->common.type->max_priority, + "priority is in range for queue") != VDO_SUCCESS) completion->priority = 0; completion->my_queue = &queue->common; @@ -222,9 +222,9 @@ static struct vdo_completion *wait_for_next_completion(struct simple_work_queue static void process_completion(struct simple_work_queue *queue, struct vdo_completion *completion) { - if (ASSERT(completion->my_queue == &queue->common, - "completion %px from queue %px marked as being in this queue (%px)", - completion, queue, completion->my_queue) == UDS_SUCCESS) + if (VDO_ASSERT(completion->my_queue == &queue->common, + "completion %px from queue %px marked as being in this queue (%px)", + completion, queue, completion->my_queue) == VDO_SUCCESS) completion->my_queue = NULL; vdo_run_completion(completion); @@ -319,9 +319,9 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na struct task_struct *thread = NULL; int result; - ASSERT_LOG_ONLY((type->max_priority <= VDO_WORK_Q_MAX_PRIORITY), - "queue priority count %u within limit %u", type->max_priority, - VDO_WORK_Q_MAX_PRIORITY); + VDO_ASSERT_LOG_ONLY((type->max_priority <= VDO_WORK_Q_MAX_PRIORITY), + "queue priority count %u within limit %u", type->max_priority, + VDO_WORK_Q_MAX_PRIORITY); result = vdo_allocate(1, struct simple_work_queue, "simple work queue", &queue); if (result != VDO_SUCCESS) diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index e82b4a8c6fc4..61bb48068c3a 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -94,7 +94,7 @@ static void count_all_bios(struct vio *vio, struct bio *bio) */ static void assert_in_bio_zone(struct vio *vio) { - ASSERT_LOG_ONLY(!in_interrupt(), "not in interrupt context"); + VDO_ASSERT_LOG_ONLY(!in_interrupt(), "not in interrupt context"); assert_vio_in_bio_zone(vio); } @@ -300,7 +300,7 @@ static bool try_bio_map_merge(struct vio *vio) mutex_unlock(&bio_queue_data->lock); /* We don't care about failure of int_map_put in this case. */ - ASSERT_LOG_ONLY(result == VDO_SUCCESS, "bio map insertion succeeds"); + VDO_ASSERT_LOG_ONLY(result == VDO_SUCCESS, "bio map insertion succeeds"); return merged; } @@ -345,8 +345,8 @@ void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, const struct admin_state_code *code = vdo_get_admin_state(completion->vdo); - ASSERT_LOG_ONLY(!code->quiescent, "I/O not allowed in state %s", code->name); - ASSERT_LOG_ONLY(vio->bio->bi_next == NULL, "metadata bio has no next bio"); + VDO_ASSERT_LOG_ONLY(!code->quiescent, "I/O not allowed in state %s", code->name); + VDO_ASSERT_LOG_ONLY(vio->bio->bi_next == NULL, "metadata bio has no next bio"); vdo_reset_completion(completion); completion->error_handler = error_handler; diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index ca5bc3be7978..300f9d2d2d5c 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -142,8 +142,8 @@ void vdo_free_logical_zones(struct logical_zones *zones) static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), - "%s() called on correct thread", what); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), + "%s() called on correct thread", what); } /** @@ -247,10 +247,10 @@ void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, sequence_number_t expected_generation) { assert_on_zone_thread(zone, __func__); - ASSERT_LOG_ONLY((zone->flush_generation == expected_generation), - "logical zone %u flush generation %llu should be %llu before increment", - zone->zone_number, (unsigned long long) zone->flush_generation, - (unsigned long long) expected_generation); + VDO_ASSERT_LOG_ONLY((zone->flush_generation == expected_generation), + "logical zone %u flush generation %llu should be %llu before increment", + zone->zone_number, (unsigned long long) zone->flush_generation, + (unsigned long long) expected_generation); zone->flush_generation++; zone->ios_in_flush_generation = 0; @@ -267,7 +267,7 @@ void vdo_acquire_flush_generation_lock(struct data_vio *data_vio) struct logical_zone *zone = data_vio->logical.zone; assert_on_zone_thread(zone, __func__); - ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal"); + VDO_ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal"); data_vio->flush_generation = zone->flush_generation; list_add_tail(&data_vio->write_entry, &zone->write_vios); @@ -332,10 +332,10 @@ void vdo_release_flush_generation_lock(struct data_vio *data_vio) return; list_del_init(&data_vio->write_entry); - ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation), - "data_vio releasing lock on generation %llu is not older than oldest active generation %llu", - (unsigned long long) data_vio->flush_generation, - (unsigned long long) zone->oldest_active_generation); + VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation), + "data_vio releasing lock on generation %llu is not older than oldest active generation %llu", + (unsigned long long) data_vio->flush_generation, + (unsigned long long) zone->oldest_active_generation); if (!update_oldest_active_generation(zone) || zone->notifying) return; diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index dd5acc582fb3..62bb717c4c50 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -385,12 +385,12 @@ void vdo_memory_init(void) void vdo_memory_exit(void) { - ASSERT_LOG_ONLY(memory_stats.kmalloc_bytes == 0, - "kmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", - memory_stats.kmalloc_bytes, memory_stats.kmalloc_blocks); - ASSERT_LOG_ONLY(memory_stats.vmalloc_bytes == 0, - "vmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", - memory_stats.vmalloc_bytes, memory_stats.vmalloc_blocks); + VDO_ASSERT_LOG_ONLY(memory_stats.kmalloc_bytes == 0, + "kmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", + memory_stats.kmalloc_bytes, memory_stats.kmalloc_blocks); + VDO_ASSERT_LOG_ONLY(memory_stats.vmalloc_bytes == 0, + "vmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", + memory_stats.vmalloc_bytes, memory_stats.vmalloc_blocks); uds_log_debug("peak usage %zd bytes", memory_stats.peak_bytes); } diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c index 5774d8fd5c5a..4d45243161a6 100644 --- a/drivers/md/dm-vdo/packer.c +++ b/drivers/md/dm-vdo/packer.c @@ -86,8 +86,8 @@ int vdo_get_compressed_block_fragment(enum block_mapping_state mapping_state, */ static inline void assert_on_packer_thread(struct packer *packer, const char *caller) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == packer->thread_id), - "%s() called from packer thread", caller); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == packer->thread_id), + "%s() called from packer thread", caller); } /** @@ -569,9 +569,9 @@ void vdo_attempt_packing(struct data_vio *data_vio) assert_on_packer_thread(packer, __func__); - result = ASSERT((status.stage == DATA_VIO_COMPRESSING), - "attempt to pack data_vio not ready for packing, stage: %u", - status.stage); + result = VDO_ASSERT((status.stage == DATA_VIO_COMPRESSING), + "attempt to pack data_vio not ready for packing, stage: %u", + status.stage); if (result != VDO_SUCCESS) return; @@ -671,7 +671,7 @@ void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion) lock_holder = vdo_forget(data_vio->compression.lock_holder); bin = lock_holder->compression.bin; - ASSERT_LOG_ONLY((bin != NULL), "data_vio in packer has a bin"); + VDO_ASSERT_LOG_ONLY((bin != NULL), "data_vio in packer has a bin"); slot = lock_holder->compression.slot; bin->slots_used--; diff --git a/drivers/md/dm-vdo/permassert.h b/drivers/md/dm-vdo/permassert.h index ee978bc115ec..8774dde7927a 100644 --- a/drivers/md/dm-vdo/permassert.h +++ b/drivers/md/dm-vdo/permassert.h @@ -13,7 +13,6 @@ /* Utilities for asserting that certain conditions are met */ #define STRINGIFY(X) #X -#define STRINGIFY_VALUE(X) STRINGIFY(X) /* * A hack to apply the "warn if unused" attribute to an integral expression. @@ -23,19 +22,23 @@ * expression. With optimization enabled, this function contributes no additional instructions, but * the warn_unused_result attribute still applies to the code calling it. */ -static inline int __must_check uds_must_use(int value) +static inline int __must_check vdo_must_use(int value) { return value; } /* Assert that an expression is true and return an error if it is not. */ -#define ASSERT(expr, ...) uds_must_use(__UDS_ASSERT(expr, __VA_ARGS__)) +#define VDO_ASSERT(expr, ...) vdo_must_use(__VDO_ASSERT(expr, __VA_ARGS__)) /* Log a message if the expression is not true. */ -#define ASSERT_LOG_ONLY(expr, ...) __UDS_ASSERT(expr, __VA_ARGS__) +#define VDO_ASSERT_LOG_ONLY(expr, ...) __VDO_ASSERT(expr, __VA_ARGS__) -#define __UDS_ASSERT(expr, ...) \ - (likely(expr) ? UDS_SUCCESS \ +/* For use by UDS */ +#define ASSERT(expr, ...) VDO_ASSERT(expr, __VA_ARGS__) +#define ASSERT_LOG_ONLY(expr, ...) __VDO_ASSERT(expr, __VA_ARGS__) + +#define __VDO_ASSERT(expr, ...) \ + (likely(expr) ? VDO_SUCCESS \ : uds_assertion_failed(STRINGIFY(expr), __FILE__, __LINE__, __VA_ARGS__)) /* Log an assertion failure message. */ diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index fadcea23288e..6678f472fb44 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -80,13 +80,13 @@ static inline void set_pbn_lock_type(struct pbn_lock *lock, enum pbn_lock_type t */ void vdo_downgrade_pbn_write_lock(struct pbn_lock *lock, bool compressed_write) { - ASSERT_LOG_ONLY(!vdo_is_pbn_read_lock(lock), - "PBN lock must not already have been downgraded"); - ASSERT_LOG_ONLY(!has_lock_type(lock, VIO_BLOCK_MAP_WRITE_LOCK), - "must not downgrade block map write locks"); - ASSERT_LOG_ONLY(lock->holder_count == 1, - "PBN write lock should have one holder but has %u", - lock->holder_count); + VDO_ASSERT_LOG_ONLY(!vdo_is_pbn_read_lock(lock), + "PBN lock must not already have been downgraded"); + VDO_ASSERT_LOG_ONLY(!has_lock_type(lock, VIO_BLOCK_MAP_WRITE_LOCK), + "must not downgrade block map write locks"); + VDO_ASSERT_LOG_ONLY(lock->holder_count == 1, + "PBN write lock should have one holder but has %u", + lock->holder_count); /* * data_vio write locks are downgraded in place--the writer retains the hold on the lock. * If this was a compressed write, the holder has not yet journaled its own inc ref, @@ -128,8 +128,8 @@ bool vdo_claim_pbn_lock_increment(struct pbn_lock *lock) */ void vdo_assign_pbn_lock_provisional_reference(struct pbn_lock *lock) { - ASSERT_LOG_ONLY(!lock->has_provisional_reference, - "lock does not have a provisional reference"); + VDO_ASSERT_LOG_ONLY(!lock->has_provisional_reference, + "lock does not have a provisional reference"); lock->has_provisional_reference = true; } @@ -221,7 +221,7 @@ static void return_pbn_lock_to_pool(struct pbn_lock_pool *pool, struct pbn_lock INIT_LIST_HEAD(&idle->entry); list_add_tail(&idle->entry, &pool->idle_list); - ASSERT_LOG_ONLY(pool->borrowed > 0, "shouldn't return more than borrowed"); + VDO_ASSERT_LOG_ONLY(pool->borrowed > 0, "shouldn't return more than borrowed"); pool->borrowed -= 1; } @@ -267,9 +267,9 @@ static void free_pbn_lock_pool(struct pbn_lock_pool *pool) if (pool == NULL) return; - ASSERT_LOG_ONLY(pool->borrowed == 0, - "All PBN locks must be returned to the pool before it is freed, but %zu locks are still on loan", - pool->borrowed); + VDO_ASSERT_LOG_ONLY(pool->borrowed == 0, + "All PBN locks must be returned to the pool before it is freed, but %zu locks are still on loan", + pool->borrowed); vdo_free(pool); } @@ -298,8 +298,8 @@ static int __must_check borrow_pbn_lock_from_pool(struct pbn_lock_pool *pool, "no free PBN locks left to borrow"); pool->borrowed += 1; - result = ASSERT(!list_empty(&pool->idle_list), - "idle list should not be empty if pool not at capacity"); + result = VDO_ASSERT(!list_empty(&pool->idle_list), + "idle list should not be empty if pool not at capacity"); if (result != VDO_SUCCESS) return result; @@ -447,7 +447,7 @@ int vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, result = borrow_pbn_lock_from_pool(zone->lock_pool, type, &new_lock); if (result != VDO_SUCCESS) { - ASSERT_LOG_ONLY(false, "must always be able to borrow a PBN lock"); + VDO_ASSERT_LOG_ONLY(false, "must always be able to borrow a PBN lock"); return result; } @@ -461,8 +461,8 @@ int vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, if (lock != NULL) { /* The lock is already held, so we don't need the borrowed one. */ return_pbn_lock_to_pool(zone->lock_pool, vdo_forget(new_lock)); - result = ASSERT(lock->holder_count > 0, "physical block %llu lock held", - (unsigned long long) pbn); + result = VDO_ASSERT(lock->holder_count > 0, "physical block %llu lock held", + (unsigned long long) pbn); if (result != VDO_SUCCESS) return result; *lock_ptr = lock; @@ -485,8 +485,8 @@ static int allocate_and_lock_block(struct allocation *allocation) int result; struct pbn_lock *lock; - ASSERT_LOG_ONLY(allocation->lock == NULL, - "must not allocate a block while already holding a lock on one"); + VDO_ASSERT_LOG_ONLY(allocation->lock == NULL, + "must not allocate a block while already holding a lock on one"); result = vdo_allocate_block(allocation->zone->allocator, &allocation->pbn); if (result != VDO_SUCCESS) @@ -617,8 +617,8 @@ void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, if (lock == NULL) return; - ASSERT_LOG_ONLY(lock->holder_count > 0, - "should not be releasing a lock that is not held"); + VDO_ASSERT_LOG_ONLY(lock->holder_count > 0, + "should not be releasing a lock that is not held"); lock->holder_count -= 1; if (lock->holder_count > 0) { @@ -627,8 +627,8 @@ void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, } holder = vdo_int_map_remove(zone->pbn_operations, locked_pbn); - ASSERT_LOG_ONLY((lock == holder), "physical block lock mismatch for block %llu", - (unsigned long long) locked_pbn); + VDO_ASSERT_LOG_ONLY((lock == holder), "physical block lock mismatch for block %llu", + (unsigned long long) locked_pbn); release_pbn_lock_provisional_reference(lock, locked_pbn, zone->allocator); return_pbn_lock_to_pool(zone->lock_pool, lock); diff --git a/drivers/md/dm-vdo/priority-table.c b/drivers/md/dm-vdo/priority-table.c index fc99268d2437..42d3d8d0e4b5 100644 --- a/drivers/md/dm-vdo/priority-table.c +++ b/drivers/md/dm-vdo/priority-table.c @@ -127,8 +127,8 @@ void vdo_reset_priority_table(struct priority_table *table) void vdo_priority_table_enqueue(struct priority_table *table, unsigned int priority, struct list_head *entry) { - ASSERT_LOG_ONLY((priority <= table->max_priority), - "entry priority must be valid for the table"); + VDO_ASSERT_LOG_ONLY((priority <= table->max_priority), + "entry priority must be valid for the table"); /* Append the entry to the queue in the specified bucket. */ list_move_tail(entry, &table->buckets[priority].queue); diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 615755697e60..6df373b88042 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -119,8 +119,8 @@ static bool is_journal_zone_locked(struct recovery_journal *journal, /* Pairs with barrier in vdo_release_journal_entry_lock() */ smp_rmb(); - ASSERT_LOG_ONLY((decrements <= journal_value), - "journal zone lock counter must not underflow"); + VDO_ASSERT_LOG_ONLY((decrements <= journal_value), + "journal zone lock counter must not underflow"); return (journal_value != decrements); } @@ -150,8 +150,8 @@ void vdo_release_recovery_journal_block_reference(struct recovery_journal *journ lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); current_value = get_counter(journal, lock_number, zone_type, zone_id); - ASSERT_LOG_ONLY((*current_value >= 1), - "decrement of lock counter must not underflow"); + VDO_ASSERT_LOG_ONLY((*current_value >= 1), + "decrement of lock counter must not underflow"); *current_value -= 1; if (zone_type == VDO_ZONE_TYPE_JOURNAL) { @@ -254,8 +254,8 @@ static inline bool __must_check is_block_full(const struct recovery_journal_bloc static void assert_on_journal_thread(struct recovery_journal *journal, const char *function_name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id), - "%s() called on journal thread", function_name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id), + "%s() called on journal thread", function_name); } /** @@ -353,14 +353,14 @@ static void check_for_drain_complete(struct recovery_journal *journal) if (vdo_is_state_saving(&journal->state)) { if (journal->active_block != NULL) { - ASSERT_LOG_ONLY(((result == VDO_READ_ONLY) || - !is_block_dirty(journal->active_block)), - "journal being saved has clean active block"); + VDO_ASSERT_LOG_ONLY(((result == VDO_READ_ONLY) || + !is_block_dirty(journal->active_block)), + "journal being saved has clean active block"); recycle_journal_block(journal->active_block); } - ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), - "all blocks in a journal being saved must be inactive"); + VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), + "all blocks in a journal being saved must be inactive"); } vdo_finish_draining_with_result(&journal->state, result); @@ -800,8 +800,8 @@ void vdo_free_recovery_journal(struct recovery_journal *journal) * requires opening before use. */ if (!vdo_is_state_quiescent(&journal->state)) { - ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), - "journal being freed has no active tail blocks"); + VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), + "journal being freed has no active tail blocks"); } else if (!vdo_is_state_saved(&journal->state) && !list_empty(&journal->active_tail_blocks)) { uds_log_warning("journal being freed has uncommitted entries"); @@ -989,8 +989,8 @@ static void initialize_lock_count(struct recovery_journal *journal) atomic_t *decrement_counter = get_decrement_counter(journal, lock_number); journal_value = get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0); - ASSERT_LOG_ONLY((*journal_value == atomic_read(decrement_counter)), - "count to be initialized not in use"); + VDO_ASSERT_LOG_ONLY((*journal_value == atomic_read(decrement_counter)), + "count to be initialized not in use"); *journal_value = journal->entries_per_block + 1; atomic_set(decrement_counter, 0); } @@ -1175,13 +1175,13 @@ static void continue_committed_waiter(struct vdo_waiter *waiter, void *context) int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS); bool has_decrement; - ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point, - &data_vio->recovery_journal_point), - "DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commit waiter point is (%llu, %u)", - (unsigned long long) journal->commit_point.sequence_number, - journal->commit_point.entry_count, - (unsigned long long) data_vio->recovery_journal_point.sequence_number, - data_vio->recovery_journal_point.entry_count); + VDO_ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point, + &data_vio->recovery_journal_point), + "DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commit waiter point is (%llu, %u)", + (unsigned long long) journal->commit_point.sequence_number, + journal->commit_point.entry_count, + (unsigned long long) data_vio->recovery_journal_point.sequence_number, + data_vio->recovery_journal_point.entry_count); journal->commit_point = data_vio->recovery_journal_point; data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS; @@ -1281,8 +1281,8 @@ static void complete_write(struct vdo_completion *completion) journal->last_write_acknowledged = block->sequence_number; last_active_block = get_journal_block(&journal->active_tail_blocks); - ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number), - "completed journal write is still active"); + VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number), + "completed journal write is still active"); notify_commit_waiters(journal); @@ -1456,8 +1456,8 @@ void vdo_add_recovery_journal_entry(struct recovery_journal *journal, return; } - ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0, - "journal lock not held for new entry"); + VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0, + "journal lock not held for new entry"); vdo_advance_journal_point(&journal->append_point, journal->entries_per_block); vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter); @@ -1564,13 +1564,13 @@ void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journ if (sequence_number == 0) return; - ASSERT_LOG_ONLY((zone_type != VDO_ZONE_TYPE_JOURNAL), - "invalid lock count increment from journal zone"); + VDO_ASSERT_LOG_ONLY((zone_type != VDO_ZONE_TYPE_JOURNAL), + "invalid lock count increment from journal zone"); lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); current_value = get_counter(journal, lock_number, zone_type, zone_id); - ASSERT_LOG_ONLY(*current_value < U16_MAX, - "increment of lock counter must not overflow"); + VDO_ASSERT_LOG_ONLY(*current_value < U16_MAX, + "increment of lock counter must not overflow"); if (*current_value == 0) { /* diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index bfcdedeedb86..c7abb8078336 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -976,8 +976,8 @@ find_entry_starting_next_page(struct repair_completion *repair, if (needs_sort) { struct numbered_block_mapping *just_sorted_entry = sort_next_heap_element(repair); - ASSERT_LOG_ONLY(just_sorted_entry < current_entry, - "heap is returning elements in an unexpected order"); + VDO_ASSERT_LOG_ONLY(just_sorted_entry < current_entry, + "heap is returning elements in an unexpected order"); } current_entry--; @@ -1129,8 +1129,8 @@ static void recover_block_map(struct vdo_completion *completion) repair->current_entry = &repair->entries[repair->block_map_entry_count - 1]; first_sorted_entry = sort_next_heap_element(repair); - ASSERT_LOG_ONLY(first_sorted_entry == repair->current_entry, - "heap is returning elements in an unexpected order"); + VDO_ASSERT_LOG_ONLY(first_sorted_entry == repair->current_entry, + "heap is returning elements in an unexpected order"); /* Prevent any page from being processed until all pages have been launched. */ repair->launching = true; @@ -1489,8 +1489,8 @@ static int extract_new_mappings(struct repair_completion *repair) repair->block_map_entry_count++; } - result = ASSERT((repair->block_map_entry_count <= repair->entry_count), - "approximate entry count is an upper bound"); + result = VDO_ASSERT((repair->block_map_entry_count <= repair->entry_count), + "approximate entry count is an upper bound"); if (result != VDO_SUCCESS) vdo_enter_read_only_mode(vdo, result); diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 97208c9e0062..00746de09c12 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -149,7 +149,7 @@ static void mark_slab_journal_dirty(struct slab_journal *journal, sequence_numbe struct slab_journal *dirty_journal; struct list_head *dirty_list = &journal->slab->allocator->dirty_slab_journals; - ASSERT_LOG_ONLY(journal->recovery_lock == 0, "slab journal was clean"); + VDO_ASSERT_LOG_ONLY(journal->recovery_lock == 0, "slab journal was clean"); journal->recovery_lock = lock; list_for_each_entry_reverse(dirty_journal, dirty_list, dirty_entry) { @@ -216,7 +216,7 @@ static u8 __must_check compute_fullness_hint(struct slab_depot *depot, { block_count_t hint; - ASSERT_LOG_ONLY((free_blocks < (1 << 23)), "free blocks must be less than 2^23"); + VDO_ASSERT_LOG_ONLY((free_blocks < (1 << 23)), "free blocks must be less than 2^23"); if (free_blocks == 0) return 0; @@ -532,13 +532,13 @@ static void adjust_slab_journal_block_reference(struct slab_journal *journal, return; } - ASSERT_LOG_ONLY((adjustment != 0), "adjustment must be non-zero"); + VDO_ASSERT_LOG_ONLY((adjustment != 0), "adjustment must be non-zero"); lock = get_lock(journal, sequence_number); if (adjustment < 0) { - ASSERT_LOG_ONLY((-adjustment <= lock->count), - "adjustment %d of lock count %u for slab journal block %llu must not underflow", - adjustment, lock->count, - (unsigned long long) sequence_number); + VDO_ASSERT_LOG_ONLY((-adjustment <= lock->count), + "adjustment %d of lock count %u for slab journal block %llu must not underflow", + adjustment, lock->count, + (unsigned long long) sequence_number); } lock->count += adjustment; @@ -661,16 +661,16 @@ static void reopen_slab_journal(struct vdo_slab *slab) struct slab_journal *journal = &slab->journal; sequence_number_t block; - ASSERT_LOG_ONLY(journal->tail_header.entry_count == 0, - "vdo_slab journal's active block empty before reopening"); + VDO_ASSERT_LOG_ONLY(journal->tail_header.entry_count == 0, + "vdo_slab journal's active block empty before reopening"); journal->head = journal->tail; initialize_journal_state(journal); /* Ensure no locks are spuriously held on an empty journal. */ for (block = 1; block <= journal->size; block++) { - ASSERT_LOG_ONLY((get_lock(journal, block)->count == 0), - "Scrubbed journal's block %llu is not locked", - (unsigned long long) block); + VDO_ASSERT_LOG_ONLY((get_lock(journal, block)->count == 0), + "Scrubbed journal's block %llu is not locked", + (unsigned long long) block); } add_entries(journal); @@ -757,7 +757,7 @@ static void write_slab_journal_block(struct vdo_waiter *waiter, void *context) /* Copy the tail block into the vio. */ memcpy(pooled->vio.data, journal->block, VDO_BLOCK_SIZE); - ASSERT_LOG_ONLY(unused_entries >= 0, "vdo_slab journal block is not overfull"); + VDO_ASSERT_LOG_ONLY(unused_entries >= 0, "vdo_slab journal block is not overfull"); if (unused_entries > 0) { /* * Release the per-entry locks for any unused entries in the block we are about to @@ -907,22 +907,22 @@ static void add_entry(struct slab_journal *journal, physical_block_number_t pbn, struct packed_slab_journal_block *block = journal->block; int result; - result = ASSERT(vdo_before_journal_point(&journal->tail_header.recovery_point, - &recovery_point), - "recovery journal point is monotonically increasing, recovery point: %llu.%u, block recovery point: %llu.%u", - (unsigned long long) recovery_point.sequence_number, - recovery_point.entry_count, - (unsigned long long) journal->tail_header.recovery_point.sequence_number, - journal->tail_header.recovery_point.entry_count); + result = VDO_ASSERT(vdo_before_journal_point(&journal->tail_header.recovery_point, + &recovery_point), + "recovery journal point is monotonically increasing, recovery point: %llu.%u, block recovery point: %llu.%u", + (unsigned long long) recovery_point.sequence_number, + recovery_point.entry_count, + (unsigned long long) journal->tail_header.recovery_point.sequence_number, + journal->tail_header.recovery_point.entry_count); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); return; } if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { - result = ASSERT((journal->tail_header.entry_count < - journal->full_entries_per_block), - "block has room for full entries"); + result = VDO_ASSERT((journal->tail_header.entry_count < + journal->full_entries_per_block), + "block has room for full entries"); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); @@ -1371,8 +1371,8 @@ static unsigned int calculate_slab_priority(struct vdo_slab *slab) */ static void prioritize_slab(struct vdo_slab *slab) { - ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), - "a slab must not already be on a ring when prioritizing"); + VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), + "a slab must not already be on a ring when prioritizing"); slab->priority = calculate_slab_priority(slab); vdo_priority_table_enqueue(slab->allocator->prioritized_slabs, slab->priority, &slab->allocq_entry); @@ -1655,8 +1655,8 @@ static int __must_check adjust_reference_count(struct vdo_slab *slab, * the last time it was clean. We must release the per-entry slab journal lock for * the entry associated with the update we are now doing. */ - result = ASSERT(is_valid_journal_point(slab_journal_point), - "Reference count adjustments need slab journal points."); + result = VDO_ASSERT(is_valid_journal_point(slab_journal_point), + "Reference count adjustments need slab journal points."); if (result != VDO_SUCCESS) return result; @@ -1825,16 +1825,16 @@ static void add_entries(struct slab_journal *journal) * scrubbing thresholds, this should never happen. */ if (lock->count > 0) { - ASSERT_LOG_ONLY((journal->head + journal->size) == journal->tail, - "New block has locks, but journal is not full"); + VDO_ASSERT_LOG_ONLY((journal->head + journal->size) == journal->tail, + "New block has locks, but journal is not full"); /* * The blocking threshold must let the journal fill up if the new * block has locks; if the blocking threshold is smaller than the * journal size, the new block cannot possibly have locks already. */ - ASSERT_LOG_ONLY((journal->blocking_threshold >= journal->size), - "New block can have locks already iff blocking threshold is at the end of the journal"); + VDO_ASSERT_LOG_ONLY((journal->blocking_threshold >= journal->size), + "New block can have locks already iff blocking threshold is at the end of the journal"); WRITE_ONCE(journal->events->disk_full_count, journal->events->disk_full_count + 1); @@ -2361,9 +2361,9 @@ static int allocate_slab_counters(struct vdo_slab *slab) int result; size_t index, bytes; - result = ASSERT(slab->reference_blocks == NULL, - "vdo_slab %u doesn't allocate refcounts twice", - slab->slab_number); + result = VDO_ASSERT(slab->reference_blocks == NULL, + "vdo_slab %u doesn't allocate refcounts twice", + slab->slab_number); if (result != VDO_SUCCESS) return result; @@ -2503,9 +2503,9 @@ static void load_slab_journal(struct vdo_slab *slab) * 1. This is impossible, due to the scrubbing threshold, on a real system, so * don't bother reading the (bogus) data off disk. */ - ASSERT_LOG_ONLY(((journal->size < 16) || - (journal->scrubbing_threshold < (journal->size - 1))), - "Scrubbing threshold protects against reads of unwritten slab journal blocks"); + VDO_ASSERT_LOG_ONLY(((journal->size < 16) || + (journal->scrubbing_threshold < (journal->size - 1))), + "Scrubbing threshold protects against reads of unwritten slab journal blocks"); vdo_finish_loading_with_result(&slab->state, allocate_counters_if_clean(slab)); return; @@ -2519,8 +2519,8 @@ static void register_slab_for_scrubbing(struct vdo_slab *slab, bool high_priorit { struct slab_scrubber *scrubber = &slab->allocator->scrubber; - ASSERT_LOG_ONLY((slab->status != VDO_SLAB_REBUILT), - "slab to be scrubbed is unrecovered"); + VDO_ASSERT_LOG_ONLY((slab->status != VDO_SLAB_REBUILT), + "slab to be scrubbed is unrecovered"); if (slab->status != VDO_SLAB_REQUIRES_SCRUBBING) return; @@ -2547,17 +2547,17 @@ static void queue_slab(struct vdo_slab *slab) block_count_t free_blocks; int result; - ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), + VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), "a requeued slab must not already be on a ring"); if (vdo_is_read_only(allocator->depot->vdo)) return; free_blocks = slab->free_blocks; - result = ASSERT((free_blocks <= allocator->depot->slab_config.data_blocks), - "rebuilt slab %u must have a valid free block count (has %llu, expected maximum %llu)", - slab->slab_number, (unsigned long long) free_blocks, - (unsigned long long) allocator->depot->slab_config.data_blocks); + result = VDO_ASSERT((free_blocks <= allocator->depot->slab_config.data_blocks), + "rebuilt slab %u must have a valid free block count (has %llu, expected maximum %llu)", + slab->slab_number, (unsigned long long) free_blocks, + (unsigned long long) allocator->depot->slab_config.data_blocks); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(allocator->depot->vdo, result); return; @@ -2880,9 +2880,9 @@ static void apply_journal_entries(struct vdo_completion *completion) * At the end of rebuild, the reference counters should be accurate to the end of the * journal we just applied. */ - result = ASSERT(!vdo_before_journal_point(&last_entry_applied, - &ref_counts_point), - "Refcounts are not more accurate than the slab journal"); + result = VDO_ASSERT(!vdo_before_journal_point(&last_entry_applied, + &ref_counts_point), + "Refcounts are not more accurate than the slab journal"); if (result != VDO_SUCCESS) { abort_scrubbing(scrubber, result); return; @@ -2993,8 +2993,8 @@ static void scrub_slabs(struct block_allocator *allocator, struct vdo_completion static inline void assert_on_allocator_thread(thread_id_t thread_id, const char *function_name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == thread_id), - "%s called on correct thread", function_name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == thread_id), + "%s called on correct thread", function_name); } static void register_slab_with_allocator(struct block_allocator *allocator, @@ -3142,8 +3142,8 @@ static int __must_check allocate_slab_block(struct vdo_slab *slab, if (!search_reference_blocks(slab, &free_index)) return VDO_NO_SPACE; - ASSERT_LOG_ONLY((slab->counters[free_index] == EMPTY_REFERENCE_COUNT), - "free block must have ref count of zero"); + VDO_ASSERT_LOG_ONLY((slab->counters[free_index] == EMPTY_REFERENCE_COUNT), + "free block must have ref count of zero"); make_provisional_reference(slab, free_index); adjust_free_block_count(slab, false); @@ -3850,8 +3850,8 @@ static bool __must_check release_recovery_journal_lock(struct slab_journal *jour sequence_number_t recovery_lock) { if (recovery_lock > journal->recovery_lock) { - ASSERT_LOG_ONLY((recovery_lock < journal->recovery_lock), - "slab journal recovery lock is not older than the recovery journal head"); + VDO_ASSERT_LOG_ONLY((recovery_lock < journal->recovery_lock), + "slab journal recovery lock is not older than the recovery journal head"); return false; } @@ -4665,8 +4665,8 @@ int vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, return VDO_INCREMENT_TOO_SMALL; /* Generate the depot configuration for the new block count. */ - ASSERT_LOG_ONLY(depot->first_block == partition->offset, - "New slab depot partition doesn't change origin"); + VDO_ASSERT_LOG_ONLY(depot->first_block == partition->offset, + "New slab depot partition doesn't change origin"); result = vdo_configure_slab_depot(partition, depot->slab_config, depot->zone_count, &new_state); if (result != VDO_SUCCESS) @@ -4740,7 +4740,7 @@ static void register_new_slabs(void *context, zone_count_t zone_number, */ void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent) { - ASSERT_LOG_ONLY(depot->new_slabs != NULL, "Must have new slabs to use"); + VDO_ASSERT_LOG_ONLY(depot->new_slabs != NULL, "Must have new slabs to use"); vdo_schedule_operation(depot->action_manager, VDO_ADMIN_STATE_SUSPENDED_OPERATION, NULL, register_new_slabs, @@ -4796,8 +4796,8 @@ static void do_drain_step(struct vdo_completion *completion) return; case VDO_DRAIN_ALLOCATOR_STEP_FINISHED: - ASSERT_LOG_ONLY(!is_vio_pool_busy(allocator->vio_pool), - "vio pool not busy"); + VDO_ASSERT_LOG_ONLY(!is_vio_pool_busy(allocator->vio_pool), + "vio pool not busy"); vdo_finish_draining_with_result(&allocator->state, completion->result); return; diff --git a/drivers/md/dm-vdo/thread-registry.c b/drivers/md/dm-vdo/thread-registry.c index 03e2f45e8e78..d4a077d58c60 100644 --- a/drivers/md/dm-vdo/thread-registry.c +++ b/drivers/md/dm-vdo/thread-registry.c @@ -44,7 +44,7 @@ void vdo_register_thread(struct thread_registry *registry, list_add_tail_rcu(&new_thread->links, ®istry->links); spin_unlock(®istry->lock); - ASSERT_LOG_ONLY(!found_it, "new thread not already in registry"); + VDO_ASSERT_LOG_ONLY(!found_it, "new thread not already in registry"); if (found_it) { /* Ensure no RCU iterators see it before re-initializing. */ synchronize_rcu(); @@ -67,7 +67,7 @@ void vdo_unregister_thread(struct thread_registry *registry) } spin_unlock(®istry->lock); - ASSERT_LOG_ONLY(found_it, "thread found in registry"); + VDO_ASSERT_LOG_ONLY(found_it, "thread found in registry"); if (found_it) { /* Ensure no RCU iterators see it before re-initializing. */ synchronize_rcu(); diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index b4dd0634a5cb..11be2ab17e29 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -425,9 +425,9 @@ int vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, type = &default_queue_type; if (thread->queue != NULL) { - return ASSERT(vdo_work_queue_type_is(thread->queue, type), - "already constructed vdo thread %u is of the correct type", - thread_id); + return VDO_ASSERT(vdo_work_queue_type_is(thread->queue, type), + "already constructed vdo thread %u is of the correct type", + thread_id); } thread->vdo = vdo; @@ -448,8 +448,8 @@ static int register_vdo(struct vdo *vdo) int result; write_lock(®istry.lock); - result = ASSERT(filter_vdos_locked(vdo_is_equal, vdo) == NULL, - "VDO not already registered"); + result = VDO_ASSERT(filter_vdos_locked(vdo_is_equal, vdo) == NULL, + "VDO not already registered"); if (result == VDO_SUCCESS) { INIT_LIST_HEAD(&vdo->registration); list_add_tail(&vdo->registration, ®istry.links); @@ -1050,8 +1050,8 @@ int vdo_register_read_only_listener(struct vdo *vdo, void *listener, struct read_only_listener *read_only_listener; int result; - result = ASSERT(thread_id != vdo->thread_config.dedupe_thread, - "read only listener not registered on dedupe thread"); + result = VDO_ASSERT(thread_id != vdo->thread_config.dedupe_thread, + "read only listener not registered on dedupe thread"); if (result != VDO_SUCCESS) return result; @@ -1704,8 +1704,8 @@ void vdo_dump_status(const struct vdo *vdo) */ void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.admin_thread), - "%s called on admin thread", name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.admin_thread), + "%s called on admin thread", name); } /** @@ -1718,9 +1718,9 @@ void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name) void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, const char *name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == - vdo->thread_config.logical_threads[logical_zone]), - "%s called on logical thread", name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == + vdo->thread_config.logical_threads[logical_zone]), + "%s called on logical thread", name); } /** @@ -1733,9 +1733,9 @@ void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logic void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone, const char *name) { - ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == - vdo->thread_config.physical_threads[physical_zone]), - "%s called on physical thread", name); + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == + vdo->thread_config.physical_threads[physical_zone]), + "%s called on physical thread", name); } /** @@ -1773,7 +1773,7 @@ int vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, /* With the PBN already checked, we should always succeed in finding a slab. */ slab = vdo_get_slab(vdo->depot, pbn); - result = ASSERT(slab != NULL, "vdo_get_slab must succeed on all valid PBNs"); + result = VDO_ASSERT(slab != NULL, "vdo_get_slab must succeed on all valid PBNs"); if (result != VDO_SUCCESS) return result; diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index 83c36f7590de..b1e4e604c2c3 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -82,14 +82,14 @@ int allocate_vio_components(struct vdo *vdo, enum vio_type vio_type, struct bio *bio; int result; - result = ASSERT(block_count <= MAX_BLOCKS_PER_VIO, - "block count %u does not exceed maximum %u", block_count, - MAX_BLOCKS_PER_VIO); + result = VDO_ASSERT(block_count <= MAX_BLOCKS_PER_VIO, + "block count %u does not exceed maximum %u", block_count, + MAX_BLOCKS_PER_VIO); if (result != VDO_SUCCESS) return result; - result = ASSERT(((vio_type != VIO_TYPE_UNINITIALIZED) && (vio_type != VIO_TYPE_DATA)), - "%d is a metadata type", vio_type); + result = VDO_ASSERT(((vio_type != VIO_TYPE_UNINITIALIZED) && (vio_type != VIO_TYPE_DATA)), + "%d is a metadata type", vio_type); if (result != VDO_SUCCESS) return result; @@ -363,13 +363,13 @@ void free_vio_pool(struct vio_pool *pool) return; /* Remove all available vios from the object pool. */ - ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&pool->waiting), - "VIO pool must not have any waiters when being freed"); - ASSERT_LOG_ONLY((pool->busy_count == 0), - "VIO pool must not have %zu busy entries when being freed", - pool->busy_count); - ASSERT_LOG_ONLY(list_empty(&pool->busy), - "VIO pool must not have busy entries when being freed"); + VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&pool->waiting), + "VIO pool must not have any waiters when being freed"); + VDO_ASSERT_LOG_ONLY((pool->busy_count == 0), + "VIO pool must not have %zu busy entries when being freed", + pool->busy_count); + VDO_ASSERT_LOG_ONLY(list_empty(&pool->busy), + "VIO pool must not have busy entries when being freed"); list_for_each_entry_safe(pooled, tmp, &pool->available, pool_entry) { list_del(&pooled->pool_entry); @@ -377,8 +377,8 @@ void free_vio_pool(struct vio_pool *pool) pool->size--; } - ASSERT_LOG_ONLY(pool->size == 0, - "VIO pool must not have missing entries when being freed"); + VDO_ASSERT_LOG_ONLY(pool->size == 0, + "VIO pool must not have missing entries when being freed"); vdo_free(vdo_forget(pool->buffer)); vdo_free(pool); @@ -403,8 +403,8 @@ void acquire_vio_from_pool(struct vio_pool *pool, struct vdo_waiter *waiter) { struct pooled_vio *pooled; - ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), - "acquire from active vio_pool called from correct thread"); + VDO_ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), + "acquire from active vio_pool called from correct thread"); if (list_empty(&pool->available)) { vdo_waitq_enqueue_waiter(&pool->waiting, waiter); @@ -424,8 +424,8 @@ void acquire_vio_from_pool(struct vio_pool *pool, struct vdo_waiter *waiter) */ void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio) { - ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), - "vio pool entry returned on same thread as it was acquired"); + VDO_ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), + "vio pool entry returned on same thread as it was acquired"); vio->vio.completion.error_handler = NULL; vio->vio.completion.parent = NULL; @@ -465,8 +465,8 @@ void vdo_count_bios(struct atomic_bio_stats *bio_stats, struct bio *bio) * shouldn't exist. */ default: - ASSERT_LOG_ONLY(0, "Bio operation %d not a write, read, discard, or empty flush", - bio_op(bio)); + VDO_ASSERT_LOG_ONLY(0, "Bio operation %d not a write, read, discard, or empty flush", + bio_op(bio)); } if ((bio->bi_opf & REQ_PREFLUSH) != 0) diff --git a/drivers/md/dm-vdo/vio.h b/drivers/md/dm-vdo/vio.h index fbfee5e3415d..3490e9f59b04 100644 --- a/drivers/md/dm-vdo/vio.h +++ b/drivers/md/dm-vdo/vio.h @@ -67,10 +67,10 @@ static inline void assert_vio_in_bio_zone(struct vio *vio) thread_id_t expected = get_vio_bio_zone_thread_id(vio); thread_id_t thread_id = vdo_get_callback_thread_id(); - ASSERT_LOG_ONLY((expected == thread_id), - "vio I/O for physical block %llu on thread %u, should be on bio zone thread %u", - (unsigned long long) pbn_from_vio_bio(vio->bio), thread_id, - expected); + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "vio I/O for physical block %llu on thread %u, should be on bio zone thread %u", + (unsigned long long) pbn_from_vio_bio(vio->bio), thread_id, + expected); } int vdo_create_bio(struct bio **bio_ptr); From fc03f737609a756dc1c2117963d9a855f56f0716 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 16:03:01 -0600 Subject: [PATCH 0950/1038] dm vdo encodings: update some stale comments Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/encodings.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index e24c31bc3524..ebb0a4edd109 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -544,8 +544,6 @@ const char *vdo_get_journal_operation_name(enum journal_operation operation) /** * encode_slab_depot_state_2_0() - Encode the state of a slab depot into a buffer. - * - * Return: UDS_SUCCESS or an error. */ static void encode_slab_depot_state_2_0(u8 *buffer, size_t *offset, struct slab_depot_state_2_0 state) @@ -573,7 +571,7 @@ static void encode_slab_depot_state_2_0(u8 *buffer, size_t *offset, /** * decode_slab_depot_state_2_0() - Decode slab depot component state version 2.0 from a buffer. * - * Return: UDS_SUCCESS or an error code. + * Return: VDO_SUCCESS or an error code. */ static int decode_slab_depot_state_2_0(u8 *buffer, size_t *offset, struct slab_depot_state_2_0 *state) From e60167367eb2959a86b39678f9566cc3aa300c73 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Wed, 28 Feb 2024 00:00:45 -0500 Subject: [PATCH 0951/1038] dm vdo indexer: update ASSERT and ASSERT_LOG_ONLY usage Update indexer uses of ASSERT and ASSERT_LOG_ONLY to VDO_ASSERT and VDO_ASSERT_LOG_ONLY, respectively. Remove ASSERT and ASSERT_LOG_ONLY. Also rename uds_assertion_failed to vdo_assertion_failed. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/indexer/chapter-index.c | 16 ++-- drivers/md/dm-vdo/indexer/config.c | 16 ++-- drivers/md/dm-vdo/indexer/delta-index.c | 90 +++++++++++------------ drivers/md/dm-vdo/indexer/index-layout.c | 5 +- drivers/md/dm-vdo/indexer/index-session.c | 12 +-- drivers/md/dm-vdo/indexer/index.c | 4 +- drivers/md/dm-vdo/indexer/volume-index.c | 32 ++++---- drivers/md/dm-vdo/indexer/volume.c | 32 ++++---- drivers/md/dm-vdo/permassert.c | 2 +- drivers/md/dm-vdo/permassert.h | 8 +- 10 files changed, 107 insertions(+), 110 deletions(-) diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c index 68d86028dbb7..47e4ed234242 100644 --- a/drivers/md/dm-vdo/indexer/chapter-index.c +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -83,10 +83,10 @@ int uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, u64 chapter_number = chapter_index->virtual_chapter_number; u32 record_pages = geometry->record_pages_per_chapter; - result = ASSERT(page_number < record_pages, - "Page number within chapter (%u) exceeds the maximum value %u", - page_number, record_pages); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(page_number < record_pages, + "Page number within chapter (%u) exceeds the maximum value %u", + page_number, record_pages); + if (result != VDO_SUCCESS) return UDS_INVALID_ARGUMENT; address = uds_hash_to_chapter_delta_address(name, geometry); @@ -97,10 +97,10 @@ int uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, return result; found = was_entry_found(&entry, address); - result = ASSERT(!(found && entry.is_collision), - "Chunk appears more than once in chapter %llu", - (unsigned long long) chapter_number); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(!(found && entry.is_collision), + "Chunk appears more than once in chapter %llu", + (unsigned long long) chapter_number); + if (result != VDO_SUCCESS) return UDS_BAD_STATE; found_name = (found ? name->name : NULL); diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c index 5da39043b9ae..69bf27a9d61b 100644 --- a/drivers/md/dm-vdo/indexer/config.c +++ b/drivers/md/dm-vdo/indexer/config.c @@ -134,10 +134,10 @@ int uds_validate_config_contents(struct buffered_reader *reader, decode_u32_le(buffer, &offset, &config.sparse_sample_rate); decode_u64_le(buffer, &offset, &config.nonce); - result = ASSERT(offset == sizeof(struct uds_configuration_6_02), - "%zu bytes read but not decoded", - sizeof(struct uds_configuration_6_02) - offset); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(struct uds_configuration_6_02), + "%zu bytes read but not decoded", + sizeof(struct uds_configuration_6_02) - offset); + if (result != VDO_SUCCESS) return UDS_CORRUPT_DATA; if (is_version(INDEX_CONFIG_VERSION_6_02, version_buffer)) { @@ -210,10 +210,10 @@ int uds_write_config_contents(struct buffered_writer *writer, encode_u32_le(buffer, &offset, config->sparse_sample_rate); encode_u64_le(buffer, &offset, config->nonce); - result = ASSERT(offset == sizeof(struct uds_configuration_6_02), - "%zu bytes encoded, of %zu expected", offset, - sizeof(struct uds_configuration_6_02)); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(struct uds_configuration_6_02), + "%zu bytes encoded, of %zu expected", offset, + sizeof(struct uds_configuration_6_02)); + if (result != VDO_SUCCESS) return result; if (version >= 4) { diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c index 5bba9a48c5a0..b49066554248 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -856,10 +856,10 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index, decode_u64_le(buffer, &offset, &header.record_count); decode_u64_le(buffer, &offset, &header.collision_count); - result = ASSERT(offset == sizeof(struct delta_index_header), - "%zu bytes decoded of %zu expected", offset, - sizeof(struct delta_index_header)); - if (result != UDS_SUCCESS) { + result = VDO_ASSERT(offset == sizeof(struct delta_index_header), + "%zu bytes decoded of %zu expected", offset, + sizeof(struct delta_index_header)); + if (result != VDO_SUCCESS) { return uds_log_warning_strerror(result, "failed to read delta index header"); } @@ -1136,10 +1136,10 @@ int uds_start_saving_delta_index(const struct delta_index *delta_index, encode_u64_le(buffer, &offset, delta_zone->record_count); encode_u64_le(buffer, &offset, delta_zone->collision_count); - result = ASSERT(offset == sizeof(struct delta_index_header), - "%zu bytes encoded of %zu expected", offset, - sizeof(struct delta_index_header)); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(struct delta_index_header), + "%zu bytes encoded of %zu expected", offset, + sizeof(struct delta_index_header)); + if (result != VDO_SUCCESS) return result; result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); @@ -1212,9 +1212,9 @@ size_t uds_compute_delta_index_save_bytes(u32 list_count, size_t memory_size) static int assert_not_at_end(const struct delta_index_entry *delta_entry) { - int result = ASSERT(!delta_entry->at_end, - "operation is invalid because the list entry is at the end of the delta list"); - if (result != UDS_SUCCESS) + int result = VDO_ASSERT(!delta_entry->at_end, + "operation is invalid because the list entry is at the end of the delta list"); + if (result != VDO_SUCCESS) result = UDS_BAD_STATE; return result; @@ -1236,19 +1236,19 @@ int uds_start_delta_index_search(const struct delta_index *delta_index, u32 list struct delta_zone *delta_zone; struct delta_list *delta_list; - result = ASSERT((list_number < delta_index->list_count), - "Delta list number (%u) is out of range (%u)", list_number, - delta_index->list_count); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((list_number < delta_index->list_count), + "Delta list number (%u) is out of range (%u)", list_number, + delta_index->list_count); + if (result != VDO_SUCCESS) return UDS_CORRUPT_DATA; zone_number = list_number / delta_index->lists_per_zone; delta_zone = &delta_index->delta_zones[zone_number]; list_number -= delta_zone->first_list; - result = ASSERT((list_number < delta_zone->list_count), - "Delta list number (%u) is out of range (%u) for zone (%u)", - list_number, delta_zone->list_count, zone_number); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((list_number < delta_zone->list_count), + "Delta list number (%u) is out of range (%u) for zone (%u)", + list_number, delta_zone->list_count, zone_number); + if (result != VDO_SUCCESS) return UDS_CORRUPT_DATA; if (delta_index->mutable) { @@ -1362,9 +1362,9 @@ noinline int uds_next_delta_index_entry(struct delta_index_entry *delta_entry) delta_entry->at_end = true; delta_entry->delta = 0; delta_entry->is_collision = false; - result = ASSERT((delta_entry->offset == size), - "next offset past end of delta list"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((delta_entry->offset == size), + "next offset past end of delta list"); + if (result != VDO_SUCCESS) result = UDS_CORRUPT_DATA; return result; @@ -1390,8 +1390,8 @@ int uds_remember_delta_index_offset(const struct delta_index_entry *delta_entry) int result; struct delta_list *delta_list = delta_entry->delta_list; - result = ASSERT(!delta_entry->is_collision, "entry is not a collision"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(!delta_entry->is_collision, "entry is not a collision"); + if (result != VDO_SUCCESS) return result; delta_list->save_key = delta_entry->key - delta_entry->delta; @@ -1489,9 +1489,9 @@ int uds_get_delta_entry_collision(const struct delta_index_entry *delta_entry, u if (result != UDS_SUCCESS) return result; - result = ASSERT(delta_entry->is_collision, - "Cannot get full block name from a non-collision delta index entry"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(delta_entry->is_collision, + "Cannot get full block name from a non-collision delta index entry"); + if (result != VDO_SUCCESS) return UDS_BAD_STATE; get_collision_name(delta_entry, name); @@ -1506,9 +1506,9 @@ u32 uds_get_delta_entry_value(const struct delta_index_entry *delta_entry) static int assert_mutable_entry(const struct delta_index_entry *delta_entry) { - int result = ASSERT((delta_entry->delta_list != &delta_entry->temp_delta_list), - "delta index is mutable"); - if (result != UDS_SUCCESS) + int result = VDO_ASSERT((delta_entry->delta_list != &delta_entry->temp_delta_list), + "delta index is mutable"); + if (result != VDO_SUCCESS) result = UDS_BAD_STATE; return result; @@ -1527,10 +1527,10 @@ int uds_set_delta_entry_value(const struct delta_index_entry *delta_entry, u32 v if (result != UDS_SUCCESS) return result; - result = ASSERT((value & value_mask) == value, - "Value (%u) being set in a delta index is too large (must fit in %u bits)", - value, delta_entry->value_bits); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((value & value_mask) == value, + "Value (%u) being set in a delta index is too large (must fit in %u bits)", + value, delta_entry->value_bits); + if (result != VDO_SUCCESS) return UDS_INVALID_ARGUMENT; set_field(value, delta_entry->delta_zone->memory, @@ -1730,9 +1730,9 @@ int uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, u3 if (result != UDS_SUCCESS) return result; - result = ASSERT((key == delta_entry->key), - "incorrect key for collision entry"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((key == delta_entry->key), + "incorrect key for collision entry"); + if (result != VDO_SUCCESS) return result; delta_entry->offset += delta_entry->entry_bits; @@ -1742,8 +1742,8 @@ int uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, u3 result = insert_bits(delta_entry, delta_entry->entry_bits); } else if (delta_entry->at_end) { /* Insert a new entry at the end of the delta list. */ - result = ASSERT((key >= delta_entry->key), "key past end of list"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((key >= delta_entry->key), "key past end of list"); + if (result != VDO_SUCCESS) return result; set_delta(delta_entry, key - delta_entry->key); @@ -1760,14 +1760,14 @@ int uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, u3 * Insert a new entry which requires the delta in the following entry to be * updated. */ - result = ASSERT((key < delta_entry->key), - "key precedes following entry"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((key < delta_entry->key), + "key precedes following entry"); + if (result != VDO_SUCCESS) return result; - result = ASSERT((key >= delta_entry->key - delta_entry->delta), - "key effects following entry's delta"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((key >= delta_entry->key - delta_entry->delta), + "key effects following entry's delta"); + if (result != VDO_SUCCESS) return result; old_entry_size = delta_entry->entry_bits; diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index 01e0db4184aa..349b3022f1e1 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -837,8 +837,9 @@ static u64 generate_index_save_nonce(u64 volume_nonce, struct index_save_layout encode_u32_le(buffer, &offset, isl->save_data.version); encode_u32_le(buffer, &offset, 0U); encode_u64_le(buffer, &offset, isl->index_save.start_block); - ASSERT_LOG_ONLY(offset == sizeof(nonce_data), - "%zu bytes encoded of %zu expected", offset, sizeof(nonce_data)); + VDO_ASSERT_LOG_ONLY(offset == sizeof(nonce_data), + "%zu bytes encoded of %zu expected", + offset, sizeof(nonce_data)); return generate_secondary_nonce(volume_nonce, buffer, sizeof(buffer)); } diff --git a/drivers/md/dm-vdo/indexer/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c index 22445dcb3fe0..9eae00548095 100644 --- a/drivers/md/dm-vdo/indexer/index-session.c +++ b/drivers/md/dm-vdo/indexer/index-session.c @@ -199,8 +199,8 @@ static void update_session_stats(struct uds_request *request) break; default: - request->status = ASSERT(false, "unknown request type: %d", - request->type); + request->status = VDO_ASSERT(false, "unknown request type: %d", + request->type); } } @@ -402,8 +402,8 @@ static void suspend_rebuild(struct uds_index_session *session) case INDEX_FREEING: default: /* These cases should not happen. */ - ASSERT_LOG_ONLY(false, "Bad load context state %u", - session->load_context.status); + VDO_ASSERT_LOG_ONLY(false, "Bad load context state %u", + session->load_context.status); break; } mutex_unlock(&session->load_context.mutex); @@ -531,8 +531,8 @@ int uds_resume_index_session(struct uds_index_session *session, case INDEX_FREEING: default: /* These cases should not happen; do nothing. */ - ASSERT_LOG_ONLY(false, "Bad load context state %u", - session->load_context.status); + VDO_ASSERT_LOG_ONLY(false, "Bad load context state %u", + session->load_context.status); break; } mutex_unlock(&session->load_context.mutex); diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c index 226713221105..221af95ca2a4 100644 --- a/drivers/md/dm-vdo/indexer/index.c +++ b/drivers/md/dm-vdo/indexer/index.c @@ -112,7 +112,7 @@ static void enqueue_barrier_messages(struct uds_index *index, u64 virtual_chapte for (zone = 0; zone < index->zone_count; zone++) { int result = launch_zone_message(message, zone, index); - ASSERT_LOG_ONLY((result == UDS_SUCCESS), "barrier message allocation"); + VDO_ASSERT_LOG_ONLY((result == UDS_SUCCESS), "barrier message allocation"); } } @@ -1380,7 +1380,7 @@ void uds_enqueue_request(struct uds_request *request, enum request_stage stage) break; default: - ASSERT_LOG_ONLY(false, "invalid index stage: %d", stage); + VDO_ASSERT_LOG_ONLY(false, "invalid index stage: %d", stage); return; } diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c index 1cc9ac4fe510..e2b0600d82b9 100644 --- a/drivers/md/dm-vdo/indexer/volume-index.c +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -832,10 +832,10 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, decode_u32_le(buffer, &offset, &header.first_list); decode_u32_le(buffer, &offset, &header.list_count); - result = ASSERT(offset == sizeof(buffer), - "%zu bytes decoded of %zu expected", offset, - sizeof(buffer)); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(buffer), + "%zu bytes decoded of %zu expected", offset, + sizeof(buffer)); + if (result != VDO_SUCCESS) result = UDS_CORRUPT_DATA; if (memcmp(header.magic, MAGIC_START_5, MAGIC_SIZE) != 0) { @@ -924,10 +924,10 @@ static int start_restoring_volume_index(struct volume_index *volume_index, offset += MAGIC_SIZE; decode_u32_le(buffer, &offset, &header.sparse_sample_rate); - result = ASSERT(offset == sizeof(buffer), - "%zu bytes decoded of %zu expected", offset, - sizeof(buffer)); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(buffer), + "%zu bytes decoded of %zu expected", offset, + sizeof(buffer)); + if (result != VDO_SUCCESS) result = UDS_CORRUPT_DATA; if (memcmp(header.magic, MAGIC_START_6, MAGIC_SIZE) != 0) @@ -1023,10 +1023,10 @@ static int start_saving_volume_sub_index(const struct volume_sub_index *sub_inde encode_u32_le(buffer, &offset, first_list); encode_u32_le(buffer, &offset, list_count); - result = ASSERT(offset == sizeof(struct sub_index_data), - "%zu bytes of config written, of %zu expected", offset, - sizeof(struct sub_index_data)); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(struct sub_index_data), + "%zu bytes of config written, of %zu expected", offset, + sizeof(struct sub_index_data)); + if (result != VDO_SUCCESS) return result; result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); @@ -1066,10 +1066,10 @@ static int start_saving_volume_index(const struct volume_index *volume_index, memcpy(buffer, MAGIC_START_6, MAGIC_SIZE); offset += MAGIC_SIZE; encode_u32_le(buffer, &offset, volume_index->sparse_sample_rate); - result = ASSERT(offset == sizeof(struct volume_index_data), - "%zu bytes of header written, of %zu expected", offset, - sizeof(struct volume_index_data)); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(offset == sizeof(struct volume_index_data), + "%zu bytes of header written, of %zu expected", offset, + sizeof(struct volume_index_data)); + if (result != VDO_SUCCESS) return result; result = uds_write_to_buffered_writer(writer, buffer, offset); diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c index 0a4beef8ac8d..701f2220d803 100644 --- a/drivers/md/dm-vdo/indexer/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -135,8 +135,8 @@ static void begin_pending_search(struct page_cache *cache, u32 physical_page, invalidate_counter.page = physical_page; invalidate_counter.counter++; set_invalidate_counter(cache, zone_number, invalidate_counter); - ASSERT_LOG_ONLY(search_pending(invalidate_counter), - "Search is pending for zone %u", zone_number); + VDO_ASSERT_LOG_ONLY(search_pending(invalidate_counter), + "Search is pending for zone %u", zone_number); /* * This memory barrier ensures that the write to the invalidate counter is seen by other * threads before this thread accesses the cached page. The corresponding read memory @@ -158,8 +158,8 @@ static void end_pending_search(struct page_cache *cache, unsigned int zone_numbe smp_mb(); invalidate_counter = get_invalidate_counter(cache, zone_number); - ASSERT_LOG_ONLY(search_pending(invalidate_counter), - "Search is pending for zone %u", zone_number); + VDO_ASSERT_LOG_ONLY(search_pending(invalidate_counter), + "Search is pending for zone %u", zone_number); invalidate_counter.counter++; set_invalidate_counter(cache, zone_number, invalidate_counter); } @@ -259,8 +259,8 @@ static int put_page_in_cache(struct page_cache *cache, u32 physical_page, int result; /* We hold the read_threads_mutex. */ - result = ASSERT((page->read_pending), "page to install has a pending read"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((page->read_pending), "page to install has a pending read"); + if (result != VDO_SUCCESS) return result; page->physical_page = physical_page; @@ -285,8 +285,8 @@ static void cancel_page_in_cache(struct page_cache *cache, u32 physical_page, int result; /* We hold the read_threads_mutex. */ - result = ASSERT((page->read_pending), "page to install has a pending read"); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((page->read_pending), "page to install has a pending read"); + if (result != VDO_SUCCESS) return; clear_cache_page(cache, page); @@ -889,10 +889,10 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req if (record_page_number == NO_CHAPTER_INDEX_ENTRY) return UDS_SUCCESS; - result = ASSERT(record_page_number < geometry->record_pages_per_chapter, - "0 <= %d < %u", record_page_number, - geometry->record_pages_per_chapter); - if (result != UDS_SUCCESS) + result = VDO_ASSERT(record_page_number < geometry->record_pages_per_chapter, + "0 <= %d < %u", record_page_number, + geometry->record_pages_per_chapter); + if (result != VDO_SUCCESS) return result; page_number = geometry->index_pages_per_chapter + record_page_number; @@ -1501,10 +1501,10 @@ static int __must_check initialize_page_cache(struct page_cache *cache, cache->zone_count = zone_count; atomic64_set(&cache->clock, 1); - result = ASSERT((cache->cache_slots <= VOLUME_CACHE_MAX_ENTRIES), - "requested cache size, %u, within limit %u", - cache->cache_slots, VOLUME_CACHE_MAX_ENTRIES); - if (result != UDS_SUCCESS) + result = VDO_ASSERT((cache->cache_slots <= VOLUME_CACHE_MAX_ENTRIES), + "requested cache size, %u, within limit %u", + cache->cache_slots, VOLUME_CACHE_MAX_ENTRIES); + if (result != VDO_SUCCESS) return result; result = vdo_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read, diff --git a/drivers/md/dm-vdo/permassert.c b/drivers/md/dm-vdo/permassert.c index 3fa752ba0061..6fe49c4b7e51 100644 --- a/drivers/md/dm-vdo/permassert.c +++ b/drivers/md/dm-vdo/permassert.c @@ -8,7 +8,7 @@ #include "errors.h" #include "logger.h" -int uds_assertion_failed(const char *expression_string, const char *file_name, +int vdo_assertion_failed(const char *expression_string, const char *file_name, int line_number, const char *format, ...) { va_list args; diff --git a/drivers/md/dm-vdo/permassert.h b/drivers/md/dm-vdo/permassert.h index 8774dde7927a..c34f2ba650e1 100644 --- a/drivers/md/dm-vdo/permassert.h +++ b/drivers/md/dm-vdo/permassert.h @@ -33,16 +33,12 @@ static inline int __must_check vdo_must_use(int value) /* Log a message if the expression is not true. */ #define VDO_ASSERT_LOG_ONLY(expr, ...) __VDO_ASSERT(expr, __VA_ARGS__) -/* For use by UDS */ -#define ASSERT(expr, ...) VDO_ASSERT(expr, __VA_ARGS__) -#define ASSERT_LOG_ONLY(expr, ...) __VDO_ASSERT(expr, __VA_ARGS__) - #define __VDO_ASSERT(expr, ...) \ (likely(expr) ? VDO_SUCCESS \ - : uds_assertion_failed(STRINGIFY(expr), __FILE__, __LINE__, __VA_ARGS__)) + : vdo_assertion_failed(STRINGIFY(expr), __FILE__, __LINE__, __VA_ARGS__)) /* Log an assertion failure message. */ -int uds_assertion_failed(const char *expression_string, const char *file_name, +int vdo_assertion_failed(const char *expression_string, const char *file_name, int line_number, const char *format, ...) __printf(4, 5); From 4e4152482b944c44a285375be072c4fb3bb50770 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 16:03:47 -0600 Subject: [PATCH 0952/1038] dm vdo target: eliminate inappropriate uses of UDS_SUCCESS Most should be VDO_SUCCESS. But comparing the return from kstrtouint() with UDS_SUCCESS (happens to be 0) makes no sense. Signed-off-by: Mike Snitzer Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dm-vdo-target.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index e493b2fec90b..5f607dbb16e9 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -318,7 +318,7 @@ static int split_string(const char *string, char separator, char ***substring_ar current_substring++; /* substrings[current_substring] is NULL already */ *substring_array_ptr = substrings; - return UDS_SUCCESS; + return VDO_SUCCESS; } /* @@ -356,7 +356,7 @@ static int join_strings(char **substring_array, size_t array_length, char separa *(current_position - 1) = '\0'; *string_ptr = output; - return UDS_SUCCESS; + return VDO_SUCCESS; } /** @@ -484,7 +484,7 @@ static int parse_one_thread_config_spec(const char *spec, int result; result = split_string(spec, '=', &fields); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; if ((fields[0] == NULL) || (fields[1] == NULL) || (fields[2] != NULL)) { @@ -495,7 +495,7 @@ static int parse_one_thread_config_spec(const char *spec, } result = kstrtouint(fields[1], 10, &count); - if (result != UDS_SUCCESS) { + if (result) { uds_log_error("thread config string error: integer value needed, found \"%s\"", fields[1]); free_string_array(fields); @@ -537,7 +537,7 @@ static int parse_thread_config_string(const char *string, unsigned int i; result = split_string(string, ',', &specs); - if (result != UDS_SUCCESS) + if (result != VDO_SUCCESS) return result; for (i = 0; specs[i] != NULL; i++) { @@ -607,7 +607,7 @@ static int parse_one_key_value_pair(const char *key, const char *value, /* The remaining arguments must have integral values. */ result = kstrtouint(value, 10, &count); - if (result != UDS_SUCCESS) { + if (result) { uds_log_error("optional config string error: integer value needed, found \"%s\"", value); return result; @@ -2913,7 +2913,7 @@ static int __init vdo_init(void) /* Add VDO errors to the set of errors registered by the indexer. */ result = vdo_register_status_codes(); - if (result != UDS_SUCCESS) { + if (result != VDO_SUCCESS) { uds_log_error("vdo_register_status_codes failed %d", result); vdo_module_destroy(); return result; From a9da0fb6d8c61e354611c181d505fd417aad51f5 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Sat, 10 Feb 2024 11:05:15 -0600 Subject: [PATCH 0953/1038] dm vdo: remove all sysfs interfaces Also update target major version number. All info is (or will be) accessible through alternative interfaces (e.g. "dmsetup message", module params, etc). Signed-off-by: Mike Snitzer Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/Makefile | 4 - drivers/md/dm-vdo/dedupe.c | 70 +- drivers/md/dm-vdo/dedupe.h | 2 +- drivers/md/dm-vdo/dm-vdo-target.c | 44 +- drivers/md/dm-vdo/logger.c | 5 - drivers/md/dm-vdo/logger.h | 2 - drivers/md/dm-vdo/pool-sysfs-stats.c | 2063 -------------------------- drivers/md/dm-vdo/pool-sysfs.c | 198 --- drivers/md/dm-vdo/pool-sysfs.h | 19 - drivers/md/dm-vdo/status-codes.c | 1 - drivers/md/dm-vdo/status-codes.h | 2 - drivers/md/dm-vdo/sysfs.c | 82 - drivers/md/dm-vdo/uds-sysfs.c | 187 --- drivers/md/dm-vdo/uds-sysfs.h | 12 - drivers/md/dm-vdo/vdo.c | 54 +- drivers/md/dm-vdo/vdo.h | 13 - 16 files changed, 9 insertions(+), 2749 deletions(-) delete mode 100644 drivers/md/dm-vdo/pool-sysfs-stats.c delete mode 100644 drivers/md/dm-vdo/pool-sysfs.c delete mode 100644 drivers/md/dm-vdo/pool-sysfs.h delete mode 100644 drivers/md/dm-vdo/sysfs.c delete mode 100644 drivers/md/dm-vdo/uds-sysfs.c delete mode 100644 drivers/md/dm-vdo/uds-sysfs.h diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 502a7a0acbdb..33e09abc6acd 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -28,19 +28,15 @@ dm-vdo-objs := \ packer.o \ permassert.o \ physical-zone.o \ - pool-sysfs.o \ - pool-sysfs-stats.o \ priority-table.o \ recovery-journal.o \ repair.o \ slab-depot.o \ status-codes.o \ string-utils.o \ - sysfs.o \ thread-device.o \ thread-registry.o \ thread-utils.o \ - uds-sysfs.o \ vdo.o \ vio.o \ wait-queue.o \ diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 52bdf657db64..8550a9a7958b 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -120,7 +120,6 @@ #include #include #include -#include #include #include #include @@ -279,7 +278,6 @@ struct hash_lock { struct hash_zones { struct action_manager *manager; - struct kobject dedupe_directory; struct uds_parameters parameters; struct uds_index_session *index_session; struct ratelimit_state ratelimiter; @@ -2022,56 +2020,6 @@ void vdo_share_compressed_write_lock(struct data_vio *data_vio, VDO_ASSERT_LOG_ONLY(claimed, "impossible to fail to claim an initial increment"); } -static void dedupe_kobj_release(struct kobject *directory) -{ - vdo_free(container_of(directory, struct hash_zones, dedupe_directory)); -} - -static ssize_t dedupe_status_show(struct kobject *directory, struct attribute *attr, - char *buf) -{ - struct uds_attribute *ua = container_of(attr, struct uds_attribute, attr); - struct hash_zones *zones = container_of(directory, struct hash_zones, - dedupe_directory); - - if (ua->show_string != NULL) - return sprintf(buf, "%s\n", ua->show_string(zones)); - else - return -EINVAL; -} - -static ssize_t dedupe_status_store(struct kobject *kobj __always_unused, - struct attribute *attr __always_unused, - const char *buf __always_unused, - size_t length __always_unused) -{ - return -EINVAL; -} - -/*----------------------------------------------------------------------*/ - -static const struct sysfs_ops dedupe_sysfs_ops = { - .show = dedupe_status_show, - .store = dedupe_status_store, -}; - -static struct uds_attribute dedupe_status_attribute = { - .attr = {.name = "status", .mode = 0444, }, - .show_string = vdo_get_dedupe_index_state_name, -}; - -static struct attribute *dedupe_attrs[] = { - &dedupe_status_attribute.attr, - NULL, -}; -ATTRIBUTE_GROUPS(dedupe); - -static const struct kobj_type dedupe_directory_type = { - .release = dedupe_kobj_release, - .sysfs_ops = &dedupe_sysfs_ops, - .default_groups = dedupe_groups, -}; - static void start_uds_queue(void *ptr) { /* @@ -2266,7 +2214,6 @@ static int initialize_index(struct vdo *vdo, struct hash_zones *zones) vdo_initialize_completion(&zones->completion, vdo, VDO_HASH_ZONES_COMPLETION); vdo_set_completion_callback(&zones->completion, change_dedupe_state, vdo->thread_config.dedupe_thread); - kobject_init(&zones->dedupe_directory, &dedupe_directory_type); return VDO_SUCCESS; } @@ -2537,10 +2484,7 @@ void vdo_free_hash_zones(struct hash_zones *zones) vdo_finish_dedupe_index(zones); ratelimit_state_exit(&zones->ratelimiter); - if (vdo_get_admin_state_code(&zones->state) == VDO_ADMIN_STATE_NEW) - vdo_free(zones); - else - kobject_put(&zones->dedupe_directory); + vdo_free(zones); } static void initiate_suspend_index(struct admin_state *state) @@ -3047,17 +2991,9 @@ int vdo_message_dedupe_index(struct hash_zones *zones, const char *name) return -EINVAL; } -int vdo_add_dedupe_index_sysfs(struct hash_zones *zones) +void vdo_set_dedupe_state_normal(struct hash_zones *zones) { - int result = kobject_add(&zones->dedupe_directory, - &zones->completion.vdo->vdo_directory, "dedupe"); - - if (result == 0) { - vdo_set_admin_state_code(&zones->state, - VDO_ADMIN_STATE_NORMAL_OPERATION); - } - - return result; + vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NORMAL_OPERATION); } /* If create_flag, create a new index without first attempting to load an existing index. */ diff --git a/drivers/md/dm-vdo/dedupe.h b/drivers/md/dm-vdo/dedupe.h index 1566fc972ea7..9000d6f3eece 100644 --- a/drivers/md/dm-vdo/dedupe.h +++ b/drivers/md/dm-vdo/dedupe.h @@ -97,7 +97,7 @@ u64 vdo_get_dedupe_index_timeout_count(struct hash_zones *zones); int vdo_message_dedupe_index(struct hash_zones *zones, const char *name); -int vdo_add_dedupe_index_sysfs(struct hash_zones *zones); +void vdo_set_dedupe_state_normal(struct hash_zones *zones); void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag); diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 5f607dbb16e9..c6ee8161ba62 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -27,7 +27,6 @@ #include "logger.h" #include "memory-alloc.h" #include "message-stats.h" -#include "pool-sysfs.h" #include "recovery-journal.h" #include "repair.h" #include "slab-depot.h" @@ -36,7 +35,6 @@ #include "thread-device.h" #include "thread-registry.h" #include "types.h" -#include "uds-sysfs.h" #include "vdo.h" #include "vio.h" @@ -54,7 +52,6 @@ enum admin_phases { GROW_PHYSICAL_PHASE_END, GROW_PHYSICAL_PHASE_ERROR, LOAD_PHASE_START, - LOAD_PHASE_STATS, LOAD_PHASE_LOAD_DEPOT, LOAD_PHASE_MAKE_DIRTY, LOAD_PHASE_PREPARE_TO_ALLOCATE, @@ -104,7 +101,6 @@ static const char * const ADMIN_PHASE_NAMES[] = { "GROW_PHYSICAL_PHASE_END", "GROW_PHYSICAL_PHASE_ERROR", "LOAD_PHASE_START", - "LOAD_PHASE_STATS", "LOAD_PHASE_LOAD_DEPOT", "LOAD_PHASE_MAKE_DIRTY", "LOAD_PHASE_PREPARE_TO_ALLOCATE", @@ -947,8 +943,8 @@ static void vdo_io_hints(struct dm_target *ti, struct queue_limits *limits) * blocked task warnings in kernel logs. In order to avoid these warnings, we choose to * use the smallest reasonable value. * - * The value is displayed in sysfs, and also used by dm-thin to determine whether to pass - * down discards. The block layer splits large discards on this boundary when this is set. + * The value is used by dm-thin to determine whether to pass down discards. The block layer + * splits large discards on this boundary when this is set. */ limits->max_discard_sectors = (vdo->device_config->max_discard_blocks * VDO_SECTORS_PER_BLOCK); @@ -2174,32 +2170,6 @@ static enum slab_depot_load_type get_load_type(struct vdo *vdo) return VDO_SLAB_DEPOT_NORMAL_LOAD; } -/** - * vdo_initialize_kobjects() - Initialize the vdo sysfs directory. - * @vdo: The vdo being initialized. - * - * Return: VDO_SUCCESS or an error code. - */ -static int vdo_initialize_kobjects(struct vdo *vdo) -{ - int result; - struct dm_target *target = vdo->device_config->owning_target; - struct mapped_device *md = dm_table_get_md(target->table); - - kobject_init(&vdo->vdo_directory, &vdo_directory_type); - vdo->sysfs_added = true; - result = kobject_add(&vdo->vdo_directory, &disk_to_dev(dm_disk(md))->kobj, - "vdo"); - if (result != 0) - return VDO_CANT_ADD_SYSFS_NODE; - - result = vdo_add_dedupe_index_sysfs(vdo->hash_zones); - if (result != 0) - return VDO_CANT_ADD_SYSFS_NODE; - - return vdo_add_sysfs_stats_dir(vdo); -} - /** * load_callback() - Callback to do the destructive parts of loading a VDO. * @completion: The sub-task completion. @@ -2225,11 +2195,8 @@ static void load_callback(struct vdo_completion *completion) vdo_allow_read_only_mode_entry(completion); return; - case LOAD_PHASE_STATS: - vdo_continue_completion(completion, vdo_initialize_kobjects(vdo)); - return; - case LOAD_PHASE_LOAD_DEPOT: + vdo_set_dedupe_state_normal(vdo->hash_zones); if (vdo_is_read_only(vdo)) { /* * In read-only mode we don't use the allocator and it may not even be @@ -2866,7 +2833,7 @@ static void vdo_resume(struct dm_target *ti) static struct target_type vdo_target_bio = { .features = DM_TARGET_SINGLETON, .name = "vdo", - .version = { 8, 2, 0 }, + .version = { 9, 0, 0 }, .module = THIS_MODULE, .ctr = vdo_ctr, .dtr = vdo_dtr, @@ -2905,8 +2872,6 @@ static int __init vdo_init(void) /* Memory tracking must be initialized first for accurate accounting. */ vdo_memory_init(); - uds_init_sysfs(); - vdo_initialize_thread_device_registry(); vdo_initialize_device_registry_once(); uds_log_info("loaded version %s", CURRENT_VERSION); @@ -2933,7 +2898,6 @@ static int __init vdo_init(void) static void __exit vdo_exit(void) { vdo_module_destroy(); - uds_put_sysfs(); /* Memory tracking cleanup must be done last. */ vdo_memory_exit(); } diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index a50edb0331fc..bac28fff622d 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -56,11 +56,6 @@ int uds_get_log_level(void) return log_level; } -void uds_set_log_level(int new_log_level) -{ - log_level = new_log_level; -} - int uds_log_string_to_priority(const char *string) { int i; diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h index 4e2f18042ba7..ceb07aa3231f 100644 --- a/drivers/md/dm-vdo/logger.h +++ b/drivers/md/dm-vdo/logger.h @@ -37,8 +37,6 @@ int uds_get_log_level(void); -void uds_set_log_level(int new_log_level); - int uds_log_string_to_priority(const char *string); const char *uds_log_priority_to_string(int priority); diff --git a/drivers/md/dm-vdo/pool-sysfs-stats.c b/drivers/md/dm-vdo/pool-sysfs-stats.c deleted file mode 100644 index ae3838894a1c..000000000000 --- a/drivers/md/dm-vdo/pool-sysfs-stats.c +++ /dev/null @@ -1,2063 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -#include - -#include "logger.h" -#include "string-utils.h" - -#include "dedupe.h" -#include "pool-sysfs.h" -#include "statistics.h" -#include "vdo.h" - -struct pool_stats_attribute { - struct attribute attr; - ssize_t (*print)(struct vdo_statistics *stats, char *buf); -}; - -static ssize_t pool_stats_attr_show(struct kobject *directory, - struct attribute *attr, - char *buf) -{ - ssize_t size; - struct pool_stats_attribute *pool_stats_attr = - container_of(attr, struct pool_stats_attribute, attr); - struct vdo *vdo = container_of(directory, struct vdo, stats_directory); - - if (pool_stats_attr->print == NULL) - return -EINVAL; - - mutex_lock(&vdo->stats_mutex); - vdo_fetch_statistics(vdo, &vdo->stats_buffer); - size = pool_stats_attr->print(&vdo->stats_buffer, buf); - mutex_unlock(&vdo->stats_mutex); - - return size; -} - -const struct sysfs_ops vdo_pool_stats_sysfs_ops = { - .show = pool_stats_attr_show, - .store = NULL, -}; - -/* Number of blocks used for data */ -static ssize_t -pool_stats_print_data_blocks_used(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->data_blocks_used); -} - -static struct pool_stats_attribute pool_stats_attr_data_blocks_used = { - .attr = { .name = "data_blocks_used", .mode = 0444, }, - .print = pool_stats_print_data_blocks_used, -}; - -/* Number of blocks used for VDO metadata */ -static ssize_t -pool_stats_print_overhead_blocks_used(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->overhead_blocks_used); -} - -static struct pool_stats_attribute pool_stats_attr_overhead_blocks_used = { - .attr = { .name = "overhead_blocks_used", .mode = 0444, }, - .print = pool_stats_print_overhead_blocks_used, -}; - -/* Number of logical blocks that are currently mapped to physical blocks */ -static ssize_t -pool_stats_print_logical_blocks_used(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->logical_blocks_used); -} - -static struct pool_stats_attribute pool_stats_attr_logical_blocks_used = { - .attr = { .name = "logical_blocks_used", .mode = 0444, }, - .print = pool_stats_print_logical_blocks_used, -}; - -/* number of physical blocks */ -static ssize_t -pool_stats_print_physical_blocks(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->physical_blocks); -} - -static struct pool_stats_attribute pool_stats_attr_physical_blocks = { - .attr = { .name = "physical_blocks", .mode = 0444, }, - .print = pool_stats_print_physical_blocks, -}; - -/* number of logical blocks */ -static ssize_t -pool_stats_print_logical_blocks(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->logical_blocks); -} - -static struct pool_stats_attribute pool_stats_attr_logical_blocks = { - .attr = { .name = "logical_blocks", .mode = 0444, }, - .print = pool_stats_print_logical_blocks, -}; - -/* Size of the block map page cache, in bytes */ -static ssize_t -pool_stats_print_block_map_cache_size(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map_cache_size); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_cache_size = { - .attr = { .name = "block_map_cache_size", .mode = 0444, }, - .print = pool_stats_print_block_map_cache_size, -}; - -/* The physical block size */ -static ssize_t -pool_stats_print_block_size(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_size); -} - -static struct pool_stats_attribute pool_stats_attr_block_size = { - .attr = { .name = "block_size", .mode = 0444, }, - .print = pool_stats_print_block_size, -}; - -/* Number of times the VDO has successfully recovered */ -static ssize_t -pool_stats_print_complete_recoveries(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->complete_recoveries); -} - -static struct pool_stats_attribute pool_stats_attr_complete_recoveries = { - .attr = { .name = "complete_recoveries", .mode = 0444, }, - .print = pool_stats_print_complete_recoveries, -}; - -/* Number of times the VDO has recovered from read-only mode */ -static ssize_t -pool_stats_print_read_only_recoveries(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->read_only_recoveries); -} - -static struct pool_stats_attribute pool_stats_attr_read_only_recoveries = { - .attr = { .name = "read_only_recoveries", .mode = 0444, }, - .print = pool_stats_print_read_only_recoveries, -}; - -/* String describing the operating mode of the VDO */ -static ssize_t -pool_stats_print_mode(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%s\n", stats->mode); -} - -static struct pool_stats_attribute pool_stats_attr_mode = { - .attr = { .name = "mode", .mode = 0444, }, - .print = pool_stats_print_mode, -}; - -/* Whether the VDO is in recovery mode */ -static ssize_t -pool_stats_print_in_recovery_mode(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%d\n", stats->in_recovery_mode); -} - -static struct pool_stats_attribute pool_stats_attr_in_recovery_mode = { - .attr = { .name = "in_recovery_mode", .mode = 0444, }, - .print = pool_stats_print_in_recovery_mode, -}; - -/* What percentage of recovery mode work has been completed */ -static ssize_t -pool_stats_print_recovery_percentage(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->recovery_percentage); -} - -static struct pool_stats_attribute pool_stats_attr_recovery_percentage = { - .attr = { .name = "recovery_percentage", .mode = 0444, }, - .print = pool_stats_print_recovery_percentage, -}; - -/* Number of compressed data items written since startup */ -static ssize_t -pool_stats_print_packer_compressed_fragments_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->packer.compressed_fragments_written); -} - -static struct pool_stats_attribute pool_stats_attr_packer_compressed_fragments_written = { - .attr = { .name = "packer_compressed_fragments_written", .mode = 0444, }, - .print = pool_stats_print_packer_compressed_fragments_written, -}; - -/* Number of blocks containing compressed items written since startup */ -static ssize_t -pool_stats_print_packer_compressed_blocks_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->packer.compressed_blocks_written); -} - -static struct pool_stats_attribute pool_stats_attr_packer_compressed_blocks_written = { - .attr = { .name = "packer_compressed_blocks_written", .mode = 0444, }, - .print = pool_stats_print_packer_compressed_blocks_written, -}; - -/* Number of VIOs that are pending in the packer */ -static ssize_t -pool_stats_print_packer_compressed_fragments_in_packer(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->packer.compressed_fragments_in_packer); -} - -static struct pool_stats_attribute pool_stats_attr_packer_compressed_fragments_in_packer = { - .attr = { .name = "packer_compressed_fragments_in_packer", .mode = 0444, }, - .print = pool_stats_print_packer_compressed_fragments_in_packer, -}; - -/* The total number of slabs from which blocks may be allocated */ -static ssize_t -pool_stats_print_allocator_slab_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->allocator.slab_count); -} - -static struct pool_stats_attribute pool_stats_attr_allocator_slab_count = { - .attr = { .name = "allocator_slab_count", .mode = 0444, }, - .print = pool_stats_print_allocator_slab_count, -}; - -/* The total number of slabs from which blocks have ever been allocated */ -static ssize_t -pool_stats_print_allocator_slabs_opened(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->allocator.slabs_opened); -} - -static struct pool_stats_attribute pool_stats_attr_allocator_slabs_opened = { - .attr = { .name = "allocator_slabs_opened", .mode = 0444, }, - .print = pool_stats_print_allocator_slabs_opened, -}; - -/* The number of times since loading that a slab has been re-opened */ -static ssize_t -pool_stats_print_allocator_slabs_reopened(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->allocator.slabs_reopened); -} - -static struct pool_stats_attribute pool_stats_attr_allocator_slabs_reopened = { - .attr = { .name = "allocator_slabs_reopened", .mode = 0444, }, - .print = pool_stats_print_allocator_slabs_reopened, -}; - -/* Number of times the on-disk journal was full */ -static ssize_t -pool_stats_print_journal_disk_full(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.disk_full); -} - -static struct pool_stats_attribute pool_stats_attr_journal_disk_full = { - .attr = { .name = "journal_disk_full", .mode = 0444, }, - .print = pool_stats_print_journal_disk_full, -}; - -/* Number of times the recovery journal requested slab journal commits. */ -static ssize_t -pool_stats_print_journal_slab_journal_commits_requested(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.slab_journal_commits_requested); -} - -static struct pool_stats_attribute pool_stats_attr_journal_slab_journal_commits_requested = { - .attr = { .name = "journal_slab_journal_commits_requested", .mode = 0444, }, - .print = pool_stats_print_journal_slab_journal_commits_requested, -}; - -/* The total number of items on which processing has started */ -static ssize_t -pool_stats_print_journal_entries_started(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.entries.started); -} - -static struct pool_stats_attribute pool_stats_attr_journal_entries_started = { - .attr = { .name = "journal_entries_started", .mode = 0444, }, - .print = pool_stats_print_journal_entries_started, -}; - -/* The total number of items for which a write operation has been issued */ -static ssize_t -pool_stats_print_journal_entries_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.entries.written); -} - -static struct pool_stats_attribute pool_stats_attr_journal_entries_written = { - .attr = { .name = "journal_entries_written", .mode = 0444, }, - .print = pool_stats_print_journal_entries_written, -}; - -/* The total number of items for which a write operation has completed */ -static ssize_t -pool_stats_print_journal_entries_committed(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.entries.committed); -} - -static struct pool_stats_attribute pool_stats_attr_journal_entries_committed = { - .attr = { .name = "journal_entries_committed", .mode = 0444, }, - .print = pool_stats_print_journal_entries_committed, -}; - -/* The total number of items on which processing has started */ -static ssize_t -pool_stats_print_journal_blocks_started(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.blocks.started); -} - -static struct pool_stats_attribute pool_stats_attr_journal_blocks_started = { - .attr = { .name = "journal_blocks_started", .mode = 0444, }, - .print = pool_stats_print_journal_blocks_started, -}; - -/* The total number of items for which a write operation has been issued */ -static ssize_t -pool_stats_print_journal_blocks_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.blocks.written); -} - -static struct pool_stats_attribute pool_stats_attr_journal_blocks_written = { - .attr = { .name = "journal_blocks_written", .mode = 0444, }, - .print = pool_stats_print_journal_blocks_written, -}; - -/* The total number of items for which a write operation has completed */ -static ssize_t -pool_stats_print_journal_blocks_committed(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->journal.blocks.committed); -} - -static struct pool_stats_attribute pool_stats_attr_journal_blocks_committed = { - .attr = { .name = "journal_blocks_committed", .mode = 0444, }, - .print = pool_stats_print_journal_blocks_committed, -}; - -/* Number of times the on-disk journal was full */ -static ssize_t -pool_stats_print_slab_journal_disk_full_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->slab_journal.disk_full_count); -} - -static struct pool_stats_attribute pool_stats_attr_slab_journal_disk_full_count = { - .attr = { .name = "slab_journal_disk_full_count", .mode = 0444, }, - .print = pool_stats_print_slab_journal_disk_full_count, -}; - -/* Number of times an entry was added over the flush threshold */ -static ssize_t -pool_stats_print_slab_journal_flush_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->slab_journal.flush_count); -} - -static struct pool_stats_attribute pool_stats_attr_slab_journal_flush_count = { - .attr = { .name = "slab_journal_flush_count", .mode = 0444, }, - .print = pool_stats_print_slab_journal_flush_count, -}; - -/* Number of times an entry was added over the block threshold */ -static ssize_t -pool_stats_print_slab_journal_blocked_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->slab_journal.blocked_count); -} - -static struct pool_stats_attribute pool_stats_attr_slab_journal_blocked_count = { - .attr = { .name = "slab_journal_blocked_count", .mode = 0444, }, - .print = pool_stats_print_slab_journal_blocked_count, -}; - -/* Number of times a tail block was written */ -static ssize_t -pool_stats_print_slab_journal_blocks_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->slab_journal.blocks_written); -} - -static struct pool_stats_attribute pool_stats_attr_slab_journal_blocks_written = { - .attr = { .name = "slab_journal_blocks_written", .mode = 0444, }, - .print = pool_stats_print_slab_journal_blocks_written, -}; - -/* Number of times we had to wait for the tail to write */ -static ssize_t -pool_stats_print_slab_journal_tail_busy_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->slab_journal.tail_busy_count); -} - -static struct pool_stats_attribute pool_stats_attr_slab_journal_tail_busy_count = { - .attr = { .name = "slab_journal_tail_busy_count", .mode = 0444, }, - .print = pool_stats_print_slab_journal_tail_busy_count, -}; - -/* Number of blocks written */ -static ssize_t -pool_stats_print_slab_summary_blocks_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->slab_summary.blocks_written); -} - -static struct pool_stats_attribute pool_stats_attr_slab_summary_blocks_written = { - .attr = { .name = "slab_summary_blocks_written", .mode = 0444, }, - .print = pool_stats_print_slab_summary_blocks_written, -}; - -/* Number of reference blocks written */ -static ssize_t -pool_stats_print_ref_counts_blocks_written(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->ref_counts.blocks_written); -} - -static struct pool_stats_attribute pool_stats_attr_ref_counts_blocks_written = { - .attr = { .name = "ref_counts_blocks_written", .mode = 0444, }, - .print = pool_stats_print_ref_counts_blocks_written, -}; - -/* number of dirty (resident) pages */ -static ssize_t -pool_stats_print_block_map_dirty_pages(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.dirty_pages); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_dirty_pages = { - .attr = { .name = "block_map_dirty_pages", .mode = 0444, }, - .print = pool_stats_print_block_map_dirty_pages, -}; - -/* number of clean (resident) pages */ -static ssize_t -pool_stats_print_block_map_clean_pages(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.clean_pages); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_clean_pages = { - .attr = { .name = "block_map_clean_pages", .mode = 0444, }, - .print = pool_stats_print_block_map_clean_pages, -}; - -/* number of free pages */ -static ssize_t -pool_stats_print_block_map_free_pages(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.free_pages); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_free_pages = { - .attr = { .name = "block_map_free_pages", .mode = 0444, }, - .print = pool_stats_print_block_map_free_pages, -}; - -/* number of pages in failed state */ -static ssize_t -pool_stats_print_block_map_failed_pages(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.failed_pages); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_failed_pages = { - .attr = { .name = "block_map_failed_pages", .mode = 0444, }, - .print = pool_stats_print_block_map_failed_pages, -}; - -/* number of pages incoming */ -static ssize_t -pool_stats_print_block_map_incoming_pages(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.incoming_pages); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_incoming_pages = { - .attr = { .name = "block_map_incoming_pages", .mode = 0444, }, - .print = pool_stats_print_block_map_incoming_pages, -}; - -/* number of pages outgoing */ -static ssize_t -pool_stats_print_block_map_outgoing_pages(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.outgoing_pages); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_outgoing_pages = { - .attr = { .name = "block_map_outgoing_pages", .mode = 0444, }, - .print = pool_stats_print_block_map_outgoing_pages, -}; - -/* how many times free page not avail */ -static ssize_t -pool_stats_print_block_map_cache_pressure(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->block_map.cache_pressure); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_cache_pressure = { - .attr = { .name = "block_map_cache_pressure", .mode = 0444, }, - .print = pool_stats_print_block_map_cache_pressure, -}; - -/* number of get_vdo_page() calls for read */ -static ssize_t -pool_stats_print_block_map_read_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.read_count); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_read_count = { - .attr = { .name = "block_map_read_count", .mode = 0444, }, - .print = pool_stats_print_block_map_read_count, -}; - -/* number of get_vdo_page() calls for write */ -static ssize_t -pool_stats_print_block_map_write_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.write_count); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_write_count = { - .attr = { .name = "block_map_write_count", .mode = 0444, }, - .print = pool_stats_print_block_map_write_count, -}; - -/* number of times pages failed to read */ -static ssize_t -pool_stats_print_block_map_failed_reads(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.failed_reads); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_failed_reads = { - .attr = { .name = "block_map_failed_reads", .mode = 0444, }, - .print = pool_stats_print_block_map_failed_reads, -}; - -/* number of times pages failed to write */ -static ssize_t -pool_stats_print_block_map_failed_writes(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.failed_writes); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_failed_writes = { - .attr = { .name = "block_map_failed_writes", .mode = 0444, }, - .print = pool_stats_print_block_map_failed_writes, -}; - -/* number of gets that are reclaimed */ -static ssize_t -pool_stats_print_block_map_reclaimed(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.reclaimed); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_reclaimed = { - .attr = { .name = "block_map_reclaimed", .mode = 0444, }, - .print = pool_stats_print_block_map_reclaimed, -}; - -/* number of gets for outgoing pages */ -static ssize_t -pool_stats_print_block_map_read_outgoing(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.read_outgoing); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_read_outgoing = { - .attr = { .name = "block_map_read_outgoing", .mode = 0444, }, - .print = pool_stats_print_block_map_read_outgoing, -}; - -/* number of gets that were already there */ -static ssize_t -pool_stats_print_block_map_found_in_cache(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.found_in_cache); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_found_in_cache = { - .attr = { .name = "block_map_found_in_cache", .mode = 0444, }, - .print = pool_stats_print_block_map_found_in_cache, -}; - -/* number of gets requiring discard */ -static ssize_t -pool_stats_print_block_map_discard_required(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.discard_required); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_discard_required = { - .attr = { .name = "block_map_discard_required", .mode = 0444, }, - .print = pool_stats_print_block_map_discard_required, -}; - -/* number of gets enqueued for their page */ -static ssize_t -pool_stats_print_block_map_wait_for_page(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.wait_for_page); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_wait_for_page = { - .attr = { .name = "block_map_wait_for_page", .mode = 0444, }, - .print = pool_stats_print_block_map_wait_for_page, -}; - -/* number of gets that have to fetch */ -static ssize_t -pool_stats_print_block_map_fetch_required(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.fetch_required); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_fetch_required = { - .attr = { .name = "block_map_fetch_required", .mode = 0444, }, - .print = pool_stats_print_block_map_fetch_required, -}; - -/* number of page fetches */ -static ssize_t -pool_stats_print_block_map_pages_loaded(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.pages_loaded); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_pages_loaded = { - .attr = { .name = "block_map_pages_loaded", .mode = 0444, }, - .print = pool_stats_print_block_map_pages_loaded, -}; - -/* number of page saves */ -static ssize_t -pool_stats_print_block_map_pages_saved(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.pages_saved); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_pages_saved = { - .attr = { .name = "block_map_pages_saved", .mode = 0444, }, - .print = pool_stats_print_block_map_pages_saved, -}; - -/* the number of flushes issued */ -static ssize_t -pool_stats_print_block_map_flush_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->block_map.flush_count); -} - -static struct pool_stats_attribute pool_stats_attr_block_map_flush_count = { - .attr = { .name = "block_map_flush_count", .mode = 0444, }, - .print = pool_stats_print_block_map_flush_count, -}; - -/* Number of times the UDS advice proved correct */ -static ssize_t -pool_stats_print_hash_lock_dedupe_advice_valid(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->hash_lock.dedupe_advice_valid); -} - -static struct pool_stats_attribute pool_stats_attr_hash_lock_dedupe_advice_valid = { - .attr = { .name = "hash_lock_dedupe_advice_valid", .mode = 0444, }, - .print = pool_stats_print_hash_lock_dedupe_advice_valid, -}; - -/* Number of times the UDS advice proved incorrect */ -static ssize_t -pool_stats_print_hash_lock_dedupe_advice_stale(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->hash_lock.dedupe_advice_stale); -} - -static struct pool_stats_attribute pool_stats_attr_hash_lock_dedupe_advice_stale = { - .attr = { .name = "hash_lock_dedupe_advice_stale", .mode = 0444, }, - .print = pool_stats_print_hash_lock_dedupe_advice_stale, -}; - -/* Number of writes with the same data as another in-flight write */ -static ssize_t -pool_stats_print_hash_lock_concurrent_data_matches(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->hash_lock.concurrent_data_matches); -} - -static struct pool_stats_attribute pool_stats_attr_hash_lock_concurrent_data_matches = { - .attr = { .name = "hash_lock_concurrent_data_matches", .mode = 0444, }, - .print = pool_stats_print_hash_lock_concurrent_data_matches, -}; - -/* Number of writes whose hash collided with an in-flight write */ -static ssize_t -pool_stats_print_hash_lock_concurrent_hash_collisions(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->hash_lock.concurrent_hash_collisions); -} - -static struct pool_stats_attribute pool_stats_attr_hash_lock_concurrent_hash_collisions = { - .attr = { .name = "hash_lock_concurrent_hash_collisions", .mode = 0444, }, - .print = pool_stats_print_hash_lock_concurrent_hash_collisions, -}; - -/* Current number of dedupe queries that are in flight */ -static ssize_t -pool_stats_print_hash_lock_curr_dedupe_queries(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->hash_lock.curr_dedupe_queries); -} - -static struct pool_stats_attribute pool_stats_attr_hash_lock_curr_dedupe_queries = { - .attr = { .name = "hash_lock_curr_dedupe_queries", .mode = 0444, }, - .print = pool_stats_print_hash_lock_curr_dedupe_queries, -}; - -/* number of times VDO got an invalid dedupe advice PBN from UDS */ -static ssize_t -pool_stats_print_errors_invalid_advice_pbn_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->errors.invalid_advice_pbn_count); -} - -static struct pool_stats_attribute pool_stats_attr_errors_invalid_advice_pbn_count = { - .attr = { .name = "errors_invalid_advice_pbn_count", .mode = 0444, }, - .print = pool_stats_print_errors_invalid_advice_pbn_count, -}; - -/* number of times a VIO completed with a VDO_NO_SPACE error */ -static ssize_t -pool_stats_print_errors_no_space_error_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->errors.no_space_error_count); -} - -static struct pool_stats_attribute pool_stats_attr_errors_no_space_error_count = { - .attr = { .name = "errors_no_space_error_count", .mode = 0444, }, - .print = pool_stats_print_errors_no_space_error_count, -}; - -/* number of times a VIO completed with a VDO_READ_ONLY error */ -static ssize_t -pool_stats_print_errors_read_only_error_count(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->errors.read_only_error_count); -} - -static struct pool_stats_attribute pool_stats_attr_errors_read_only_error_count = { - .attr = { .name = "errors_read_only_error_count", .mode = 0444, }, - .print = pool_stats_print_errors_read_only_error_count, -}; - -/* The VDO instance */ -static ssize_t -pool_stats_print_instance(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->instance); -} - -static struct pool_stats_attribute pool_stats_attr_instance = { - .attr = { .name = "instance", .mode = 0444, }, - .print = pool_stats_print_instance, -}; - -/* Current number of active VIOs */ -static ssize_t -pool_stats_print_current_vios_in_progress(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->current_vios_in_progress); -} - -static struct pool_stats_attribute pool_stats_attr_current_vios_in_progress = { - .attr = { .name = "current_vios_in_progress", .mode = 0444, }, - .print = pool_stats_print_current_vios_in_progress, -}; - -/* Maximum number of active VIOs */ -static ssize_t -pool_stats_print_max_vios(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%u\n", stats->max_vios); -} - -static struct pool_stats_attribute pool_stats_attr_max_vios = { - .attr = { .name = "max_vios", .mode = 0444, }, - .print = pool_stats_print_max_vios, -}; - -/* Number of times the UDS index was too slow in responding */ -static ssize_t -pool_stats_print_dedupe_advice_timeouts(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->dedupe_advice_timeouts); -} - -static struct pool_stats_attribute pool_stats_attr_dedupe_advice_timeouts = { - .attr = { .name = "dedupe_advice_timeouts", .mode = 0444, }, - .print = pool_stats_print_dedupe_advice_timeouts, -}; - -/* Number of flush requests submitted to the storage device */ -static ssize_t -pool_stats_print_flush_out(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->flush_out); -} - -static struct pool_stats_attribute pool_stats_attr_flush_out = { - .attr = { .name = "flush_out", .mode = 0444, }, - .print = pool_stats_print_flush_out, -}; - -/* Logical block size */ -static ssize_t -pool_stats_print_logical_block_size(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->logical_block_size); -} - -static struct pool_stats_attribute pool_stats_attr_logical_block_size = { - .attr = { .name = "logical_block_size", .mode = 0444, }, - .print = pool_stats_print_logical_block_size, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_in_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_read = { - .attr = { .name = "bios_in_read", .mode = 0444, }, - .print = pool_stats_print_bios_in_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_in_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_write = { - .attr = { .name = "bios_in_write", .mode = 0444, }, - .print = pool_stats_print_bios_in_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_in_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_empty_flush = { - .attr = { .name = "bios_in_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_in_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_in_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_discard = { - .attr = { .name = "bios_in_discard", .mode = 0444, }, - .print = pool_stats_print_bios_in_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_in_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_flush = { - .attr = { .name = "bios_in_flush", .mode = 0444, }, - .print = pool_stats_print_bios_in_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_in_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_fua = { - .attr = { .name = "bios_in_fua", .mode = 0444, }, - .print = pool_stats_print_bios_in_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_in_partial_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_partial.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_partial_read = { - .attr = { .name = "bios_in_partial_read", .mode = 0444, }, - .print = pool_stats_print_bios_in_partial_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_in_partial_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_partial.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_partial_write = { - .attr = { .name = "bios_in_partial_write", .mode = 0444, }, - .print = pool_stats_print_bios_in_partial_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_in_partial_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_partial.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_partial_empty_flush = { - .attr = { .name = "bios_in_partial_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_in_partial_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_in_partial_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_partial.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_partial_discard = { - .attr = { .name = "bios_in_partial_discard", .mode = 0444, }, - .print = pool_stats_print_bios_in_partial_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_in_partial_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_partial.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_partial_flush = { - .attr = { .name = "bios_in_partial_flush", .mode = 0444, }, - .print = pool_stats_print_bios_in_partial_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_in_partial_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_partial.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_partial_fua = { - .attr = { .name = "bios_in_partial_fua", .mode = 0444, }, - .print = pool_stats_print_bios_in_partial_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_out_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_read = { - .attr = { .name = "bios_out_read", .mode = 0444, }, - .print = pool_stats_print_bios_out_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_out_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_write = { - .attr = { .name = "bios_out_write", .mode = 0444, }, - .print = pool_stats_print_bios_out_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_out_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_empty_flush = { - .attr = { .name = "bios_out_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_out_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_out_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_discard = { - .attr = { .name = "bios_out_discard", .mode = 0444, }, - .print = pool_stats_print_bios_out_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_out_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_flush = { - .attr = { .name = "bios_out_flush", .mode = 0444, }, - .print = pool_stats_print_bios_out_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_out_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_fua = { - .attr = { .name = "bios_out_fua", .mode = 0444, }, - .print = pool_stats_print_bios_out_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_meta_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_read = { - .attr = { .name = "bios_meta_read", .mode = 0444, }, - .print = pool_stats_print_bios_meta_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_meta_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_write = { - .attr = { .name = "bios_meta_write", .mode = 0444, }, - .print = pool_stats_print_bios_meta_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_meta_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_empty_flush = { - .attr = { .name = "bios_meta_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_meta_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_meta_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_discard = { - .attr = { .name = "bios_meta_discard", .mode = 0444, }, - .print = pool_stats_print_bios_meta_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_meta_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_flush = { - .attr = { .name = "bios_meta_flush", .mode = 0444, }, - .print = pool_stats_print_bios_meta_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_meta_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_fua = { - .attr = { .name = "bios_meta_fua", .mode = 0444, }, - .print = pool_stats_print_bios_meta_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_journal_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_read = { - .attr = { .name = "bios_journal_read", .mode = 0444, }, - .print = pool_stats_print_bios_journal_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_journal_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_write = { - .attr = { .name = "bios_journal_write", .mode = 0444, }, - .print = pool_stats_print_bios_journal_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_journal_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_empty_flush = { - .attr = { .name = "bios_journal_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_journal_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_journal_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_discard = { - .attr = { .name = "bios_journal_discard", .mode = 0444, }, - .print = pool_stats_print_bios_journal_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_journal_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_flush = { - .attr = { .name = "bios_journal_flush", .mode = 0444, }, - .print = pool_stats_print_bios_journal_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_journal_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_fua = { - .attr = { .name = "bios_journal_fua", .mode = 0444, }, - .print = pool_stats_print_bios_journal_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_page_cache_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_read = { - .attr = { .name = "bios_page_cache_read", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_page_cache_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_write = { - .attr = { .name = "bios_page_cache_write", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_page_cache_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_empty_flush = { - .attr = { .name = "bios_page_cache_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_page_cache_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_discard = { - .attr = { .name = "bios_page_cache_discard", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_page_cache_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_flush = { - .attr = { .name = "bios_page_cache_flush", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_page_cache_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_fua = { - .attr = { .name = "bios_page_cache_fua", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_out_completed_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out_completed.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_completed_read = { - .attr = { .name = "bios_out_completed_read", .mode = 0444, }, - .print = pool_stats_print_bios_out_completed_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_out_completed_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out_completed.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_completed_write = { - .attr = { .name = "bios_out_completed_write", .mode = 0444, }, - .print = pool_stats_print_bios_out_completed_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_out_completed_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out_completed.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_completed_empty_flush = { - .attr = { .name = "bios_out_completed_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_out_completed_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_out_completed_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out_completed.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_completed_discard = { - .attr = { .name = "bios_out_completed_discard", .mode = 0444, }, - .print = pool_stats_print_bios_out_completed_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_out_completed_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out_completed.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_completed_flush = { - .attr = { .name = "bios_out_completed_flush", .mode = 0444, }, - .print = pool_stats_print_bios_out_completed_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_out_completed_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_out_completed.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_out_completed_fua = { - .attr = { .name = "bios_out_completed_fua", .mode = 0444, }, - .print = pool_stats_print_bios_out_completed_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_meta_completed_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta_completed.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_read = { - .attr = { .name = "bios_meta_completed_read", .mode = 0444, }, - .print = pool_stats_print_bios_meta_completed_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_meta_completed_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta_completed.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_write = { - .attr = { .name = "bios_meta_completed_write", .mode = 0444, }, - .print = pool_stats_print_bios_meta_completed_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_meta_completed_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta_completed.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_empty_flush = { - .attr = { .name = "bios_meta_completed_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_meta_completed_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_meta_completed_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta_completed.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_discard = { - .attr = { .name = "bios_meta_completed_discard", .mode = 0444, }, - .print = pool_stats_print_bios_meta_completed_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_meta_completed_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta_completed.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_flush = { - .attr = { .name = "bios_meta_completed_flush", .mode = 0444, }, - .print = pool_stats_print_bios_meta_completed_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_meta_completed_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_meta_completed.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_meta_completed_fua = { - .attr = { .name = "bios_meta_completed_fua", .mode = 0444, }, - .print = pool_stats_print_bios_meta_completed_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_journal_completed_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal_completed.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_read = { - .attr = { .name = "bios_journal_completed_read", .mode = 0444, }, - .print = pool_stats_print_bios_journal_completed_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_journal_completed_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal_completed.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_write = { - .attr = { .name = "bios_journal_completed_write", .mode = 0444, }, - .print = pool_stats_print_bios_journal_completed_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_journal_completed_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal_completed.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_empty_flush = { - .attr = { .name = "bios_journal_completed_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_journal_completed_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_journal_completed_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal_completed.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_discard = { - .attr = { .name = "bios_journal_completed_discard", .mode = 0444, }, - .print = pool_stats_print_bios_journal_completed_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_journal_completed_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal_completed.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_flush = { - .attr = { .name = "bios_journal_completed_flush", .mode = 0444, }, - .print = pool_stats_print_bios_journal_completed_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_journal_completed_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_journal_completed.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_journal_completed_fua = { - .attr = { .name = "bios_journal_completed_fua", .mode = 0444, }, - .print = pool_stats_print_bios_journal_completed_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_page_cache_completed_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_read = { - .attr = { .name = "bios_page_cache_completed_read", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_completed_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_page_cache_completed_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_write = { - .attr = { .name = "bios_page_cache_completed_write", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_completed_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_page_cache_completed_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_empty_flush = { - .attr = { .name = "bios_page_cache_completed_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_completed_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_page_cache_completed_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_discard = { - .attr = { .name = "bios_page_cache_completed_discard", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_completed_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_page_cache_completed_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_flush = { - .attr = { .name = "bios_page_cache_completed_flush", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_completed_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_page_cache_completed_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_page_cache_completed.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_page_cache_completed_fua = { - .attr = { .name = "bios_page_cache_completed_fua", .mode = 0444, }, - .print = pool_stats_print_bios_page_cache_completed_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_acknowledged_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_read = { - .attr = { .name = "bios_acknowledged_read", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_acknowledged_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_write = { - .attr = { .name = "bios_acknowledged_write", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_acknowledged_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_empty_flush = { - .attr = { .name = "bios_acknowledged_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_acknowledged_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_discard = { - .attr = { .name = "bios_acknowledged_discard", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_acknowledged_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_flush = { - .attr = { .name = "bios_acknowledged_flush", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_acknowledged_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_fua = { - .attr = { .name = "bios_acknowledged_fua", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_acknowledged_partial_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_read = { - .attr = { .name = "bios_acknowledged_partial_read", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_partial_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_acknowledged_partial_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_write = { - .attr = { .name = "bios_acknowledged_partial_write", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_partial_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_acknowledged_partial_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_empty_flush = { - .attr = { .name = "bios_acknowledged_partial_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_partial_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_acknowledged_partial_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_discard = { - .attr = { .name = "bios_acknowledged_partial_discard", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_partial_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_acknowledged_partial_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_flush = { - .attr = { .name = "bios_acknowledged_partial_flush", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_partial_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_acknowledged_partial_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_acknowledged_partial.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_acknowledged_partial_fua = { - .attr = { .name = "bios_acknowledged_partial_fua", .mode = 0444, }, - .print = pool_stats_print_bios_acknowledged_partial_fua, -}; - -/* Number of REQ_OP_READ bios */ -static ssize_t -pool_stats_print_bios_in_progress_read(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_progress.read); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_progress_read = { - .attr = { .name = "bios_in_progress_read", .mode = 0444, }, - .print = pool_stats_print_bios_in_progress_read, -}; - -/* Number of REQ_OP_WRITE bios with data */ -static ssize_t -pool_stats_print_bios_in_progress_write(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_progress.write); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_progress_write = { - .attr = { .name = "bios_in_progress_write", .mode = 0444, }, - .print = pool_stats_print_bios_in_progress_write, -}; - -/* Number of bios tagged with REQ_PREFLUSH and containing no data */ -static ssize_t -pool_stats_print_bios_in_progress_empty_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_progress.empty_flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_progress_empty_flush = { - .attr = { .name = "bios_in_progress_empty_flush", .mode = 0444, }, - .print = pool_stats_print_bios_in_progress_empty_flush, -}; - -/* Number of REQ_OP_DISCARD bios */ -static ssize_t -pool_stats_print_bios_in_progress_discard(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_progress.discard); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_progress_discard = { - .attr = { .name = "bios_in_progress_discard", .mode = 0444, }, - .print = pool_stats_print_bios_in_progress_discard, -}; - -/* Number of bios tagged with REQ_PREFLUSH */ -static ssize_t -pool_stats_print_bios_in_progress_flush(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_progress.flush); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_progress_flush = { - .attr = { .name = "bios_in_progress_flush", .mode = 0444, }, - .print = pool_stats_print_bios_in_progress_flush, -}; - -/* Number of bios tagged with REQ_FUA */ -static ssize_t -pool_stats_print_bios_in_progress_fua(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->bios_in_progress.fua); -} - -static struct pool_stats_attribute pool_stats_attr_bios_in_progress_fua = { - .attr = { .name = "bios_in_progress_fua", .mode = 0444, }, - .print = pool_stats_print_bios_in_progress_fua, -}; - -/* Tracked bytes currently allocated. */ -static ssize_t -pool_stats_print_memory_usage_bytes_used(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->memory_usage.bytes_used); -} - -static struct pool_stats_attribute pool_stats_attr_memory_usage_bytes_used = { - .attr = { .name = "memory_usage_bytes_used", .mode = 0444, }, - .print = pool_stats_print_memory_usage_bytes_used, -}; - -/* Maximum tracked bytes allocated. */ -static ssize_t -pool_stats_print_memory_usage_peak_bytes_used(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->memory_usage.peak_bytes_used); -} - -static struct pool_stats_attribute pool_stats_attr_memory_usage_peak_bytes_used = { - .attr = { .name = "memory_usage_peak_bytes_used", .mode = 0444, }, - .print = pool_stats_print_memory_usage_peak_bytes_used, -}; - -/* Number of records stored in the index */ -static ssize_t -pool_stats_print_index_entries_indexed(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.entries_indexed); -} - -static struct pool_stats_attribute pool_stats_attr_index_entries_indexed = { - .attr = { .name = "index_entries_indexed", .mode = 0444, }, - .print = pool_stats_print_index_entries_indexed, -}; - -/* Number of post calls that found an existing entry */ -static ssize_t -pool_stats_print_index_posts_found(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.posts_found); -} - -static struct pool_stats_attribute pool_stats_attr_index_posts_found = { - .attr = { .name = "index_posts_found", .mode = 0444, }, - .print = pool_stats_print_index_posts_found, -}; - -/* Number of post calls that added a new entry */ -static ssize_t -pool_stats_print_index_posts_not_found(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.posts_not_found); -} - -static struct pool_stats_attribute pool_stats_attr_index_posts_not_found = { - .attr = { .name = "index_posts_not_found", .mode = 0444, }, - .print = pool_stats_print_index_posts_not_found, -}; - -/* Number of query calls that found an existing entry */ -static ssize_t -pool_stats_print_index_queries_found(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.queries_found); -} - -static struct pool_stats_attribute pool_stats_attr_index_queries_found = { - .attr = { .name = "index_queries_found", .mode = 0444, }, - .print = pool_stats_print_index_queries_found, -}; - -/* Number of query calls that added a new entry */ -static ssize_t -pool_stats_print_index_queries_not_found(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.queries_not_found); -} - -static struct pool_stats_attribute pool_stats_attr_index_queries_not_found = { - .attr = { .name = "index_queries_not_found", .mode = 0444, }, - .print = pool_stats_print_index_queries_not_found, -}; - -/* Number of update calls that found an existing entry */ -static ssize_t -pool_stats_print_index_updates_found(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.updates_found); -} - -static struct pool_stats_attribute pool_stats_attr_index_updates_found = { - .attr = { .name = "index_updates_found", .mode = 0444, }, - .print = pool_stats_print_index_updates_found, -}; - -/* Number of update calls that added a new entry */ -static ssize_t -pool_stats_print_index_updates_not_found(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.updates_not_found); -} - -static struct pool_stats_attribute pool_stats_attr_index_updates_not_found = { - .attr = { .name = "index_updates_not_found", .mode = 0444, }, - .print = pool_stats_print_index_updates_not_found, -}; - -/* Number of entries discarded */ -static ssize_t -pool_stats_print_index_entries_discarded(struct vdo_statistics *stats, char *buf) -{ - return sprintf(buf, "%llu\n", stats->index.entries_discarded); -} - -static struct pool_stats_attribute pool_stats_attr_index_entries_discarded = { - .attr = { .name = "index_entries_discarded", .mode = 0444, }, - .print = pool_stats_print_index_entries_discarded, -}; - -struct attribute *vdo_pool_stats_attrs[] = { - &pool_stats_attr_data_blocks_used.attr, - &pool_stats_attr_overhead_blocks_used.attr, - &pool_stats_attr_logical_blocks_used.attr, - &pool_stats_attr_physical_blocks.attr, - &pool_stats_attr_logical_blocks.attr, - &pool_stats_attr_block_map_cache_size.attr, - &pool_stats_attr_block_size.attr, - &pool_stats_attr_complete_recoveries.attr, - &pool_stats_attr_read_only_recoveries.attr, - &pool_stats_attr_mode.attr, - &pool_stats_attr_in_recovery_mode.attr, - &pool_stats_attr_recovery_percentage.attr, - &pool_stats_attr_packer_compressed_fragments_written.attr, - &pool_stats_attr_packer_compressed_blocks_written.attr, - &pool_stats_attr_packer_compressed_fragments_in_packer.attr, - &pool_stats_attr_allocator_slab_count.attr, - &pool_stats_attr_allocator_slabs_opened.attr, - &pool_stats_attr_allocator_slabs_reopened.attr, - &pool_stats_attr_journal_disk_full.attr, - &pool_stats_attr_journal_slab_journal_commits_requested.attr, - &pool_stats_attr_journal_entries_started.attr, - &pool_stats_attr_journal_entries_written.attr, - &pool_stats_attr_journal_entries_committed.attr, - &pool_stats_attr_journal_blocks_started.attr, - &pool_stats_attr_journal_blocks_written.attr, - &pool_stats_attr_journal_blocks_committed.attr, - &pool_stats_attr_slab_journal_disk_full_count.attr, - &pool_stats_attr_slab_journal_flush_count.attr, - &pool_stats_attr_slab_journal_blocked_count.attr, - &pool_stats_attr_slab_journal_blocks_written.attr, - &pool_stats_attr_slab_journal_tail_busy_count.attr, - &pool_stats_attr_slab_summary_blocks_written.attr, - &pool_stats_attr_ref_counts_blocks_written.attr, - &pool_stats_attr_block_map_dirty_pages.attr, - &pool_stats_attr_block_map_clean_pages.attr, - &pool_stats_attr_block_map_free_pages.attr, - &pool_stats_attr_block_map_failed_pages.attr, - &pool_stats_attr_block_map_incoming_pages.attr, - &pool_stats_attr_block_map_outgoing_pages.attr, - &pool_stats_attr_block_map_cache_pressure.attr, - &pool_stats_attr_block_map_read_count.attr, - &pool_stats_attr_block_map_write_count.attr, - &pool_stats_attr_block_map_failed_reads.attr, - &pool_stats_attr_block_map_failed_writes.attr, - &pool_stats_attr_block_map_reclaimed.attr, - &pool_stats_attr_block_map_read_outgoing.attr, - &pool_stats_attr_block_map_found_in_cache.attr, - &pool_stats_attr_block_map_discard_required.attr, - &pool_stats_attr_block_map_wait_for_page.attr, - &pool_stats_attr_block_map_fetch_required.attr, - &pool_stats_attr_block_map_pages_loaded.attr, - &pool_stats_attr_block_map_pages_saved.attr, - &pool_stats_attr_block_map_flush_count.attr, - &pool_stats_attr_hash_lock_dedupe_advice_valid.attr, - &pool_stats_attr_hash_lock_dedupe_advice_stale.attr, - &pool_stats_attr_hash_lock_concurrent_data_matches.attr, - &pool_stats_attr_hash_lock_concurrent_hash_collisions.attr, - &pool_stats_attr_hash_lock_curr_dedupe_queries.attr, - &pool_stats_attr_errors_invalid_advice_pbn_count.attr, - &pool_stats_attr_errors_no_space_error_count.attr, - &pool_stats_attr_errors_read_only_error_count.attr, - &pool_stats_attr_instance.attr, - &pool_stats_attr_current_vios_in_progress.attr, - &pool_stats_attr_max_vios.attr, - &pool_stats_attr_dedupe_advice_timeouts.attr, - &pool_stats_attr_flush_out.attr, - &pool_stats_attr_logical_block_size.attr, - &pool_stats_attr_bios_in_read.attr, - &pool_stats_attr_bios_in_write.attr, - &pool_stats_attr_bios_in_empty_flush.attr, - &pool_stats_attr_bios_in_discard.attr, - &pool_stats_attr_bios_in_flush.attr, - &pool_stats_attr_bios_in_fua.attr, - &pool_stats_attr_bios_in_partial_read.attr, - &pool_stats_attr_bios_in_partial_write.attr, - &pool_stats_attr_bios_in_partial_empty_flush.attr, - &pool_stats_attr_bios_in_partial_discard.attr, - &pool_stats_attr_bios_in_partial_flush.attr, - &pool_stats_attr_bios_in_partial_fua.attr, - &pool_stats_attr_bios_out_read.attr, - &pool_stats_attr_bios_out_write.attr, - &pool_stats_attr_bios_out_empty_flush.attr, - &pool_stats_attr_bios_out_discard.attr, - &pool_stats_attr_bios_out_flush.attr, - &pool_stats_attr_bios_out_fua.attr, - &pool_stats_attr_bios_meta_read.attr, - &pool_stats_attr_bios_meta_write.attr, - &pool_stats_attr_bios_meta_empty_flush.attr, - &pool_stats_attr_bios_meta_discard.attr, - &pool_stats_attr_bios_meta_flush.attr, - &pool_stats_attr_bios_meta_fua.attr, - &pool_stats_attr_bios_journal_read.attr, - &pool_stats_attr_bios_journal_write.attr, - &pool_stats_attr_bios_journal_empty_flush.attr, - &pool_stats_attr_bios_journal_discard.attr, - &pool_stats_attr_bios_journal_flush.attr, - &pool_stats_attr_bios_journal_fua.attr, - &pool_stats_attr_bios_page_cache_read.attr, - &pool_stats_attr_bios_page_cache_write.attr, - &pool_stats_attr_bios_page_cache_empty_flush.attr, - &pool_stats_attr_bios_page_cache_discard.attr, - &pool_stats_attr_bios_page_cache_flush.attr, - &pool_stats_attr_bios_page_cache_fua.attr, - &pool_stats_attr_bios_out_completed_read.attr, - &pool_stats_attr_bios_out_completed_write.attr, - &pool_stats_attr_bios_out_completed_empty_flush.attr, - &pool_stats_attr_bios_out_completed_discard.attr, - &pool_stats_attr_bios_out_completed_flush.attr, - &pool_stats_attr_bios_out_completed_fua.attr, - &pool_stats_attr_bios_meta_completed_read.attr, - &pool_stats_attr_bios_meta_completed_write.attr, - &pool_stats_attr_bios_meta_completed_empty_flush.attr, - &pool_stats_attr_bios_meta_completed_discard.attr, - &pool_stats_attr_bios_meta_completed_flush.attr, - &pool_stats_attr_bios_meta_completed_fua.attr, - &pool_stats_attr_bios_journal_completed_read.attr, - &pool_stats_attr_bios_journal_completed_write.attr, - &pool_stats_attr_bios_journal_completed_empty_flush.attr, - &pool_stats_attr_bios_journal_completed_discard.attr, - &pool_stats_attr_bios_journal_completed_flush.attr, - &pool_stats_attr_bios_journal_completed_fua.attr, - &pool_stats_attr_bios_page_cache_completed_read.attr, - &pool_stats_attr_bios_page_cache_completed_write.attr, - &pool_stats_attr_bios_page_cache_completed_empty_flush.attr, - &pool_stats_attr_bios_page_cache_completed_discard.attr, - &pool_stats_attr_bios_page_cache_completed_flush.attr, - &pool_stats_attr_bios_page_cache_completed_fua.attr, - &pool_stats_attr_bios_acknowledged_read.attr, - &pool_stats_attr_bios_acknowledged_write.attr, - &pool_stats_attr_bios_acknowledged_empty_flush.attr, - &pool_stats_attr_bios_acknowledged_discard.attr, - &pool_stats_attr_bios_acknowledged_flush.attr, - &pool_stats_attr_bios_acknowledged_fua.attr, - &pool_stats_attr_bios_acknowledged_partial_read.attr, - &pool_stats_attr_bios_acknowledged_partial_write.attr, - &pool_stats_attr_bios_acknowledged_partial_empty_flush.attr, - &pool_stats_attr_bios_acknowledged_partial_discard.attr, - &pool_stats_attr_bios_acknowledged_partial_flush.attr, - &pool_stats_attr_bios_acknowledged_partial_fua.attr, - &pool_stats_attr_bios_in_progress_read.attr, - &pool_stats_attr_bios_in_progress_write.attr, - &pool_stats_attr_bios_in_progress_empty_flush.attr, - &pool_stats_attr_bios_in_progress_discard.attr, - &pool_stats_attr_bios_in_progress_flush.attr, - &pool_stats_attr_bios_in_progress_fua.attr, - &pool_stats_attr_memory_usage_bytes_used.attr, - &pool_stats_attr_memory_usage_peak_bytes_used.attr, - &pool_stats_attr_index_entries_indexed.attr, - &pool_stats_attr_index_posts_found.attr, - &pool_stats_attr_index_posts_not_found.attr, - &pool_stats_attr_index_queries_found.attr, - &pool_stats_attr_index_queries_not_found.attr, - &pool_stats_attr_index_updates_found.attr, - &pool_stats_attr_index_updates_not_found.attr, - &pool_stats_attr_index_entries_discarded.attr, - NULL, -}; diff --git a/drivers/md/dm-vdo/pool-sysfs.c b/drivers/md/dm-vdo/pool-sysfs.c deleted file mode 100644 index 6769c5711cbc..000000000000 --- a/drivers/md/dm-vdo/pool-sysfs.c +++ /dev/null @@ -1,198 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -#include "pool-sysfs.h" - -#include - -#include "memory-alloc.h" -#include "string-utils.h" - -#include "data-vio.h" -#include "dedupe.h" -#include "vdo.h" - -struct pool_attribute { - struct attribute attr; - ssize_t (*show)(struct vdo *vdo, char *buf); - ssize_t (*store)(struct vdo *vdo, const char *value, size_t count); -}; - -static ssize_t vdo_pool_attr_show(struct kobject *directory, struct attribute *attr, - char *buf) -{ - struct pool_attribute *pool_attr = container_of(attr, struct pool_attribute, - attr); - struct vdo *vdo = container_of(directory, struct vdo, vdo_directory); - - if (pool_attr->show == NULL) - return -EINVAL; - return pool_attr->show(vdo, buf); -} - -static ssize_t vdo_pool_attr_store(struct kobject *directory, struct attribute *attr, - const char *buf, size_t length) -{ - struct pool_attribute *pool_attr = container_of(attr, struct pool_attribute, - attr); - struct vdo *vdo = container_of(directory, struct vdo, vdo_directory); - - if (pool_attr->store == NULL) - return -EINVAL; - return pool_attr->store(vdo, buf, length); -} - -static const struct sysfs_ops vdo_pool_sysfs_ops = { - .show = vdo_pool_attr_show, - .store = vdo_pool_attr_store, -}; - -static ssize_t pool_compressing_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%s\n", (vdo_get_compressing(vdo) ? "1" : "0")); -} - -static ssize_t pool_discards_active_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", - get_data_vio_pool_active_discards(vdo->data_vio_pool)); -} - -static ssize_t pool_discards_limit_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", get_data_vio_pool_discard_limit(vdo->data_vio_pool)); -} - -static ssize_t pool_discards_limit_store(struct vdo *vdo, const char *buf, size_t length) -{ - unsigned int value; - int result; - - if ((length > 12) || (kstrtouint(buf, 10, &value) < 0) || (value < 1)) - return -EINVAL; - - result = set_data_vio_pool_discard_limit(vdo->data_vio_pool, value); - if (result != VDO_SUCCESS) - return -EINVAL; - - return length; -} - -static ssize_t pool_discards_maximum_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", - get_data_vio_pool_maximum_discards(vdo->data_vio_pool)); -} - -static ssize_t pool_instance_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", vdo->instance); -} - -static ssize_t pool_requests_active_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", - get_data_vio_pool_active_requests(vdo->data_vio_pool)); -} - -static ssize_t pool_requests_limit_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", get_data_vio_pool_request_limit(vdo->data_vio_pool)); -} - -static ssize_t pool_requests_maximum_show(struct vdo *vdo, char *buf) -{ - return sprintf(buf, "%u\n", - get_data_vio_pool_maximum_requests(vdo->data_vio_pool)); -} - -static void vdo_pool_release(struct kobject *directory) -{ - vdo_free(container_of(directory, struct vdo, vdo_directory)); -} - -static struct pool_attribute vdo_pool_compressing_attr = { - .attr = { - .name = "compressing", - .mode = 0444, - }, - .show = pool_compressing_show, -}; - -static struct pool_attribute vdo_pool_discards_active_attr = { - .attr = { - .name = "discards_active", - .mode = 0444, - }, - .show = pool_discards_active_show, -}; - -static struct pool_attribute vdo_pool_discards_limit_attr = { - .attr = { - .name = "discards_limit", - .mode = 0644, - }, - .show = pool_discards_limit_show, - .store = pool_discards_limit_store, -}; - -static struct pool_attribute vdo_pool_discards_maximum_attr = { - .attr = { - .name = "discards_maximum", - .mode = 0444, - }, - .show = pool_discards_maximum_show, -}; - -static struct pool_attribute vdo_pool_instance_attr = { - .attr = { - .name = "instance", - .mode = 0444, - }, - .show = pool_instance_show, -}; - -static struct pool_attribute vdo_pool_requests_active_attr = { - .attr = { - .name = "requests_active", - .mode = 0444, - }, - .show = pool_requests_active_show, -}; - -static struct pool_attribute vdo_pool_requests_limit_attr = { - .attr = { - .name = "requests_limit", - .mode = 0444, - }, - .show = pool_requests_limit_show, -}; - -static struct pool_attribute vdo_pool_requests_maximum_attr = { - .attr = { - .name = "requests_maximum", - .mode = 0444, - }, - .show = pool_requests_maximum_show, -}; - -static struct attribute *pool_attrs[] = { - &vdo_pool_compressing_attr.attr, - &vdo_pool_discards_active_attr.attr, - &vdo_pool_discards_limit_attr.attr, - &vdo_pool_discards_maximum_attr.attr, - &vdo_pool_instance_attr.attr, - &vdo_pool_requests_active_attr.attr, - &vdo_pool_requests_limit_attr.attr, - &vdo_pool_requests_maximum_attr.attr, - NULL, -}; -ATTRIBUTE_GROUPS(pool); - -const struct kobj_type vdo_directory_type = { - .release = vdo_pool_release, - .sysfs_ops = &vdo_pool_sysfs_ops, - .default_groups = pool_groups, -}; diff --git a/drivers/md/dm-vdo/pool-sysfs.h b/drivers/md/dm-vdo/pool-sysfs.h deleted file mode 100644 index 00e680924dc1..000000000000 --- a/drivers/md/dm-vdo/pool-sysfs.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2023 Red Hat - */ - -#ifndef VDO_POOL_SYSFS_H -#define VDO_POOL_SYSFS_H - -#include - -/* The kobj_type used for setting up the kernel layer kobject. */ -extern const struct kobj_type vdo_directory_type; - -/* The sysfs_ops used for the "statistics" subdirectory. */ -extern const struct sysfs_ops vdo_pool_stats_sysfs_ops; -/* The attribute used for the "statistics" subdirectory. */ -extern struct attribute *vdo_pool_stats_attrs[]; - -#endif /* VDO_POOL_SYSFS_H */ diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index 92c42b8bbb8b..42e87b2344bc 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -38,7 +38,6 @@ const struct error_info vdo_status_list[] = { { "VDO_BAD_NONCE", "Bad nonce" }, { "VDO_JOURNAL_OVERFLOW", "Journal sequence number overflow" }, { "VDO_INVALID_ADMIN_STATE", "Invalid operation for current state" }, - { "VDO_CANT_ADD_SYSFS_NODE", "Failed to add sysfs node" }, }; static atomic_t vdo_status_codes_registered = ATOMIC_INIT(0); diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h index eb847def8eb4..72da04159f88 100644 --- a/drivers/md/dm-vdo/status-codes.h +++ b/drivers/md/dm-vdo/status-codes.h @@ -72,8 +72,6 @@ enum vdo_status_codes { VDO_JOURNAL_OVERFLOW, /* the VDO is not in a state to perform an admin operation */ VDO_INVALID_ADMIN_STATE, - /* failure adding a sysfs node */ - VDO_CANT_ADD_SYSFS_NODE, /* one more than last error code */ VDO_STATUS_CODE_LAST, VDO_STATUS_CODE_BLOCK_END = VDO_ERRORS_BLOCK_END diff --git a/drivers/md/dm-vdo/sysfs.c b/drivers/md/dm-vdo/sysfs.c deleted file mode 100644 index 70feffe9d4c4..000000000000 --- a/drivers/md/dm-vdo/sysfs.c +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -#include - -#include "logger.h" - -#include "constants.h" -#include "dedupe.h" -#include "vdo.h" - -static int vdo_log_level_show(char *buf, const struct kernel_param *kp) -{ - return sprintf(buf, "%s\n", uds_log_priority_to_string(uds_get_log_level())); -} - -static int vdo_log_level_store(const char *buf, const struct kernel_param *kp) -{ - static char internal_buf[11]; - - int n = strlen(buf); - - if (n > 10) - return -EINVAL; - - memset(internal_buf, '\000', sizeof(internal_buf)); - memcpy(internal_buf, buf, n); - if (internal_buf[n - 1] == '\n') - internal_buf[n - 1] = '\000'; - uds_set_log_level(uds_log_string_to_priority(internal_buf)); - return 0; -} - - -static int vdo_dedupe_timeout_interval_store(const char *buf, - const struct kernel_param *kp) -{ - int result = param_set_uint(buf, kp); - - if (result != 0) - return result; - vdo_set_dedupe_index_timeout_interval(*(uint *)kp->arg); - return 0; -} - -static int vdo_min_dedupe_timer_interval_store(const char *buf, - const struct kernel_param *kp) -{ - int result = param_set_uint(buf, kp); - - if (result != 0) - return result; - vdo_set_dedupe_index_min_timer_interval(*(uint *)kp->arg); - return 0; -} - -static const struct kernel_param_ops log_level_ops = { - .set = vdo_log_level_store, - .get = vdo_log_level_show, -}; - - -static const struct kernel_param_ops dedupe_timeout_ops = { - .set = vdo_dedupe_timeout_interval_store, - .get = param_get_uint, -}; - -static const struct kernel_param_ops dedupe_timer_ops = { - .set = vdo_min_dedupe_timer_interval_store, - .get = param_get_uint, -}; - -module_param_cb(log_level, &log_level_ops, NULL, 0644); - - -module_param_cb(deduplication_timeout_interval, &dedupe_timeout_ops, - &vdo_dedupe_index_timeout_interval, 0644); - -module_param_cb(min_deduplication_timer_interval, &dedupe_timer_ops, - &vdo_dedupe_index_min_timer_interval, 0644); diff --git a/drivers/md/dm-vdo/uds-sysfs.c b/drivers/md/dm-vdo/uds-sysfs.c deleted file mode 100644 index 753d81d6f207..000000000000 --- a/drivers/md/dm-vdo/uds-sysfs.c +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Red Hat - */ - -#include "uds-sysfs.h" - -#include -#include -#include - -#include "logger.h" -#include "memory-alloc.h" -#include "string-utils.h" - -#include "indexer.h" - -#define UDS_SYSFS_NAME "uds" - -static struct { - /* /sys/uds */ - struct kobject kobj; - /* /sys/uds/parameter */ - struct kobject parameter_kobj; - - /* These flags are used to ensure a clean shutdown */ - - /* /sys/uds flag */ - bool flag; - /* /sys/uds/parameter flag */ - bool parameter_flag; -} object_root; - -static char *buffer_to_string(const char *buf, size_t length) -{ - char *string; - - if (vdo_allocate(length + 1, char, __func__, &string) != VDO_SUCCESS) - return NULL; - - memcpy(string, buf, length); - string[length] = '\0'; - if (string[length - 1] == '\n') - string[length - 1] = '\0'; - - return string; -} - -/* - * This is the code for any directory in the /sys/ tree that contains no regular files - * (only subdirectories). - */ - -static void empty_release(struct kobject *kobj) -{ -} - -static ssize_t empty_show(struct kobject *kobj, struct attribute *attr, char *buf) -{ - return 0; -} - -static ssize_t empty_store(struct kobject *kobj, struct attribute *attr, const char *buf, - size_t length) -{ - return length; -} - -static const struct sysfs_ops empty_ops = { - .show = empty_show, - .store = empty_store, -}; - -static struct attribute *empty_attrs[] = { - NULL, -}; -ATTRIBUTE_GROUPS(empty); - -static const struct kobj_type empty_object_type = { - .release = empty_release, - .sysfs_ops = &empty_ops, - .default_groups = empty_groups, -}; - -/* - * This is the code for the /sys//parameter directory. - * /log_level UDS_LOG_LEVEL - */ - -struct parameter_attribute { - struct attribute attr; - const char *(*show_string)(void); - void (*store_string)(const char *string); -}; - -static ssize_t parameter_show(struct kobject *kobj, struct attribute *attr, char *buf) -{ - struct parameter_attribute *pa; - - pa = container_of(attr, struct parameter_attribute, attr); - if (pa->show_string != NULL) - return sprintf(buf, "%s\n", pa->show_string()); - else - return -EINVAL; -} - -static ssize_t parameter_store(struct kobject *kobj, struct attribute *attr, - const char *buf, size_t length) -{ - char *string; - struct parameter_attribute *pa; - - pa = container_of(attr, struct parameter_attribute, attr); - if (pa->store_string == NULL) - return -EINVAL; - string = buffer_to_string(buf, length); - if (string == NULL) - return -ENOMEM; - - pa->store_string(string); - vdo_free(string); - return length; -} - -static const char *parameter_show_log_level(void) -{ - return uds_log_priority_to_string(uds_get_log_level()); -} - -static void parameter_store_log_level(const char *string) -{ - uds_set_log_level(uds_log_string_to_priority(string)); -} - -static struct parameter_attribute log_level_attr = { - .attr = { .name = "log_level", .mode = 0600 }, - .show_string = parameter_show_log_level, - .store_string = parameter_store_log_level, -}; - -static struct attribute *parameter_attrs[] = { - &log_level_attr.attr, - NULL, -}; -ATTRIBUTE_GROUPS(parameter); - -static const struct sysfs_ops parameter_ops = { - .show = parameter_show, - .store = parameter_store, -}; - -static const struct kobj_type parameter_object_type = { - .release = empty_release, - .sysfs_ops = ¶meter_ops, - .default_groups = parameter_groups, -}; - -int uds_init_sysfs(void) -{ - int result; - - memset(&object_root, 0, sizeof(object_root)); - kobject_init(&object_root.kobj, &empty_object_type); - result = kobject_add(&object_root.kobj, NULL, UDS_SYSFS_NAME); - if (result == 0) { - object_root.flag = true; - kobject_init(&object_root.parameter_kobj, ¶meter_object_type); - result = kobject_add(&object_root.parameter_kobj, &object_root.kobj, - "parameter"); - if (result == 0) - object_root.parameter_flag = true; - } - - if (result != 0) - uds_put_sysfs(); - - return result; -} - -void uds_put_sysfs(void) -{ - if (object_root.parameter_flag) - kobject_put(&object_root.parameter_kobj); - - if (object_root.flag) - kobject_put(&object_root.kobj); -} diff --git a/drivers/md/dm-vdo/uds-sysfs.h b/drivers/md/dm-vdo/uds-sysfs.h deleted file mode 100644 index c3d00a7187bd..000000000000 --- a/drivers/md/dm-vdo/uds-sysfs.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2023 Red Hat - */ - -#ifndef UDS_SYSFS_H -#define UDS_SYSFS_H - -int uds_init_sysfs(void); -void uds_put_sysfs(void); - -#endif /* UDS_SYSFS_H */ diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index 11be2ab17e29..28e6352c758e 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -53,7 +53,6 @@ #include "logical-zone.h" #include "packer.h" #include "physical-zone.h" -#include "pool-sysfs.h" #include "recovery-journal.h" #include "slab-depot.h" #include "statistics.h" @@ -691,13 +690,6 @@ void vdo_destroy(struct vdo *vdo) vdo->allocations_allowed = true; - /* Stop services that need to gather VDO statistics from the worker threads. */ - if (vdo->sysfs_added) { - init_completion(&vdo->stats_shutdown); - kobject_put(&vdo->stats_directory); - wait_for_completion(&vdo->stats_shutdown); - } - finish_vdo(vdo); unregister_vdo(vdo); free_data_vio_pool(vdo->data_vio_pool); @@ -732,15 +724,7 @@ void vdo_destroy(struct vdo *vdo) vdo_free(vdo_forget(vdo->compression_context)); } - - /* - * The call to kobject_put on the kobj sysfs node will decrement its reference count; when - * the count goes to zero the VDO object will be freed as a side effect. - */ - if (!vdo->sysfs_added) - vdo_free(vdo); - else - kobject_put(&vdo->vdo_directory); + vdo_free(vdo); } static int initialize_super_block(struct vdo *vdo, struct vdo_super_block *super_block) @@ -817,42 +801,6 @@ void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent) REQ_OP_READ); } -/** - * pool_stats_release() - Signal that sysfs stats have been shut down. - * @directory: The vdo stats directory. - */ -static void pool_stats_release(struct kobject *directory) -{ - struct vdo *vdo = container_of(directory, struct vdo, stats_directory); - - complete(&vdo->stats_shutdown); -} - -ATTRIBUTE_GROUPS(vdo_pool_stats); -static const struct kobj_type stats_directory_type = { - .release = pool_stats_release, - .sysfs_ops = &vdo_pool_stats_sysfs_ops, - .default_groups = vdo_pool_stats_groups, -}; - -/** - * vdo_add_sysfs_stats_dir() - Add the stats directory to the vdo sysfs directory. - * @vdo: The vdo. - * - * Return: VDO_SUCCESS or an error. - */ -int vdo_add_sysfs_stats_dir(struct vdo *vdo) -{ - int result; - - kobject_init(&vdo->stats_directory, &stats_directory_type); - result = kobject_add(&vdo->stats_directory, &vdo->vdo_directory, "statistics"); - if (result != 0) - return VDO_CANT_ADD_SYSFS_NODE; - - return VDO_SUCCESS; -} - /** * vdo_get_backing_device() - Get the block device object underlying a vdo. * @vdo: The vdo. diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h index 01558fc67f24..483ae873e002 100644 --- a/drivers/md/dm-vdo/vdo.h +++ b/drivers/md/dm-vdo/vdo.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -248,11 +247,6 @@ struct vdo { struct vdo_statistics stats_buffer; /* Protects the stats_buffer */ struct mutex stats_mutex; - /* true if sysfs directory is set up */ - bool sysfs_added; - /* Used when shutting down the sysfs statistics */ - struct completion stats_shutdown; - /* A list of all device_configs referencing this vdo */ struct list_head device_config_list; @@ -264,15 +258,10 @@ struct vdo { u64 starting_sector_offset; struct volume_geometry geometry; - /* For sysfs */ - struct kobject vdo_directory; - struct kobject stats_directory; - /* N blobs of context data for LZ4 code, one per CPU thread. */ char **compression_context; }; - /** * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue * for acknowledging received and processed bios. @@ -315,8 +304,6 @@ void vdo_destroy(struct vdo *vdo); void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent); -int __must_check vdo_add_sysfs_stats_dir(struct vdo *vdo); - struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo); const char * __must_check vdo_get_device_name(const struct dm_target *target); From 25315e967aab0a3b732c076e4cbce52bd0fa4385 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Sat, 10 Feb 2024 10:42:00 -0600 Subject: [PATCH 0954/1038] dm vdo: add 'log_level' module parameter Expose control over dm-vdo's log-level in terms of a module param. It can be read and written via /sys/module/dm_vdo/parameters/log_level. Signed-off-by: Mike Snitzer Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dm-vdo-target.c | 3 +++ drivers/md/dm-vdo/logger.c | 10 ++++++++-- drivers/md/dm-vdo/logger.h | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index c6ee8161ba62..288e9b79bf16 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -2905,6 +2905,9 @@ static void __exit vdo_exit(void) module_init(vdo_init); module_exit(vdo_exit); +module_param_named(log_level, vdo_log_level, uint, 0644); +MODULE_PARM_DESC(log_level, "Log level for log messages"); + MODULE_DESCRIPTION(DM_NAME " target for transparent deduplication"); MODULE_AUTHOR("Red Hat, Inc."); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index bac28fff622d..5d27cd6483a9 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -49,11 +49,17 @@ static const char *const PRIORITY_STRINGS[] = { "DEBUG", }; -static int log_level = UDS_LOG_INFO; +int vdo_log_level = UDS_LOG_DEFAULT; int uds_get_log_level(void) { - return log_level; + int log_level_latch = READ_ONCE(vdo_log_level); + + if (unlikely(log_level_latch > UDS_LOG_MAX)) { + log_level_latch = UDS_LOG_DEFAULT; + WRITE_ONCE(vdo_log_level, log_level_latch); + } + return log_level_latch; } int uds_log_string_to_priority(const char *string) diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h index ceb07aa3231f..095532afc9a3 100644 --- a/drivers/md/dm-vdo/logger.h +++ b/drivers/md/dm-vdo/logger.h @@ -6,20 +6,28 @@ #ifndef UDS_LOGGER_H #define UDS_LOGGER_H +#include #include #include #include /* Custom logging utilities for UDS */ -#define UDS_LOG_EMERG 0 -#define UDS_LOG_ALERT 1 -#define UDS_LOG_CRIT 2 -#define UDS_LOG_ERR 3 -#define UDS_LOG_WARNING 4 -#define UDS_LOG_NOTICE 5 -#define UDS_LOG_INFO 6 -#define UDS_LOG_DEBUG 7 +enum { + UDS_LOG_EMERG = LOGLEVEL_EMERG, + UDS_LOG_ALERT = LOGLEVEL_ALERT, + UDS_LOG_CRIT = LOGLEVEL_CRIT, + UDS_LOG_ERR = LOGLEVEL_ERR, + UDS_LOG_WARNING = LOGLEVEL_WARNING, + UDS_LOG_NOTICE = LOGLEVEL_NOTICE, + UDS_LOG_INFO = LOGLEVEL_INFO, + UDS_LOG_DEBUG = LOGLEVEL_DEBUG, + + UDS_LOG_MAX = UDS_LOG_DEBUG, + UDS_LOG_DEFAULT = UDS_LOG_INFO, +}; + +extern int vdo_log_level; #define DM_MSG_PREFIX "vdo" #define UDS_LOGGING_MODULE_NAME DM_NAME ": " DM_MSG_PREFIX From fd5b92b4ce16476d6d460df78c676ad74a1c117a Mon Sep 17 00:00:00 2001 From: Ken Raeburn Date: Tue, 20 Feb 2024 22:28:38 -0500 Subject: [PATCH 0955/1038] dm vdo: document log_level parameter Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- Documentation/admin-guide/device-mapper/vdo.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/admin-guide/device-mapper/vdo.rst b/Documentation/admin-guide/device-mapper/vdo.rst index a199009979d9..a3572582ead8 100644 --- a/Documentation/admin-guide/device-mapper/vdo.rst +++ b/Documentation/admin-guide/device-mapper/vdo.rst @@ -315,6 +315,13 @@ GB of RAM per 1 TB of window. For sparse indexes, the index requires 1 GB of RAM per 10 TB of window. The index configuration is set when the target is formatted and may not be modified. +Module Parameters +================= + +The vdo driver has a numeric parameter 'log_level' which controls the +verbosity of logging from the driver. The default setting is 6 +(LOGLEVEL_INFO and more severe messages). + Run-time Usage ============== From 7979d907574a7013a1a35f47e798b1b671c334f1 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 14 Feb 2024 08:53:29 -0600 Subject: [PATCH 0956/1038] dm vdo logger: remove log level to string conversion code Was only used by sysfs code, can be reinstated if/when needed. Signed-off-by: Mike Snitzer Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/logger.c | 53 -------------------------------------- drivers/md/dm-vdo/logger.h | 4 --- 2 files changed, 57 deletions(-) diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 5d27cd6483a9..80f1e4c62ac6 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -16,39 +16,6 @@ #include "thread-device.h" #include "thread-utils.h" -struct priority_name { - const char *name; - const int priority; -}; - -static const struct priority_name PRIORITIES[] = { - { "ALERT", UDS_LOG_ALERT }, - { "CRITICAL", UDS_LOG_CRIT }, - { "CRIT", UDS_LOG_CRIT }, - { "DEBUG", UDS_LOG_DEBUG }, - { "EMERGENCY", UDS_LOG_EMERG }, - { "EMERG", UDS_LOG_EMERG }, - { "ERROR", UDS_LOG_ERR }, - { "ERR", UDS_LOG_ERR }, - { "INFO", UDS_LOG_INFO }, - { "NOTICE", UDS_LOG_NOTICE }, - { "PANIC", UDS_LOG_EMERG }, - { "WARN", UDS_LOG_WARNING }, - { "WARNING", UDS_LOG_WARNING }, - { NULL, -1 }, -}; - -static const char *const PRIORITY_STRINGS[] = { - "EMERGENCY", - "ALERT", - "CRITICAL", - "ERROR", - "WARN", - "NOTICE", - "INFO", - "DEBUG", -}; - int vdo_log_level = UDS_LOG_DEFAULT; int uds_get_log_level(void) @@ -62,26 +29,6 @@ int uds_get_log_level(void) return log_level_latch; } -int uds_log_string_to_priority(const char *string) -{ - int i; - - for (i = 0; PRIORITIES[i].name != NULL; i++) { - if (strcasecmp(string, PRIORITIES[i].name) == 0) - return PRIORITIES[i].priority; - } - - return UDS_LOG_INFO; -} - -const char *uds_log_priority_to_string(int priority) -{ - if ((priority < 0) || (priority >= (int) ARRAY_SIZE(PRIORITY_STRINGS))) - return "unknown"; - - return PRIORITY_STRINGS[priority]; -} - static const char *get_current_interrupt_type(void) { if (in_nmi()) diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h index 095532afc9a3..a8cdf46b6fc1 100644 --- a/drivers/md/dm-vdo/logger.h +++ b/drivers/md/dm-vdo/logger.h @@ -45,10 +45,6 @@ extern int vdo_log_level; int uds_get_log_level(void); -int uds_log_string_to_priority(const char *string); - -const char *uds_log_priority_to_string(int priority); - void uds_log_embedded_message(int priority, const char *module, const char *prefix, const char *fmt1, va_list args1, const char *fmt2, ...) __printf(4, 0) __printf(6, 7); From 41c58a36e2c0496abebb1cb3100b38e32f49f80f Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 29 Feb 2024 16:54:43 -0500 Subject: [PATCH 0957/1038] dm vdo indexer: fix use after free Fixes: b46d79bdb82a ("dm vdo: add deduplication index storage interface") Reported-by: Harshit Mogalapalli Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/indexer/index-layout.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index 349b3022f1e1..74fd44c20e5c 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -1561,10 +1561,11 @@ static int __must_check load_index_save(struct index_save_layout *isl, if (table->header.type != RH_TYPE_SAVE) { + uds_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save %u header type %u", + instance, table->header.type); vdo_free(table); - return uds_log_error_strerror(UDS_CORRUPT_DATA, - "unexpected index save %u header type %u", - instance, table->header.type); + return UDS_CORRUPT_DATA; } result = read_index_save_data(reader, isl, table->header.payload); From 66214ed0001eaea245b5d9f01b128aa82d6963a8 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 13 Feb 2024 13:35:28 -0600 Subject: [PATCH 0958/1038] dm vdo funnel-queue: change from uds_ to vdo_ namespace Also return VDO_SUCCESS from vdo_make_funnel_queue. Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/data-vio.c | 12 ++++----- drivers/md/dm-vdo/dedupe.c | 10 +++---- drivers/md/dm-vdo/funnel-queue.c | 18 ++++++------- drivers/md/dm-vdo/funnel-queue.h | 26 +++++++++---------- drivers/md/dm-vdo/funnel-workqueue.c | 10 +++---- .../md/dm-vdo/indexer/funnel-requestqueue.c | 22 ++++++++-------- 6 files changed, 49 insertions(+), 49 deletions(-) diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 51c49fad1b8b..2b0d42c77e05 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -718,7 +718,7 @@ static void process_release_callback(struct vdo_completion *completion) for (processed = 0; processed < DATA_VIO_RELEASE_BATCH_SIZE; processed++) { struct data_vio *data_vio; - struct funnel_queue_entry *entry = uds_funnel_queue_poll(pool->queue); + struct funnel_queue_entry *entry = vdo_funnel_queue_poll(pool->queue); if (entry == NULL) break; @@ -748,7 +748,7 @@ static void process_release_callback(struct vdo_completion *completion) /* Pairs with the barrier in schedule_releases(). */ smp_mb(); - reschedule = !uds_is_funnel_queue_empty(pool->queue); + reschedule = !vdo_is_funnel_queue_empty(pool->queue); drained = (!reschedule && vdo_is_state_draining(&pool->state) && check_for_drain_complete_locked(pool)); @@ -865,8 +865,8 @@ int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, process_release_callback, vdo->thread_config.cpu_thread, NULL); - result = uds_make_funnel_queue(&pool->queue); - if (result != UDS_SUCCESS) { + result = vdo_make_funnel_queue(&pool->queue); + if (result != VDO_SUCCESS) { free_data_vio_pool(vdo_forget(pool)); return result; } @@ -924,7 +924,7 @@ void free_data_vio_pool(struct data_vio_pool *pool) destroy_data_vio(data_vio); } - uds_free_funnel_queue(vdo_forget(pool->queue)); + vdo_free_funnel_queue(vdo_forget(pool->queue)); vdo_free(pool); } @@ -1283,7 +1283,7 @@ static void finish_cleanup(struct data_vio *data_vio) (completion->result != VDO_SUCCESS)) { struct data_vio_pool *pool = completion->vdo->data_vio_pool; - uds_funnel_queue_put(pool->queue, &completion->work_queue_entry_link); + vdo_funnel_queue_put(pool->queue, &completion->work_queue_entry_link); schedule_releases(pool); return; } diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 8550a9a7958b..c031ab01054d 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -2246,7 +2246,7 @@ static void finish_index_operation(struct uds_request *request) atomic_read(&context->state)); } - uds_funnel_queue_put(context->zone->timed_out_complete, &context->queue_entry); + vdo_funnel_queue_put(context->zone->timed_out_complete, &context->queue_entry); } /** @@ -2275,7 +2275,7 @@ static void check_for_drain_complete(struct hash_zone *zone) struct dedupe_context *context; struct funnel_queue_entry *entry; - entry = uds_funnel_queue_poll(zone->timed_out_complete); + entry = vdo_funnel_queue_poll(zone->timed_out_complete); if (entry == NULL) break; @@ -2373,7 +2373,7 @@ static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zone INIT_LIST_HEAD(&zone->available); INIT_LIST_HEAD(&zone->pending); - result = uds_make_funnel_queue(&zone->timed_out_complete); + result = vdo_make_funnel_queue(&zone->timed_out_complete); if (result != VDO_SUCCESS) return result; @@ -2475,7 +2475,7 @@ void vdo_free_hash_zones(struct hash_zones *zones) for (i = 0; i < zones->zone_count; i++) { struct hash_zone *zone = &zones->zones[i]; - uds_free_funnel_queue(vdo_forget(zone->timed_out_complete)); + vdo_free_funnel_queue(vdo_forget(zone->timed_out_complete)); vdo_int_map_free(vdo_forget(zone->hash_lock_map)); vdo_free(vdo_forget(zone->lock_array)); } @@ -2875,7 +2875,7 @@ static struct dedupe_context * __must_check acquire_context(struct hash_zone *zo return context; } - entry = uds_funnel_queue_poll(zone->timed_out_complete); + entry = vdo_funnel_queue_poll(zone->timed_out_complete); return ((entry == NULL) ? NULL : container_of(entry, struct dedupe_context, queue_entry)); } diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c index ce0e801fd955..a63b2f2bfd7d 100644 --- a/drivers/md/dm-vdo/funnel-queue.c +++ b/drivers/md/dm-vdo/funnel-queue.c @@ -9,7 +9,7 @@ #include "memory-alloc.h" #include "permassert.h" -int uds_make_funnel_queue(struct funnel_queue **queue_ptr) +int vdo_make_funnel_queue(struct funnel_queue **queue_ptr) { int result; struct funnel_queue *queue; @@ -27,10 +27,10 @@ int uds_make_funnel_queue(struct funnel_queue **queue_ptr) queue->oldest = &queue->stub; *queue_ptr = queue; - return UDS_SUCCESS; + return VDO_SUCCESS; } -void uds_free_funnel_queue(struct funnel_queue *queue) +void vdo_free_funnel_queue(struct funnel_queue *queue) { vdo_free(queue); } @@ -40,7 +40,7 @@ static struct funnel_queue_entry *get_oldest(struct funnel_queue *queue) /* * Barrier requirements: We need a read barrier between reading a "next" field pointer * value and reading anything it points to. There's an accompanying barrier in - * uds_funnel_queue_put() between its caller setting up the entry and making it visible. + * vdo_funnel_queue_put() between its caller setting up the entry and making it visible. */ struct funnel_queue_entry *oldest = queue->oldest; struct funnel_queue_entry *next = READ_ONCE(oldest->next); @@ -80,7 +80,7 @@ static struct funnel_queue_entry *get_oldest(struct funnel_queue *queue) * Put the stub entry back on the queue, ensuring a successor will eventually be * seen. */ - uds_funnel_queue_put(queue, &queue->stub); + vdo_funnel_queue_put(queue, &queue->stub); /* Check again for a successor. */ next = READ_ONCE(oldest->next); @@ -100,7 +100,7 @@ static struct funnel_queue_entry *get_oldest(struct funnel_queue *queue) * Poll a queue, removing the oldest entry if the queue is not empty. This function must only be * called from a single consumer thread. */ -struct funnel_queue_entry *uds_funnel_queue_poll(struct funnel_queue *queue) +struct funnel_queue_entry *vdo_funnel_queue_poll(struct funnel_queue *queue) { struct funnel_queue_entry *oldest = get_oldest(queue); @@ -134,7 +134,7 @@ struct funnel_queue_entry *uds_funnel_queue_poll(struct funnel_queue *queue) * or more entries being added such that the list view is incomplete, this function will report the * queue as empty. */ -bool uds_is_funnel_queue_empty(struct funnel_queue *queue) +bool vdo_is_funnel_queue_empty(struct funnel_queue *queue) { return get_oldest(queue) == NULL; } @@ -143,9 +143,9 @@ bool uds_is_funnel_queue_empty(struct funnel_queue *queue) * Check whether the funnel queue is idle or not. If the queue has entries available to be * retrieved, it is not idle. If the queue is in a transition state with one or more entries being * added such that the list view is incomplete, it may not be possible to retrieve an entry with - * the uds_funnel_queue_poll() function, but the queue will not be considered idle. + * the vdo_funnel_queue_poll() function, but the queue will not be considered idle. */ -bool uds_is_funnel_queue_idle(struct funnel_queue *queue) +bool vdo_is_funnel_queue_idle(struct funnel_queue *queue) { /* * Oldest is not the stub, so there's another entry, though if next is NULL we can't diff --git a/drivers/md/dm-vdo/funnel-queue.h b/drivers/md/dm-vdo/funnel-queue.h index 88a30c593fdc..bde0f1deff98 100644 --- a/drivers/md/dm-vdo/funnel-queue.h +++ b/drivers/md/dm-vdo/funnel-queue.h @@ -3,8 +3,8 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_FUNNEL_QUEUE_H -#define UDS_FUNNEL_QUEUE_H +#ifndef VDO_FUNNEL_QUEUE_H +#define VDO_FUNNEL_QUEUE_H #include #include @@ -25,19 +25,19 @@ * the queue entries, and pointers to those structures are used exclusively by the queue. No macros * are defined to template the queue, so the offset of the funnel_queue_entry in the records placed * in the queue must all be the same so the client can derive their structure pointer from the - * entry pointer returned by uds_funnel_queue_poll(). + * entry pointer returned by vdo_funnel_queue_poll(). * * Callers are wholly responsible for allocating and freeing the entries. Entries may be freed as * soon as they are returned since this queue is not susceptible to the "ABA problem" present in * many lock-free data structures. The queue is dynamically allocated to ensure cache-line * alignment, but no other dynamic allocation is used. * - * The algorithm is not actually 100% lock-free. There is a single point in uds_funnel_queue_put() + * The algorithm is not actually 100% lock-free. There is a single point in vdo_funnel_queue_put() * at which a preempted producer will prevent the consumers from seeing items added to the queue by * later producers, and only if the queue is short enough or the consumer fast enough for it to * reach what was the end of the queue at the time of the preemption. * - * The consumer function, uds_funnel_queue_poll(), will return NULL when the queue is empty. To + * The consumer function, vdo_funnel_queue_poll(), will return NULL when the queue is empty. To * wait for data to consume, spin (if safe) or combine the queue with a struct event_count to * signal the presence of new entries. */ @@ -51,7 +51,7 @@ struct funnel_queue_entry { /* * The dynamically allocated queue structure, which is allocated on a cache line boundary so the * producer and consumer fields in the structure will land on separate cache lines. This should be - * consider opaque but it is exposed here so uds_funnel_queue_put() can be inlined. + * consider opaque but it is exposed here so vdo_funnel_queue_put() can be inlined. */ struct __aligned(L1_CACHE_BYTES) funnel_queue { /* @@ -67,9 +67,9 @@ struct __aligned(L1_CACHE_BYTES) funnel_queue { struct funnel_queue_entry stub; }; -int __must_check uds_make_funnel_queue(struct funnel_queue **queue_ptr); +int __must_check vdo_make_funnel_queue(struct funnel_queue **queue_ptr); -void uds_free_funnel_queue(struct funnel_queue *queue); +void vdo_free_funnel_queue(struct funnel_queue *queue); /* * Put an entry on the end of the queue. @@ -79,7 +79,7 @@ void uds_free_funnel_queue(struct funnel_queue *queue); * from the pointer that passed in here, so every entry in the queue must have the struct * funnel_queue_entry at the same offset within the client's structure. */ -static inline void uds_funnel_queue_put(struct funnel_queue *queue, +static inline void vdo_funnel_queue_put(struct funnel_queue *queue, struct funnel_queue_entry *entry) { struct funnel_queue_entry *previous; @@ -101,10 +101,10 @@ static inline void uds_funnel_queue_put(struct funnel_queue *queue, WRITE_ONCE(previous->next, entry); } -struct funnel_queue_entry *__must_check uds_funnel_queue_poll(struct funnel_queue *queue); +struct funnel_queue_entry *__must_check vdo_funnel_queue_poll(struct funnel_queue *queue); -bool __must_check uds_is_funnel_queue_empty(struct funnel_queue *queue); +bool __must_check vdo_is_funnel_queue_empty(struct funnel_queue *queue); -bool __must_check uds_is_funnel_queue_idle(struct funnel_queue *queue); +bool __must_check vdo_is_funnel_queue_idle(struct funnel_queue *queue); -#endif /* UDS_FUNNEL_QUEUE_H */ +#endif /* VDO_FUNNEL_QUEUE_H */ diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c index 03296e7fec12..cf04cdef0750 100644 --- a/drivers/md/dm-vdo/funnel-workqueue.c +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -98,7 +98,7 @@ static struct vdo_completion *poll_for_completion(struct simple_work_queue *queu int i; for (i = queue->common.type->max_priority; i >= 0; i--) { - struct funnel_queue_entry *link = uds_funnel_queue_poll(queue->priority_lists[i]); + struct funnel_queue_entry *link = vdo_funnel_queue_poll(queue->priority_lists[i]); if (link != NULL) return container_of(link, struct vdo_completion, work_queue_entry_link); @@ -123,7 +123,7 @@ static void enqueue_work_queue_completion(struct simple_work_queue *queue, completion->my_queue = &queue->common; /* Funnel queue handles the synchronization for the put. */ - uds_funnel_queue_put(queue->priority_lists[completion->priority], + vdo_funnel_queue_put(queue->priority_lists[completion->priority], &completion->work_queue_entry_link); /* @@ -275,7 +275,7 @@ static void free_simple_work_queue(struct simple_work_queue *queue) unsigned int i; for (i = 0; i <= VDO_WORK_Q_MAX_PRIORITY; i++) - uds_free_funnel_queue(queue->priority_lists[i]); + vdo_free_funnel_queue(queue->priority_lists[i]); vdo_free(queue->common.name); vdo_free(queue); } @@ -340,8 +340,8 @@ static int make_simple_work_queue(const char *thread_name_prefix, const char *na } for (i = 0; i <= type->max_priority; i++) { - result = uds_make_funnel_queue(&queue->priority_lists[i]); - if (result != UDS_SUCCESS) { + result = vdo_make_funnel_queue(&queue->priority_lists[i]); + if (result != VDO_SUCCESS) { free_simple_work_queue(queue); return result; } diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c index 84c7c1ae1333..1a5735375ddc 100644 --- a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c +++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c @@ -69,11 +69,11 @@ static inline struct uds_request *poll_queues(struct uds_request_queue *queue) { struct funnel_queue_entry *entry; - entry = uds_funnel_queue_poll(queue->retry_queue); + entry = vdo_funnel_queue_poll(queue->retry_queue); if (entry != NULL) return container_of(entry, struct uds_request, queue_link); - entry = uds_funnel_queue_poll(queue->main_queue); + entry = vdo_funnel_queue_poll(queue->main_queue); if (entry != NULL) return container_of(entry, struct uds_request, queue_link); @@ -82,8 +82,8 @@ static inline struct uds_request *poll_queues(struct uds_request_queue *queue) static inline bool are_queues_idle(struct uds_request_queue *queue) { - return uds_is_funnel_queue_idle(queue->retry_queue) && - uds_is_funnel_queue_idle(queue->main_queue); + return vdo_is_funnel_queue_idle(queue->retry_queue) && + vdo_is_funnel_queue_idle(queue->main_queue); } /* @@ -207,14 +207,14 @@ int uds_make_request_queue(const char *queue_name, atomic_set(&queue->dormant, false); init_waitqueue_head(&queue->wait_head); - result = uds_make_funnel_queue(&queue->main_queue); - if (result != UDS_SUCCESS) { + result = vdo_make_funnel_queue(&queue->main_queue); + if (result != VDO_SUCCESS) { uds_request_queue_finish(queue); return result; } - result = uds_make_funnel_queue(&queue->retry_queue); - if (result != UDS_SUCCESS) { + result = vdo_make_funnel_queue(&queue->retry_queue); + if (result != VDO_SUCCESS) { uds_request_queue_finish(queue); return result; } @@ -244,7 +244,7 @@ void uds_request_queue_enqueue(struct uds_request_queue *queue, bool unbatched = request->unbatched; sub_queue = request->requeued ? queue->retry_queue : queue->main_queue; - uds_funnel_queue_put(sub_queue, &request->queue_link); + vdo_funnel_queue_put(sub_queue, &request->queue_link); /* * We must wake the worker thread when it is dormant. A read fence isn't needed here since @@ -273,7 +273,7 @@ void uds_request_queue_finish(struct uds_request_queue *queue) vdo_join_threads(queue->thread); } - uds_free_funnel_queue(queue->main_queue); - uds_free_funnel_queue(queue->retry_queue); + vdo_free_funnel_queue(queue->main_queue); + vdo_free_funnel_queue(queue->retry_queue); vdo_free(queue); } From 3584240b9ce4adf63c3a985a730eb3f75806c26d Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 14 Feb 2024 09:22:04 -0600 Subject: [PATCH 0959/1038] dm vdo logger: change from uds_ to vdo_ namespace Rename all uds_log_* to vdo_log_*. Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/admin-state.c | 8 +- drivers/md/dm-vdo/block-map.c | 20 +-- drivers/md/dm-vdo/data-vio.c | 18 +-- drivers/md/dm-vdo/dedupe.c | 36 +++--- drivers/md/dm-vdo/dm-vdo-target.c | 144 ++++++++++----------- drivers/md/dm-vdo/dump.c | 14 +- drivers/md/dm-vdo/encodings.c | 26 ++-- drivers/md/dm-vdo/errors.c | 6 +- drivers/md/dm-vdo/errors.h | 4 +- drivers/md/dm-vdo/flush.c | 8 +- drivers/md/dm-vdo/funnel-workqueue.c | 2 +- drivers/md/dm-vdo/indexer/chapter-index.c | 4 +- drivers/md/dm-vdo/indexer/config.c | 48 +++---- drivers/md/dm-vdo/indexer/delta-index.c | 50 +++---- drivers/md/dm-vdo/indexer/index-layout.c | 82 ++++++------ drivers/md/dm-vdo/indexer/index-page-map.c | 2 +- drivers/md/dm-vdo/indexer/index-session.c | 44 +++---- drivers/md/dm-vdo/indexer/index.c | 52 ++++---- drivers/md/dm-vdo/indexer/io-factory.c | 2 +- drivers/md/dm-vdo/indexer/open-chapter.c | 6 +- drivers/md/dm-vdo/indexer/volume-index.c | 46 +++---- drivers/md/dm-vdo/indexer/volume.c | 64 ++++----- drivers/md/dm-vdo/int-map.c | 2 +- drivers/md/dm-vdo/io-submitter.c | 4 +- drivers/md/dm-vdo/logger.c | 52 ++++---- drivers/md/dm-vdo/logger.h | 82 ++++++------ drivers/md/dm-vdo/logical-zone.c | 4 +- drivers/md/dm-vdo/memory-alloc.c | 14 +- drivers/md/dm-vdo/message-stats.c | 2 +- drivers/md/dm-vdo/packer.c | 6 +- drivers/md/dm-vdo/permassert.c | 4 +- drivers/md/dm-vdo/physical-zone.c | 6 +- drivers/md/dm-vdo/recovery-journal.c | 16 +-- drivers/md/dm-vdo/repair.c | 46 +++---- drivers/md/dm-vdo/slab-depot.c | 54 ++++---- drivers/md/dm-vdo/status-codes.c | 6 +- drivers/md/dm-vdo/thread-utils.c | 2 +- drivers/md/dm-vdo/vdo.c | 14 +- drivers/md/dm-vdo/vio.c | 10 +- 39 files changed, 505 insertions(+), 505 deletions(-) diff --git a/drivers/md/dm-vdo/admin-state.c b/drivers/md/dm-vdo/admin-state.c index d695af42d140..3f9dba525154 100644 --- a/drivers/md/dm-vdo/admin-state.c +++ b/drivers/md/dm-vdo/admin-state.c @@ -228,12 +228,12 @@ static int __must_check begin_operation(struct admin_state *state, const struct admin_state_code *next_state = get_next_state(state, operation); if (next_state == NULL) { - result = uds_log_error_strerror(VDO_INVALID_ADMIN_STATE, + result = vdo_log_error_strerror(VDO_INVALID_ADMIN_STATE, "Can't start %s from %s", operation->name, vdo_get_admin_state_code(state)->name); } else if (state->waiter != NULL) { - result = uds_log_error_strerror(VDO_COMPONENT_BUSY, + result = vdo_log_error_strerror(VDO_COMPONENT_BUSY, "Can't start %s with extant waiter", operation->name); } else { @@ -291,7 +291,7 @@ static bool check_code(bool valid, const struct admin_state_code *code, const ch if (valid) return true; - result = uds_log_error_strerror(VDO_INVALID_ADMIN_STATE, + result = vdo_log_error_strerror(VDO_INVALID_ADMIN_STATE, "%s is not a %s", code->name, what); if (waiter != NULL) vdo_continue_completion(waiter, result); @@ -334,7 +334,7 @@ bool vdo_start_draining(struct admin_state *state, } if (!code->normal) { - uds_log_error_strerror(VDO_INVALID_ADMIN_STATE, "can't start %s from %s", + vdo_log_error_strerror(VDO_INVALID_ADMIN_STATE, "can't start %s from %s", operation->name, code->name); vdo_continue_completion(waiter, VDO_INVALID_ADMIN_STATE); return false; diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index b70294d8bb61..e79156dc7cc9 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -264,7 +264,7 @@ static void report_cache_pressure(struct vdo_page_cache *cache) ADD_ONCE(cache->stats.cache_pressure, 1); if (cache->waiter_count > cache->page_count) { if ((cache->pressure_report % LOG_INTERVAL) == 0) - uds_log_info("page cache pressure %u", cache->stats.cache_pressure); + vdo_log_info("page cache pressure %u", cache->stats.cache_pressure); if (++cache->pressure_report >= DISPLAY_INTERVAL) cache->pressure_report = 0; @@ -483,7 +483,7 @@ static void complete_with_page(struct page_info *info, bool available = vdo_page_comp->writable ? is_present(info) : is_valid(info); if (!available) { - uds_log_error_strerror(VDO_BAD_PAGE, + vdo_log_error_strerror(VDO_BAD_PAGE, "Requested cache page %llu in state %s is not %s", (unsigned long long) info->pbn, get_page_state_name(info->state), @@ -563,7 +563,7 @@ static void set_persistent_error(struct vdo_page_cache *cache, const char *conte struct vdo *vdo = cache->vdo; if ((result != VDO_READ_ONLY) && !vdo_is_read_only(vdo)) { - uds_log_error_strerror(result, "VDO Page Cache persistent error: %s", + vdo_log_error_strerror(result, "VDO Page Cache persistent error: %s", context); vdo_enter_read_only_mode(vdo, result); } @@ -704,7 +704,7 @@ static void page_is_loaded(struct vdo_completion *completion) validity = vdo_validate_block_map_page(page, nonce, info->pbn); if (validity == VDO_BLOCK_MAP_PAGE_BAD) { physical_block_number_t pbn = vdo_get_block_map_page_pbn(page); - int result = uds_log_error_strerror(VDO_BAD_PAGE, + int result = vdo_log_error_strerror(VDO_BAD_PAGE, "Expected page %llu but got page %llu instead", (unsigned long long) info->pbn, (unsigned long long) pbn); @@ -894,7 +894,7 @@ static void allocate_free_page(struct page_info *info) if (!vdo_waitq_has_waiters(&cache->free_waiters)) { if (cache->stats.cache_pressure > 0) { - uds_log_info("page cache pressure relieved"); + vdo_log_info("page cache pressure relieved"); WRITE_ONCE(cache->stats.cache_pressure, 0); } @@ -1012,7 +1012,7 @@ static void handle_page_write_error(struct vdo_completion *completion) /* If we're already read-only, write failures are to be expected. */ if (result != VDO_READ_ONLY) { - uds_log_ratelimit(uds_log_error, + vdo_log_ratelimit(vdo_log_error, "failed to write block map page %llu", (unsigned long long) info->pbn); } @@ -1397,7 +1397,7 @@ bool vdo_copy_valid_page(char *buffer, nonce_t nonce, } if (validity == VDO_BLOCK_MAP_PAGE_BAD) { - uds_log_error_strerror(VDO_BAD_PAGE, + vdo_log_error_strerror(VDO_BAD_PAGE, "Expected page %llu but got page %llu instead", (unsigned long long) pbn, (unsigned long long) vdo_get_block_map_page_pbn(loaded)); @@ -1785,7 +1785,7 @@ static void continue_with_loaded_page(struct data_vio *data_vio, vdo_unpack_block_map_entry(&page->entries[slot.block_map_slot.slot]); if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { - uds_log_error_strerror(VDO_BAD_MAPPING, + vdo_log_error_strerror(VDO_BAD_MAPPING, "Invalid block map tree PBN: %llu with state %u for page index %u at height %u", (unsigned long long) mapping.pbn, mapping.state, lock->tree_slots[lock->height - 1].page_index, @@ -2263,7 +2263,7 @@ void vdo_find_block_map_slot(struct data_vio *data_vio) /* The page at this height has been allocated and loaded. */ mapping = vdo_unpack_block_map_entry(&page->entries[tree_slot.block_map_slot.slot]); if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { - uds_log_error_strerror(VDO_BAD_MAPPING, + vdo_log_error_strerror(VDO_BAD_MAPPING, "Invalid block map tree PBN: %llu with state %u for page index %u at height %u", (unsigned long long) mapping.pbn, mapping.state, lock->tree_slots[lock->height - 1].page_index, @@ -3140,7 +3140,7 @@ static int __must_check set_mapped_location(struct data_vio *data_vio, * Log the corruption even if we wind up ignoring it for write VIOs, converting all cases * to VDO_BAD_MAPPING. */ - uds_log_error_strerror(VDO_BAD_MAPPING, + vdo_log_error_strerror(VDO_BAD_MAPPING, "PBN %llu with state %u read from the block map was invalid", (unsigned long long) mapped.pbn, mapped.state); diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index 2b0d42c77e05..94f6f1ccfb7d 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -792,25 +792,25 @@ static int initialize_data_vio(struct data_vio *data_vio, struct vdo *vdo) result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "data_vio data", &data_vio->vio.data); if (result != VDO_SUCCESS) - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "data_vio data allocation failure"); result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "compressed block", &data_vio->compression.block); if (result != VDO_SUCCESS) { - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "data_vio compressed block allocation failure"); } result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "vio scratch", &data_vio->scratch_block); if (result != VDO_SUCCESS) - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "data_vio scratch allocation failure"); result = vdo_create_bio(&bio); if (result != VDO_SUCCESS) - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "data_vio data bio allocation failure"); vdo_initialize_completion(&data_vio->decrement_completion, vdo, @@ -1025,7 +1025,7 @@ void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *com static void dump_limiter(const char *name, struct limiter *limiter) { - uds_log_info("%s: %u of %u busy (max %u), %s", name, limiter->busy, + vdo_log_info("%s: %u of %u busy (max %u), %s", name, limiter->busy, limiter->limit, limiter->max_busy, ((bio_list_empty(&limiter->waiters) && bio_list_empty(&limiter->new_waiters)) ? @@ -1323,7 +1323,7 @@ static void perform_cleanup_stage(struct data_vio *data_vio, if ((data_vio->recovery_sequence_number > 0) && (READ_ONCE(vdo->read_only_notifier.read_only_error) == VDO_SUCCESS) && (data_vio->vio.completion.result != VDO_READ_ONLY)) - uds_log_warning("VDO not read-only when cleaning data_vio with RJ lock"); + vdo_log_warning("VDO not read-only when cleaning data_vio with RJ lock"); fallthrough; case VIO_RELEASE_LOGICAL: @@ -1353,7 +1353,7 @@ static void enter_read_only_mode(struct vdo_completion *completion) if (completion->result != VDO_READ_ONLY) { struct data_vio *data_vio = as_data_vio(completion); - uds_log_error_strerror(completion->result, + vdo_log_error_strerror(completion->result, "Preparing to enter read-only mode: data_vio for LBN %llu (becoming mapped to %llu, previously mapped to %llu, allocated %llu) is completing with a fatal error after operation %s", (unsigned long long) data_vio->logical.lbn, (unsigned long long) data_vio->new_mapped.pbn, @@ -1449,14 +1449,14 @@ int uncompress_data_vio(struct data_vio *data_vio, &fragment_offset, &fragment_size); if (result != VDO_SUCCESS) { - uds_log_debug("%s: compressed fragment error %d", __func__, result); + vdo_log_debug("%s: compressed fragment error %d", __func__, result); return result; } size = LZ4_decompress_safe((block->data + fragment_offset), buffer, fragment_size, VDO_BLOCK_SIZE); if (size != VDO_BLOCK_SIZE) { - uds_log_debug("%s: lz4 error", __func__); + vdo_log_debug("%s: lz4 error", __func__); return VDO_INVALID_FRAGMENT; } diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index c031ab01054d..117266e1b3ae 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -1287,7 +1287,7 @@ static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_loc if (result == VDO_SUCCESS) return true; - uds_log_warning_strerror(result, + vdo_log_warning_strerror(result, "Error acquiring provisional reference for dedupe candidate; aborting dedupe"); agent->is_duplicate = false; vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, @@ -1614,7 +1614,7 @@ static bool decode_uds_advice(struct dedupe_context *context) version = encoding->data[offset++]; if (version != UDS_ADVICE_VERSION) { - uds_log_error("invalid UDS advice version code %u", version); + vdo_log_error("invalid UDS advice version code %u", version); return false; } @@ -1625,7 +1625,7 @@ static bool decode_uds_advice(struct dedupe_context *context) /* Don't use advice that's clearly meaningless. */ if ((advice->state == VDO_MAPPING_STATE_UNMAPPED) || (advice->pbn == VDO_ZERO_BLOCK)) { - uds_log_debug("Invalid advice from deduplication server: pbn %llu, state %u. Giving up on deduplication of logical block %llu", + vdo_log_debug("Invalid advice from deduplication server: pbn %llu, state %u. Giving up on deduplication of logical block %llu", (unsigned long long) advice->pbn, advice->state, (unsigned long long) data_vio->logical.lbn); atomic64_inc(&vdo->stats.invalid_advice_pbn_count); @@ -1634,7 +1634,7 @@ static bool decode_uds_advice(struct dedupe_context *context) result = vdo_get_physical_zone(vdo, advice->pbn, &advice->zone); if ((result != VDO_SUCCESS) || (advice->zone == NULL)) { - uds_log_debug("Invalid physical block number from deduplication server: %llu, giving up on deduplication of logical block %llu", + vdo_log_debug("Invalid physical block number from deduplication server: %llu, giving up on deduplication of logical block %llu", (unsigned long long) advice->pbn, (unsigned long long) data_vio->logical.lbn); atomic64_inc(&vdo->stats.invalid_advice_pbn_count); @@ -2053,7 +2053,7 @@ static void close_index(struct hash_zones *zones) result = uds_close_index(zones->index_session); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "Error closing index"); + vdo_log_error_strerror(result, "Error closing index"); spin_lock(&zones->lock); zones->index_state = IS_CLOSED; zones->error_flag |= result != UDS_SUCCESS; @@ -2080,7 +2080,7 @@ static void open_index(struct hash_zones *zones) result = uds_open_index(create_flag ? UDS_CREATE : UDS_LOAD, &zones->parameters, zones->index_session); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "Error opening index"); + vdo_log_error_strerror(result, "Error opening index"); spin_lock(&zones->lock); if (!create_flag) { @@ -2104,7 +2104,7 @@ static void open_index(struct hash_zones *zones) zones->index_target = IS_CLOSED; zones->error_flag = true; spin_unlock(&zones->lock); - uds_log_info("Setting UDS index target state to error"); + vdo_log_info("Setting UDS index target state to error"); spin_lock(&zones->lock); } /* @@ -2160,7 +2160,7 @@ static void report_dedupe_timeouts(struct hash_zones *zones, unsigned int timeou u64 unreported = atomic64_read(&zones->timeouts); unreported -= zones->reported_timeouts; - uds_log_debug("UDS index timeout on %llu requests", + vdo_log_debug("UDS index timeout on %llu requests", (unsigned long long) unreported); zones->reported_timeouts += unreported; } @@ -2207,7 +2207,7 @@ static int initialize_index(struct vdo *vdo, struct hash_zones *zones) 1, NULL); if (result != VDO_SUCCESS) { uds_destroy_index_session(vdo_forget(zones->index_session)); - uds_log_error("UDS index queue initialization failed (%d)", result); + vdo_log_error("UDS index queue initialization failed (%d)", result); return result; } @@ -2502,7 +2502,7 @@ static void initiate_suspend_index(struct admin_state *state) result = uds_suspend_index_session(zones->index_session, save); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "Error suspending dedupe index"); + vdo_log_error_strerror(result, "Error suspending dedupe index"); } vdo_finish_draining(state); @@ -2585,7 +2585,7 @@ static void resume_index(void *context, struct vdo_completion *parent) zones->parameters.bdev = config->owned_device->bdev; result = uds_resume_index_session(zones->index_session, zones->parameters.bdev); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "Error resuming dedupe index"); + vdo_log_error_strerror(result, "Error resuming dedupe index"); spin_lock(&zones->lock); vdo_resume_if_quiescent(&zones->state); @@ -2665,7 +2665,7 @@ static void get_index_statistics(struct hash_zones *zones, result = uds_get_index_session_stats(zones->index_session, &index_stats); if (result != UDS_SUCCESS) { - uds_log_error_strerror(result, "Error reading index stats"); + vdo_log_error_strerror(result, "Error reading index stats"); return; } @@ -2750,7 +2750,7 @@ static void dump_hash_lock(const struct hash_lock *lock) * unambiguous. 'U' indicates a lock not registered in the map. */ state = get_hash_lock_state_name(lock->state); - uds_log_info(" hl %px: %3.3s %c%llu/%u rc=%u wc=%zu agt=%px", + vdo_log_info(" hl %px: %3.3s %c%llu/%u rc=%u wc=%zu agt=%px", lock, state, (lock->registered ? 'D' : 'U'), (unsigned long long) lock->duplicate.pbn, lock->duplicate.state, lock->reference_count, @@ -2784,11 +2784,11 @@ static void dump_hash_zone(const struct hash_zone *zone) data_vio_count_t i; if (zone->hash_lock_map == NULL) { - uds_log_info("struct hash_zone %u: NULL map", zone->zone_number); + vdo_log_info("struct hash_zone %u: NULL map", zone->zone_number); return; } - uds_log_info("struct hash_zone %u: mapSize=%zu", + vdo_log_info("struct hash_zone %u: mapSize=%zu", zone->zone_number, vdo_int_map_size(zone->hash_lock_map)); for (i = 0; i < LOCK_POOL_CAPACITY; i++) dump_hash_lock(&zone->lock_array[i]); @@ -2808,9 +2808,9 @@ void vdo_dump_hash_zones(struct hash_zones *zones) target = (zones->changing ? index_state_to_string(zones, zones->index_target) : NULL); spin_unlock(&zones->lock); - uds_log_info("UDS index: state: %s", state); + vdo_log_info("UDS index: state: %s", state); if (target != NULL) - uds_log_info("UDS index: changing to state: %s", target); + vdo_log_info("UDS index: changing to state: %s", target); for (zone = 0; zone < zones->zone_count; zone++) dump_hash_zone(&zones->zones[zone]); @@ -2957,7 +2957,7 @@ static void set_target_state(struct hash_zones *zones, enum index_state target, spin_unlock(&zones->lock); if (old_state != new_state) - uds_log_info("Setting UDS index target state to %s", new_state); + vdo_log_info("Setting UDS index target state to %s", new_state); } const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 288e9b79bf16..4908996f5224 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -232,9 +232,9 @@ static int get_version_number(int argc, char **argv, char **error_ptr, } if (*version_ptr != TABLE_VERSION) { - uds_log_warning("Detected version mismatch between kernel module and tools kernel: %d, tool: %d", + vdo_log_warning("Detected version mismatch between kernel module and tools kernel: %d, tool: %d", TABLE_VERSION, *version_ptr); - uds_log_warning("Please consider upgrading management tools to match kernel."); + vdo_log_warning("Please consider upgrading management tools to match kernel."); } return VDO_SUCCESS; } @@ -399,10 +399,10 @@ static int process_one_thread_config_spec(const char *thread_param_type, /* Handle limited thread parameters */ if (strcmp(thread_param_type, "bioRotationInterval") == 0) { if (count == 0) { - uds_log_error("thread config string error: 'bioRotationInterval' of at least 1 is required"); + vdo_log_error("thread config string error: 'bioRotationInterval' of at least 1 is required"); return -EINVAL; } else if (count > VDO_BIO_ROTATION_INTERVAL_LIMIT) { - uds_log_error("thread config string error: 'bioRotationInterval' cannot be higher than %d", + vdo_log_error("thread config string error: 'bioRotationInterval' cannot be higher than %d", VDO_BIO_ROTATION_INTERVAL_LIMIT); return -EINVAL; } @@ -411,7 +411,7 @@ static int process_one_thread_config_spec(const char *thread_param_type, } if (strcmp(thread_param_type, "logical") == 0) { if (count > MAX_VDO_LOGICAL_ZONES) { - uds_log_error("thread config string error: at most %d 'logical' threads are allowed", + vdo_log_error("thread config string error: at most %d 'logical' threads are allowed", MAX_VDO_LOGICAL_ZONES); return -EINVAL; } @@ -420,7 +420,7 @@ static int process_one_thread_config_spec(const char *thread_param_type, } if (strcmp(thread_param_type, "physical") == 0) { if (count > MAX_VDO_PHYSICAL_ZONES) { - uds_log_error("thread config string error: at most %d 'physical' threads are allowed", + vdo_log_error("thread config string error: at most %d 'physical' threads are allowed", MAX_VDO_PHYSICAL_ZONES); return -EINVAL; } @@ -429,7 +429,7 @@ static int process_one_thread_config_spec(const char *thread_param_type, } /* Handle other thread count parameters */ if (count > MAXIMUM_VDO_THREADS) { - uds_log_error("thread config string error: at most %d '%s' threads are allowed", + vdo_log_error("thread config string error: at most %d '%s' threads are allowed", MAXIMUM_VDO_THREADS, thread_param_type); return -EINVAL; } @@ -439,7 +439,7 @@ static int process_one_thread_config_spec(const char *thread_param_type, } if (strcmp(thread_param_type, "cpu") == 0) { if (count == 0) { - uds_log_error("thread config string error: at least one 'cpu' thread required"); + vdo_log_error("thread config string error: at least one 'cpu' thread required"); return -EINVAL; } config->cpu_threads = count; @@ -451,7 +451,7 @@ static int process_one_thread_config_spec(const char *thread_param_type, } if (strcmp(thread_param_type, "bio") == 0) { if (count == 0) { - uds_log_error("thread config string error: at least one 'bio' thread required"); + vdo_log_error("thread config string error: at least one 'bio' thread required"); return -EINVAL; } config->bio_threads = count; @@ -462,7 +462,7 @@ static int process_one_thread_config_spec(const char *thread_param_type, * Don't fail, just log. This will handle version mismatches between user mode tools and * kernel. */ - uds_log_info("unknown thread parameter type \"%s\"", thread_param_type); + vdo_log_info("unknown thread parameter type \"%s\"", thread_param_type); return VDO_SUCCESS; } @@ -484,7 +484,7 @@ static int parse_one_thread_config_spec(const char *spec, return result; if ((fields[0] == NULL) || (fields[1] == NULL) || (fields[2] != NULL)) { - uds_log_error("thread config string error: expected thread parameter assignment, saw \"%s\"", + vdo_log_error("thread config string error: expected thread parameter assignment, saw \"%s\"", spec); free_string_array(fields); return -EINVAL; @@ -492,7 +492,7 @@ static int parse_one_thread_config_spec(const char *spec, result = kstrtouint(fields[1], 10, &count); if (result) { - uds_log_error("thread config string error: integer value needed, found \"%s\"", + vdo_log_error("thread config string error: integer value needed, found \"%s\"", fields[1]); free_string_array(fields); return result; @@ -564,12 +564,12 @@ static int process_one_key_value_pair(const char *key, unsigned int value, /* Non thread optional parameters */ if (strcmp(key, "maxDiscard") == 0) { if (value == 0) { - uds_log_error("optional parameter error: at least one max discard block required"); + vdo_log_error("optional parameter error: at least one max discard block required"); return -EINVAL; } /* Max discard sectors in blkdev_issue_discard is UINT_MAX >> 9 */ if (value > (UINT_MAX / VDO_BLOCK_SIZE)) { - uds_log_error("optional parameter error: at most %d max discard blocks are allowed", + vdo_log_error("optional parameter error: at most %d max discard blocks are allowed", UINT_MAX / VDO_BLOCK_SIZE); return -EINVAL; } @@ -604,7 +604,7 @@ static int parse_one_key_value_pair(const char *key, const char *value, /* The remaining arguments must have integral values. */ result = kstrtouint(value, 10, &count); if (result) { - uds_log_error("optional config string error: integer value needed, found \"%s\"", + vdo_log_error("optional config string error: integer value needed, found \"%s\"", value); return result; } @@ -745,7 +745,7 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti, return VDO_BAD_CONFIGURATION; } - uds_log_info("table line: %s", config->original_string); + vdo_log_info("table line: %s", config->original_string); config->thread_counts = (struct thread_count_config) { .bio_ack_threads = 1, @@ -872,7 +872,7 @@ static int parse_device_config(int argc, char **argv, struct dm_target *ti, result = dm_get_device(ti, config->parent_device_name, dm_table_get_mode(ti->table), &config->owned_device); if (result != 0) { - uds_log_error("couldn't open device \"%s\": error %d", + vdo_log_error("couldn't open device \"%s\": error %d", config->parent_device_name, result); handle_parse_error(config, error_ptr, "Unable to open storage device"); return VDO_BAD_CONFIGURATION; @@ -1029,12 +1029,12 @@ static int __must_check process_vdo_message_locked(struct vdo *vdo, unsigned int return 0; } - uds_log_warning("invalid argument '%s' to dmsetup compression message", + vdo_log_warning("invalid argument '%s' to dmsetup compression message", argv[1]); return -EINVAL; } - uds_log_warning("unrecognized dmsetup message '%s' received", argv[0]); + vdo_log_warning("unrecognized dmsetup message '%s' received", argv[0]); return -EINVAL; } @@ -1091,7 +1091,7 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, int result; if (argc == 0) { - uds_log_warning("unspecified dmsetup message"); + vdo_log_warning("unspecified dmsetup message"); return -EINVAL; } @@ -1211,7 +1211,7 @@ static int perform_admin_operation(struct vdo *vdo, u32 starting_phase, struct vdo_administrator *admin = &vdo->admin; if (atomic_cmpxchg(&admin->busy, 0, 1) != 0) { - return uds_log_error_strerror(VDO_COMPONENT_BUSY, + return vdo_log_error_strerror(VDO_COMPONENT_BUSY, "Can't start %s operation, another operation is already in progress", type); } @@ -1285,7 +1285,7 @@ static int __must_check decode_from_super_block(struct vdo *vdo) * block, just accept it. */ if (vdo->states.vdo.config.logical_blocks < config->logical_blocks) { - uds_log_warning("Growing logical size: a logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", + vdo_log_warning("Growing logical size: a logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", (unsigned long long) config->logical_blocks, (unsigned long long) vdo->states.vdo.config.logical_blocks); vdo->states.vdo.config.logical_blocks = config->logical_blocks; @@ -1328,14 +1328,14 @@ static int __must_check decode_vdo(struct vdo *vdo) journal_length = vdo_get_recovery_journal_length(vdo->states.vdo.config.recovery_journal_size); if (maximum_age > (journal_length / 2)) { - return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, "maximum age: %llu exceeds limit %llu", (unsigned long long) maximum_age, (unsigned long long) (journal_length / 2)); } if (maximum_age == 0) { - return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, "maximum age must be greater than 0"); } @@ -1451,19 +1451,19 @@ static int vdo_initialize(struct dm_target *ti, unsigned int instance, u64 logical_size = to_bytes(ti->len); block_count_t logical_blocks = logical_size / block_size; - uds_log_info("loading device '%s'", vdo_get_device_name(ti)); - uds_log_debug("Logical block size = %llu", (u64) config->logical_block_size); - uds_log_debug("Logical blocks = %llu", logical_blocks); - uds_log_debug("Physical block size = %llu", (u64) block_size); - uds_log_debug("Physical blocks = %llu", config->physical_blocks); - uds_log_debug("Block map cache blocks = %u", config->cache_size); - uds_log_debug("Block map maximum age = %u", config->block_map_maximum_age); - uds_log_debug("Deduplication = %s", (config->deduplication ? "on" : "off")); - uds_log_debug("Compression = %s", (config->compression ? "on" : "off")); + vdo_log_info("loading device '%s'", vdo_get_device_name(ti)); + vdo_log_debug("Logical block size = %llu", (u64) config->logical_block_size); + vdo_log_debug("Logical blocks = %llu", logical_blocks); + vdo_log_debug("Physical block size = %llu", (u64) block_size); + vdo_log_debug("Physical blocks = %llu", config->physical_blocks); + vdo_log_debug("Block map cache blocks = %u", config->cache_size); + vdo_log_debug("Block map maximum age = %u", config->block_map_maximum_age); + vdo_log_debug("Deduplication = %s", (config->deduplication ? "on" : "off")); + vdo_log_debug("Compression = %s", (config->compression ? "on" : "off")); vdo = vdo_find_matching(vdo_uses_device, config); if (vdo != NULL) { - uds_log_error("Existing vdo already uses device %s", + vdo_log_error("Existing vdo already uses device %s", vdo->device_config->parent_device_name); ti->error = "Cannot share storage device with already-running VDO"; return VDO_BAD_CONFIGURATION; @@ -1471,7 +1471,7 @@ static int vdo_initialize(struct dm_target *ti, unsigned int instance, result = vdo_make(instance, config, &ti->error, &vdo); if (result != VDO_SUCCESS) { - uds_log_error("Could not create VDO device. (VDO error %d, message %s)", + vdo_log_error("Could not create VDO device. (VDO error %d, message %s)", result, ti->error); vdo_destroy(vdo); return result; @@ -1483,7 +1483,7 @@ static int vdo_initialize(struct dm_target *ti, unsigned int instance, ti->error = ((result == VDO_INVALID_ADMIN_STATE) ? "Pre-load is only valid immediately after initialization" : "Cannot load metadata from device"); - uds_log_error("Could not start VDO device. (VDO error %d, message %s)", + vdo_log_error("Could not start VDO device. (VDO error %d, message %s)", result, ti->error); vdo_destroy(vdo); return result; @@ -1594,7 +1594,7 @@ static int construct_new_vdo_registered(struct dm_target *ti, unsigned int argc, result = parse_device_config(argc, argv, ti, &config); if (result != VDO_SUCCESS) { - uds_log_error_strerror(result, "parsing failed: %s", ti->error); + vdo_log_error_strerror(result, "parsing failed: %s", ti->error); release_instance(instance); return -EINVAL; } @@ -1723,7 +1723,7 @@ static int prepare_to_grow_physical(struct vdo *vdo, block_count_t new_physical_ int result; block_count_t current_physical_blocks = vdo->states.vdo.config.physical_blocks; - uds_log_info("Preparing to resize physical to %llu", + vdo_log_info("Preparing to resize physical to %llu", (unsigned long long) new_physical_blocks); VDO_ASSERT_LOG_ONLY((new_physical_blocks > current_physical_blocks), "New physical size is larger than current physical size"); @@ -1746,7 +1746,7 @@ static int prepare_to_grow_physical(struct vdo *vdo, block_count_t new_physical_ return result; } - uds_log_info("Done preparing to resize physical"); + vdo_log_info("Done preparing to resize physical"); return VDO_SUCCESS; } @@ -1823,7 +1823,7 @@ static int prepare_to_modify(struct dm_target *ti, struct device_config *config, if (config->logical_blocks > vdo->device_config->logical_blocks) { block_count_t logical_blocks = vdo->states.vdo.config.logical_blocks; - uds_log_info("Preparing to resize logical to %llu", + vdo_log_info("Preparing to resize logical to %llu", (unsigned long long) config->logical_blocks); VDO_ASSERT_LOG_ONLY((config->logical_blocks > logical_blocks), "New logical size is larger than current size"); @@ -1835,7 +1835,7 @@ static int prepare_to_modify(struct dm_target *ti, struct device_config *config, return result; } - uds_log_info("Done preparing to resize logical"); + vdo_log_info("Done preparing to resize logical"); } if (config->physical_blocks > vdo->device_config->physical_blocks) { @@ -1861,7 +1861,7 @@ static int prepare_to_modify(struct dm_target *ti, struct device_config *config, if (strcmp(config->parent_device_name, vdo->device_config->parent_device_name) != 0) { const char *device_name = vdo_get_device_name(config->owning_target); - uds_log_info("Updating backing device of %s from %s to %s", device_name, + vdo_log_info("Updating backing device of %s from %s to %s", device_name, vdo->device_config->parent_device_name, config->parent_device_name); } @@ -1879,7 +1879,7 @@ static int update_existing_vdo(const char *device_name, struct dm_target *ti, if (result != VDO_SUCCESS) return -EINVAL; - uds_log_info("preparing to modify device '%s'", device_name); + vdo_log_info("preparing to modify device '%s'", device_name); result = prepare_to_modify(ti, config, vdo); if (result != VDO_SUCCESS) { free_device_config(config); @@ -1929,12 +1929,12 @@ static void vdo_dtr(struct dm_target *ti) vdo_register_allocating_thread(&allocating_thread, NULL); device_name = vdo_get_device_name(ti); - uds_log_info("stopping device '%s'", device_name); + vdo_log_info("stopping device '%s'", device_name); if (vdo->dump_on_shutdown) vdo_dump_all(vdo, "device shutdown"); vdo_destroy(vdo_forget(vdo)); - uds_log_info("device '%s' stopped", device_name); + vdo_log_info("device '%s' stopped", device_name); vdo_unregister_thread_device_id(); vdo_unregister_allocating_thread(); release_instance(instance); @@ -2096,7 +2096,7 @@ static void vdo_postsuspend(struct dm_target *ti) vdo_register_thread_device_id(&instance_thread, &vdo->instance); device_name = vdo_get_device_name(vdo->device_config->owning_target); - uds_log_info("suspending device '%s'", device_name); + vdo_log_info("suspending device '%s'", device_name); /* * It's important to note any error here does not actually stop device-mapper from @@ -2110,12 +2110,12 @@ static void vdo_postsuspend(struct dm_target *ti) * Treat VDO_READ_ONLY as a success since a read-only suspension still leaves the * VDO suspended. */ - uds_log_info("device '%s' suspended", device_name); + vdo_log_info("device '%s' suspended", device_name); } else if (result == VDO_INVALID_ADMIN_STATE) { - uds_log_error("Suspend invoked while in unexpected state: %s", + vdo_log_error("Suspend invoked while in unexpected state: %s", vdo_get_admin_state(vdo)->name); } else { - uds_log_error_strerror(result, "Suspend of device '%s' failed", + vdo_log_error_strerror(result, "Suspend of device '%s' failed", device_name); } @@ -2288,13 +2288,13 @@ static void handle_load_error(struct vdo_completion *completion) if (vdo_state_requires_read_only_rebuild(vdo->load_state) && (vdo->admin.phase == LOAD_PHASE_MAKE_DIRTY)) { - uds_log_error_strerror(completion->result, "aborting load"); + vdo_log_error_strerror(completion->result, "aborting load"); vdo->admin.phase = LOAD_PHASE_DRAIN_JOURNAL; load_callback(vdo_forget(completion)); return; } - uds_log_error_strerror(completion->result, + vdo_log_error_strerror(completion->result, "Entering read-only mode due to load error"); vdo->admin.phase = LOAD_PHASE_WAIT_FOR_READ_ONLY; vdo_enter_read_only_mode(vdo, completion->result); @@ -2386,7 +2386,7 @@ static void resume_callback(struct vdo_completion *completion) if (enable != was_enabled) WRITE_ONCE(vdo->compressing, enable); - uds_log_info("compression is %s", (enable ? "enabled" : "disabled")); + vdo_log_info("compression is %s", (enable ? "enabled" : "disabled")); vdo_resume_packer(vdo->packer, completion); return; @@ -2426,7 +2426,7 @@ static void grow_logical_callback(struct vdo_completion *completion) switch (advance_phase(vdo)) { case GROW_LOGICAL_PHASE_START: if (vdo_is_read_only(vdo)) { - uds_log_error_strerror(VDO_READ_ONLY, + vdo_log_error_strerror(VDO_READ_ONLY, "Can't grow logical size of a read-only VDO"); vdo_set_completion_result(completion, VDO_READ_ONLY); break; @@ -2505,7 +2505,7 @@ static int perform_grow_logical(struct vdo *vdo, block_count_t new_logical_block return VDO_SUCCESS; } - uds_log_info("Resizing logical to %llu", + vdo_log_info("Resizing logical to %llu", (unsigned long long) new_logical_blocks); if (vdo->block_map->next_entry_count != new_logical_blocks) return VDO_PARAMETER_MISMATCH; @@ -2516,7 +2516,7 @@ static int perform_grow_logical(struct vdo *vdo, block_count_t new_logical_block if (result != VDO_SUCCESS) return result; - uds_log_info("Logical blocks now %llu", (unsigned long long) new_logical_blocks); + vdo_log_info("Logical blocks now %llu", (unsigned long long) new_logical_blocks); return VDO_SUCCESS; } @@ -2576,7 +2576,7 @@ static void grow_physical_callback(struct vdo_completion *completion) switch (advance_phase(vdo)) { case GROW_PHYSICAL_PHASE_START: if (vdo_is_read_only(vdo)) { - uds_log_error_strerror(VDO_READ_ONLY, + vdo_log_error_strerror(VDO_READ_ONLY, "Can't grow physical size of a read-only VDO"); vdo_set_completion_result(completion, VDO_READ_ONLY); break; @@ -2685,7 +2685,7 @@ static int perform_grow_physical(struct vdo *vdo, block_count_t new_physical_blo if (result != VDO_SUCCESS) return result; - uds_log_info("Physical block count was %llu, now %llu", + vdo_log_info("Physical block count was %llu, now %llu", (unsigned long long) old_physical_blocks, (unsigned long long) new_physical_blocks); return VDO_SUCCESS; @@ -2707,13 +2707,13 @@ static int __must_check apply_new_vdo_configuration(struct vdo *vdo, result = perform_grow_logical(vdo, config->logical_blocks); if (result != VDO_SUCCESS) { - uds_log_error("grow logical operation failed, result = %d", result); + vdo_log_error("grow logical operation failed, result = %d", result); return result; } result = perform_grow_physical(vdo, config->physical_blocks); if (result != VDO_SUCCESS) - uds_log_error("resize operation failed, result = %d", result); + vdo_log_error("resize operation failed, result = %d", result); return result; } @@ -2728,14 +2728,14 @@ static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) backing_blocks = get_underlying_device_block_count(vdo); if (backing_blocks < config->physical_blocks) { /* FIXME: can this still happen? */ - uds_log_error("resume of device '%s' failed: backing device has %llu blocks but VDO physical size is %llu blocks", + vdo_log_error("resume of device '%s' failed: backing device has %llu blocks but VDO physical size is %llu blocks", device_name, (unsigned long long) backing_blocks, (unsigned long long) config->physical_blocks); return -EINVAL; } if (vdo_get_admin_state(vdo) == VDO_ADMIN_STATE_PRE_LOADED) { - uds_log_info("starting device '%s'", device_name); + vdo_log_info("starting device '%s'", device_name); result = perform_admin_operation(vdo, LOAD_PHASE_START, load_callback, handle_load_error, "load"); if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { @@ -2743,7 +2743,7 @@ static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) * Something has gone very wrong. Make sure everything has drained and * leave the device in an unresumable state. */ - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "Start failed, could not load VDO metadata"); vdo->suspend_type = VDO_ADMIN_STATE_STOPPING; perform_admin_operation(vdo, SUSPEND_PHASE_START, @@ -2753,10 +2753,10 @@ static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) } /* Even if the VDO is read-only, it is now able to handle read requests. */ - uds_log_info("device '%s' started", device_name); + vdo_log_info("device '%s' started", device_name); } - uds_log_info("resuming device '%s'", device_name); + vdo_log_info("resuming device '%s'", device_name); /* If this fails, the VDO was not in a state to be resumed. This should never happen. */ result = apply_new_vdo_configuration(vdo, config); @@ -2774,7 +2774,7 @@ static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) * written to disk. */ if (result != VDO_SUCCESS) { - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "Commit of modifications to device '%s' failed", device_name); vdo_enter_read_only_mode(vdo, result); @@ -2795,7 +2795,7 @@ static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) } if (result != VDO_SUCCESS) - uds_log_error("resume of device '%s' failed with error: %d", device_name, + vdo_log_error("resume of device '%s' failed with error: %d", device_name, result); return result; @@ -2821,7 +2821,7 @@ static void vdo_resume(struct dm_target *ti) vdo_register_thread_device_id(&instance_thread, &get_vdo_for_target(ti)->instance); - uds_log_info("device '%s' resumed", vdo_get_device_name(ti)); + vdo_log_info("device '%s' resumed", vdo_get_device_name(ti)); vdo_unregister_thread_device_id(); } @@ -2852,7 +2852,7 @@ static bool dm_registered; static void vdo_module_destroy(void) { - uds_log_debug("unloading"); + vdo_log_debug("unloading"); if (dm_registered) dm_unregister_target(&vdo_target_bio); @@ -2863,7 +2863,7 @@ static void vdo_module_destroy(void) vdo_free(instances.words); memset(&instances, 0, sizeof(struct instance_tracker)); - uds_log_info("unloaded version %s", CURRENT_VERSION); + vdo_log_info("unloaded version %s", CURRENT_VERSION); } static int __init vdo_init(void) @@ -2874,19 +2874,19 @@ static int __init vdo_init(void) vdo_memory_init(); vdo_initialize_thread_device_registry(); vdo_initialize_device_registry_once(); - uds_log_info("loaded version %s", CURRENT_VERSION); + vdo_log_info("loaded version %s", CURRENT_VERSION); /* Add VDO errors to the set of errors registered by the indexer. */ result = vdo_register_status_codes(); if (result != VDO_SUCCESS) { - uds_log_error("vdo_register_status_codes failed %d", result); + vdo_log_error("vdo_register_status_codes failed %d", result); vdo_module_destroy(); return result; } result = dm_register_target(&vdo_target_bio); if (result < 0) { - uds_log_error("dm_register_target failed %d", result); + vdo_log_error("dm_register_target failed %d", result); vdo_module_destroy(); return result; } diff --git a/drivers/md/dm-vdo/dump.c b/drivers/md/dm-vdo/dump.c index 52ee9a72781c..00e575d7d773 100644 --- a/drivers/md/dm-vdo/dump.c +++ b/drivers/md/dm-vdo/dump.c @@ -58,12 +58,12 @@ static void do_dump(struct vdo *vdo, unsigned int dump_options_requested, u32 active, maximum; s64 outstanding; - uds_log_info("%s dump triggered via %s", UDS_LOGGING_MODULE_NAME, why); + vdo_log_info("%s dump triggered via %s", VDO_LOGGING_MODULE_NAME, why); active = get_data_vio_pool_active_requests(vdo->data_vio_pool); maximum = get_data_vio_pool_maximum_requests(vdo->data_vio_pool); outstanding = (atomic64_read(&vdo->stats.bios_submitted) - atomic64_read(&vdo->stats.bios_completed)); - uds_log_info("%u device requests outstanding (max %u), %lld bio requests outstanding, device '%s'", + vdo_log_info("%u device requests outstanding (max %u), %lld bio requests outstanding, device '%s'", active, maximum, outstanding, vdo_get_device_name(vdo->device_config->owning_target)); if (((dump_options_requested & FLAG_SHOW_QUEUES) != 0) && (vdo->threads != NULL)) { @@ -80,7 +80,7 @@ static void do_dump(struct vdo *vdo, unsigned int dump_options_requested, vdo_dump_status(vdo); vdo_report_memory_usage(); - uds_log_info("end of %s dump", UDS_LOGGING_MODULE_NAME); + vdo_log_info("end of %s dump", VDO_LOGGING_MODULE_NAME); } static int parse_dump_options(unsigned int argc, char *const *argv, @@ -114,7 +114,7 @@ static int parse_dump_options(unsigned int argc, char *const *argv, } } if (j == ARRAY_SIZE(option_names)) { - uds_log_warning("dump option name '%s' unknown", argv[i]); + vdo_log_warning("dump option name '%s' unknown", argv[i]); options_okay = false; } } @@ -159,13 +159,13 @@ static void dump_vio_waiters(struct vdo_wait_queue *waitq, char *wait_on) data_vio = vdo_waiter_as_data_vio(first); - uds_log_info(" %s is locked. Waited on by: vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", + vdo_log_info(" %s is locked. Waited on by: vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", wait_on, data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio)); for (waiter = first->next_waiter; waiter != first; waiter = waiter->next_waiter) { data_vio = vdo_waiter_as_data_vio(waiter); - uds_log_info(" ... and : vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", + vdo_log_info(" ... and : vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio)); @@ -258,7 +258,7 @@ void dump_data_vio(void *data) encode_vio_dump_flags(data_vio, flags_dump_buffer); - uds_log_info(" vio %px %s%s %s %s%s", data_vio, + vdo_log_info(" vio %px %s%s %s %s%s", data_vio, vio_block_number_dump_buffer, vio_flush_generation_buffer, get_data_vio_operation_name(data_vio), diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c index ebb0a4edd109..a34ea0229d53 100644 --- a/drivers/md/dm-vdo/encodings.c +++ b/drivers/md/dm-vdo/encodings.c @@ -146,7 +146,7 @@ static int __must_check validate_version(struct version_number expected_version, const char *component_name) { if (!vdo_are_same_version(expected_version, actual_version)) { - return uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, "%s version mismatch, expected %d.%d, got %d.%d", component_name, expected_version.major_version, @@ -179,7 +179,7 @@ int vdo_validate_header(const struct header *expected_header, int result; if (expected_header->id != actual_header->id) { - return uds_log_error_strerror(VDO_INCORRECT_COMPONENT, + return vdo_log_error_strerror(VDO_INCORRECT_COMPONENT, "%s ID mismatch, expected %d, got %d", name, expected_header->id, actual_header->id); @@ -192,7 +192,7 @@ int vdo_validate_header(const struct header *expected_header, if ((expected_header->size > actual_header->size) || (exact_size && (expected_header->size < actual_header->size))) { - return uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, "%s size mismatch, expected %zu, got %zu", name, expected_header->size, actual_header->size); @@ -653,7 +653,7 @@ int vdo_configure_slab_depot(const struct partition *partition, physical_block_number_t last_block; block_count_t slab_size = slab_config.slab_blocks; - uds_log_debug("slabDepot %s(block_count=%llu, first_block=%llu, slab_size=%llu, zone_count=%u)", + vdo_log_debug("slabDepot %s(block_count=%llu, first_block=%llu, slab_size=%llu, zone_count=%u)", __func__, (unsigned long long) partition->count, (unsigned long long) partition->offset, (unsigned long long) slab_size, zone_count); @@ -677,7 +677,7 @@ int vdo_configure_slab_depot(const struct partition *partition, .zone_count = zone_count, }; - uds_log_debug("slab_depot last_block=%llu, total_data_blocks=%llu, slab_count=%zu, left_over=%llu", + vdo_log_debug("slab_depot last_block=%llu, total_data_blocks=%llu, slab_count=%zu, left_over=%llu", (unsigned long long) last_block, (unsigned long long) total_data_blocks, slab_count, (unsigned long long) (partition->count - (last_block - partition->offset))); @@ -875,7 +875,7 @@ int vdo_initialize_layout(block_count_t size, physical_block_number_t offset, (offset + block_map_blocks + journal_blocks + summary_blocks); if (necessary_size > size) - return uds_log_error_strerror(VDO_NO_SPACE, + return vdo_log_error_strerror(VDO_NO_SPACE, "Not enough space to make a VDO"); *layout = (struct layout) { @@ -1045,7 +1045,7 @@ static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t sta layout->num_partitions = layout_header.partition_count; if (layout->num_partitions > VDO_PARTITION_COUNT) { - return uds_log_error_strerror(VDO_UNKNOWN_PARTITION, + return vdo_log_error_strerror(VDO_UNKNOWN_PARTITION, "layout has extra partitions"); } @@ -1070,7 +1070,7 @@ static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t sta result = vdo_get_partition(layout, REQUIRED_PARTITIONS[i], &partition); if (result != VDO_SUCCESS) { vdo_uninitialize_layout(layout); - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "layout is missing required partition %u", REQUIRED_PARTITIONS[i]); } @@ -1080,7 +1080,7 @@ static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t sta if (start != size) { vdo_uninitialize_layout(layout); - return uds_log_error_strerror(UDS_BAD_STATE, + return vdo_log_error_strerror(UDS_BAD_STATE, "partitions do not cover the layout"); } @@ -1253,7 +1253,7 @@ int vdo_validate_config(const struct vdo_config *config, return VDO_OUT_OF_RANGE; if (physical_block_count != config->physical_blocks) { - uds_log_error("A physical size of %llu blocks was specified, not the %llu blocks configured in the vdo super block", + vdo_log_error("A physical size of %llu blocks was specified, not the %llu blocks configured in the vdo super block", (unsigned long long) physical_block_count, (unsigned long long) config->physical_blocks); return VDO_PARAMETER_MISMATCH; @@ -1266,7 +1266,7 @@ int vdo_validate_config(const struct vdo_config *config, return result; if (logical_block_count != config->logical_blocks) { - uds_log_error("A logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", + vdo_log_error("A logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", (unsigned long long) logical_block_count, (unsigned long long) config->logical_blocks); return VDO_PARAMETER_MISMATCH; @@ -1390,7 +1390,7 @@ int vdo_validate_component_states(struct vdo_component_states *states, block_count_t logical_size) { if (geometry_nonce != states->vdo.nonce) { - return uds_log_error_strerror(VDO_BAD_NONCE, + return vdo_log_error_strerror(VDO_BAD_NONCE, "Geometry nonce %llu does not match superblock nonce %llu", (unsigned long long) geometry_nonce, (unsigned long long) states->vdo.nonce); @@ -1463,7 +1463,7 @@ int vdo_decode_super_block(u8 *buffer) * We can't check release version or checksum until we know the content size, so we * have to assume a version mismatch on unexpected values. */ - return uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, "super block contents too large: %zu", header.size); } diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c index 3b5fddad8ddf..8b2d22381274 100644 --- a/drivers/md/dm-vdo/errors.c +++ b/drivers/md/dm-vdo/errors.c @@ -215,8 +215,8 @@ const char *uds_string_error_name(int errnum, char *buf, size_t buflen) */ int uds_status_to_errno(int error) { - char error_name[UDS_MAX_ERROR_NAME_SIZE]; - char error_message[UDS_MAX_ERROR_MESSAGE_SIZE]; + char error_name[VDO_MAX_ERROR_NAME_SIZE]; + char error_message[VDO_MAX_ERROR_MESSAGE_SIZE]; /* 0 is success, and negative values are already system error codes. */ if (likely(error <= 0)) @@ -248,7 +248,7 @@ int uds_status_to_errno(int error) default: /* Translate an unexpected error into something generic. */ - uds_log_info("%s: mapping status code %d (%s: %s) to -EIO", + vdo_log_info("%s: mapping status code %d (%s: %s) to -EIO", __func__, error, uds_string_error_name(error, error_name, sizeof(error_name)), diff --git a/drivers/md/dm-vdo/errors.h b/drivers/md/dm-vdo/errors.h index c6c085b26a0e..24e0e745fd5f 100644 --- a/drivers/md/dm-vdo/errors.h +++ b/drivers/md/dm-vdo/errors.h @@ -51,8 +51,8 @@ enum uds_status_codes { }; enum { - UDS_MAX_ERROR_NAME_SIZE = 80, - UDS_MAX_ERROR_MESSAGE_SIZE = 128, + VDO_MAX_ERROR_NAME_SIZE = 80, + VDO_MAX_ERROR_MESSAGE_SIZE = 128, }; struct error_info { diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c index e03679e4d1ba..57e87f0d7069 100644 --- a/drivers/md/dm-vdo/flush.c +++ b/drivers/md/dm-vdo/flush.c @@ -108,7 +108,7 @@ static void *allocate_flush(gfp_t gfp_mask, void *pool_data) int result = vdo_allocate(1, struct vdo_flush, __func__, &flush); if (result != VDO_SUCCESS) - uds_log_error_strerror(result, "failed to allocate spare flush"); + vdo_log_error_strerror(result, "failed to allocate spare flush"); } if (flush != NULL) { @@ -349,11 +349,11 @@ void vdo_complete_flushes(struct flusher *flusher) */ void vdo_dump_flusher(const struct flusher *flusher) { - uds_log_info("struct flusher"); - uds_log_info(" flush_generation=%llu first_unacknowledged_generation=%llu", + vdo_log_info("struct flusher"); + vdo_log_info(" flush_generation=%llu first_unacknowledged_generation=%llu", (unsigned long long) flusher->flush_generation, (unsigned long long) flusher->first_unacknowledged_generation); - uds_log_info(" notifiers queue is %s; pending_flushes queue is %s", + vdo_log_info(" notifiers queue is %s; pending_flushes queue is %s", (vdo_waitq_has_waiters(&flusher->notifiers) ? "not empty" : "empty"), (vdo_waitq_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty")); } diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c index cf04cdef0750..ae11941c90a9 100644 --- a/drivers/md/dm-vdo/funnel-workqueue.c +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -485,7 +485,7 @@ static void dump_simple_work_queue(struct simple_work_queue *queue) thread_status = atomic_read(&queue->idle) ? "idle" : "running"; } - uds_log_info("workQ %px (%s) %s (%c)", &queue->common, queue->common.name, + vdo_log_info("workQ %px (%s) %s (%c)", &queue->common, queue->common.name, thread_status, task_state_report); /* ->waiting_worker_threads wait queue status? anyone waiting? */ diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c index 47e4ed234242..7e32a25d3f2f 100644 --- a/drivers/md/dm-vdo/indexer/chapter-index.c +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -166,7 +166,7 @@ int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, if (removals == 0) { uds_get_delta_index_stats(delta_index, &stats); - uds_log_warning("The chapter index for chapter %llu contains %llu entries with %llu collisions", + vdo_log_warning("The chapter index for chapter %llu contains %llu entries with %llu collisions", (unsigned long long) chapter_number, (unsigned long long) stats.record_count, (unsigned long long) stats.collision_count); @@ -198,7 +198,7 @@ int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, } if (removals > 0) { - uds_log_warning("To avoid chapter index page overflow in chapter %llu, %u entries were removed from the chapter index", + vdo_log_warning("To avoid chapter index page overflow in chapter %llu, %u entries were removed from the chapter index", (unsigned long long) chapter_number, removals); } diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c index 69bf27a9d61b..5532371b952f 100644 --- a/drivers/md/dm-vdo/indexer/config.c +++ b/drivers/md/dm-vdo/indexer/config.c @@ -33,54 +33,54 @@ static bool are_matching_configurations(struct uds_configuration *saved_config, bool result = true; if (saved_geometry->record_pages_per_chapter != geometry->record_pages_per_chapter) { - uds_log_error("Record pages per chapter (%u) does not match (%u)", + vdo_log_error("Record pages per chapter (%u) does not match (%u)", saved_geometry->record_pages_per_chapter, geometry->record_pages_per_chapter); result = false; } if (saved_geometry->chapters_per_volume != geometry->chapters_per_volume) { - uds_log_error("Chapter count (%u) does not match (%u)", + vdo_log_error("Chapter count (%u) does not match (%u)", saved_geometry->chapters_per_volume, geometry->chapters_per_volume); result = false; } if (saved_geometry->sparse_chapters_per_volume != geometry->sparse_chapters_per_volume) { - uds_log_error("Sparse chapter count (%u) does not match (%u)", + vdo_log_error("Sparse chapter count (%u) does not match (%u)", saved_geometry->sparse_chapters_per_volume, geometry->sparse_chapters_per_volume); result = false; } if (saved_config->cache_chapters != user->cache_chapters) { - uds_log_error("Cache size (%u) does not match (%u)", + vdo_log_error("Cache size (%u) does not match (%u)", saved_config->cache_chapters, user->cache_chapters); result = false; } if (saved_config->volume_index_mean_delta != user->volume_index_mean_delta) { - uds_log_error("Volume index mean delta (%u) does not match (%u)", + vdo_log_error("Volume index mean delta (%u) does not match (%u)", saved_config->volume_index_mean_delta, user->volume_index_mean_delta); result = false; } if (saved_geometry->bytes_per_page != geometry->bytes_per_page) { - uds_log_error("Bytes per page value (%zu) does not match (%zu)", + vdo_log_error("Bytes per page value (%zu) does not match (%zu)", saved_geometry->bytes_per_page, geometry->bytes_per_page); result = false; } if (saved_config->sparse_sample_rate != user->sparse_sample_rate) { - uds_log_error("Sparse sample rate (%u) does not match (%u)", + vdo_log_error("Sparse sample rate (%u) does not match (%u)", saved_config->sparse_sample_rate, user->sparse_sample_rate); result = false; } if (saved_config->nonce != user->nonce) { - uds_log_error("Nonce (%llu) does not match (%llu)", + vdo_log_error("Nonce (%llu) does not match (%llu)", (unsigned long long) saved_config->nonce, (unsigned long long) user->nonce); result = false; @@ -109,11 +109,11 @@ int uds_validate_config_contents(struct buffered_reader *reader, result = uds_read_from_buffered_reader(reader, version_buffer, INDEX_CONFIG_VERSION_LENGTH); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "cannot read index config version"); + return vdo_log_error_strerror(result, "cannot read index config version"); if (!is_version(INDEX_CONFIG_VERSION_6_02, version_buffer) && !is_version(INDEX_CONFIG_VERSION_8_02, version_buffer)) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "unsupported configuration version: '%.*s'", INDEX_CONFIG_VERSION_LENGTH, version_buffer); @@ -121,7 +121,7 @@ int uds_validate_config_contents(struct buffered_reader *reader, result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "cannot read config data"); + return vdo_log_error_strerror(result, "cannot read config data"); decode_u32_le(buffer, &offset, &geometry.record_pages_per_chapter); decode_u32_le(buffer, &offset, &geometry.chapters_per_volume); @@ -149,7 +149,7 @@ int uds_validate_config_contents(struct buffered_reader *reader, result = uds_read_from_buffered_reader(reader, remapping, sizeof(remapping)); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "cannot read converted config"); + return vdo_log_error_strerror(result, "cannot read converted config"); offset = 0; decode_u64_le(remapping, &offset, @@ -159,7 +159,7 @@ int uds_validate_config_contents(struct buffered_reader *reader, } if (!are_matching_configurations(&config, &geometry, user_config)) { - uds_log_warning("Supplied configuration does not match save"); + vdo_log_warning("Supplied configuration does not match save"); return UDS_NO_INDEX; } @@ -263,7 +263,7 @@ static int compute_memory_sizes(uds_memory_config_size_t mem_gb, bool sparse, DEFAULT_CHAPTERS_PER_VOLUME); *record_pages_per_chapter = DEFAULT_RECORD_PAGES_PER_CHAPTER; } else { - uds_log_error("received invalid memory size"); + vdo_log_error("received invalid memory size"); return -EINVAL; } @@ -292,7 +292,7 @@ static unsigned int __must_check normalize_zone_count(unsigned int requested) if (zone_count > MAX_ZONES) zone_count = MAX_ZONES; - uds_log_info("Using %u indexing zone%s for concurrency.", + vdo_log_info("Using %u indexing zone%s for concurrency.", zone_count, zone_count == 1 ? "" : "s"); return zone_count; } @@ -364,13 +364,13 @@ void uds_log_configuration(struct uds_configuration *config) { struct index_geometry *geometry = config->geometry; - uds_log_debug("Configuration:"); - uds_log_debug(" Record pages per chapter: %10u", geometry->record_pages_per_chapter); - uds_log_debug(" Chapters per volume: %10u", geometry->chapters_per_volume); - uds_log_debug(" Sparse chapters per volume: %10u", geometry->sparse_chapters_per_volume); - uds_log_debug(" Cache size (chapters): %10u", config->cache_chapters); - uds_log_debug(" Volume index mean delta: %10u", config->volume_index_mean_delta); - uds_log_debug(" Bytes per page: %10zu", geometry->bytes_per_page); - uds_log_debug(" Sparse sample rate: %10u", config->sparse_sample_rate); - uds_log_debug(" Nonce: %llu", (unsigned long long) config->nonce); + vdo_log_debug("Configuration:"); + vdo_log_debug(" Record pages per chapter: %10u", geometry->record_pages_per_chapter); + vdo_log_debug(" Chapters per volume: %10u", geometry->chapters_per_volume); + vdo_log_debug(" Sparse chapters per volume: %10u", geometry->sparse_chapters_per_volume); + vdo_log_debug(" Cache size (chapters): %10u", config->cache_chapters); + vdo_log_debug(" Volume index mean delta: %10u", config->volume_index_mean_delta); + vdo_log_debug(" Bytes per page: %10zu", geometry->bytes_per_page); + vdo_log_debug(" Sparse sample rate: %10u", config->sparse_sample_rate); + vdo_log_debug(" Nonce: %llu", (unsigned long long) config->nonce); } diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c index b49066554248..0ac2443f0df3 100644 --- a/drivers/md/dm-vdo/indexer/delta-index.c +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -375,7 +375,7 @@ int uds_initialize_delta_index(struct delta_index *delta_index, unsigned int zon */ if (delta_index->list_count <= first_list_in_zone) { uds_uninitialize_delta_index(delta_index); - return uds_log_error_strerror(UDS_INVALID_ARGUMENT, + return vdo_log_error_strerror(UDS_INVALID_ARGUMENT, "%u delta lists not enough for %u zones", list_count, zone_count); } @@ -732,7 +732,7 @@ int uds_pack_delta_index_page(const struct delta_index *delta_index, u64 header_ free_bits -= GUARD_BITS; if (free_bits < IMMUTABLE_HEADER_SIZE) { /* This page is too small to store any delta lists. */ - return uds_log_error_strerror(UDS_OVERFLOW, + return vdo_log_error_strerror(UDS_OVERFLOW, "Chapter Index Page of %zu bytes is too small", memory_size); } @@ -843,7 +843,7 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index, result = uds_read_from_buffered_reader(buffered_readers[z], buffer, sizeof(buffer)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read delta index header"); } @@ -860,23 +860,23 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index, "%zu bytes decoded of %zu expected", offset, sizeof(struct delta_index_header)); if (result != VDO_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read delta index header"); } if (memcmp(header.magic, DELTA_INDEX_MAGIC, MAGIC_SIZE) != 0) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "delta index file has bad magic number"); } if (zone_count != header.zone_count) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "delta index files contain mismatched zone counts (%u,%u)", zone_count, header.zone_count); } if (header.zone_number != z) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "delta index zone %u found in slot %u", header.zone_number, z); } @@ -887,7 +887,7 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index, collision_count += header.collision_count; if (first_list[z] != list_next) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "delta index file for zone %u starts with list %u instead of list %u", z, first_list[z], list_next); } @@ -896,13 +896,13 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index, } if (list_next != delta_index->list_count) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "delta index files contain %u delta lists instead of %u delta lists", list_next, delta_index->list_count); } if (collision_count > record_count) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "delta index files contain %llu collisions and %llu records", (unsigned long long) collision_count, (unsigned long long) record_count); @@ -927,7 +927,7 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index, size_data, sizeof(size_data)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read delta index size"); } @@ -960,7 +960,7 @@ static int restore_delta_list_to_zone(struct delta_zone *delta_zone, u32 list_number = save_info->index - delta_zone->first_list; if (list_number >= delta_zone->list_count) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "invalid delta list number %u not in range [%u,%u)", save_info->index, delta_zone->first_list, delta_zone->first_list + delta_zone->list_count); @@ -968,7 +968,7 @@ static int restore_delta_list_to_zone(struct delta_zone *delta_zone, delta_list = &delta_zone->delta_lists[list_number + 1]; if (delta_list->size == 0) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "unexpected delta list number %u", save_info->index); } @@ -976,7 +976,7 @@ static int restore_delta_list_to_zone(struct delta_zone *delta_zone, bit_count = delta_list->size + save_info->bit_offset; byte_count = BITS_TO_BYTES(bit_count); if (save_info->byte_count != byte_count) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "unexpected delta list size %u != %u", save_info->byte_count, byte_count); } @@ -996,7 +996,7 @@ static int restore_delta_list_data(struct delta_index *delta_index, unsigned int result = uds_read_from_buffered_reader(buffered_reader, buffer, sizeof(buffer)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read delta list data"); } @@ -1009,7 +1009,7 @@ static int restore_delta_list_data(struct delta_index *delta_index, unsigned int if ((save_info.bit_offset >= BITS_PER_BYTE) || (save_info.byte_count > DELTA_LIST_MAX_BYTE_COUNT)) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "corrupt delta list data"); } @@ -1018,7 +1018,7 @@ static int restore_delta_list_data(struct delta_index *delta_index, unsigned int return UDS_CORRUPT_DATA; if (save_info.index >= delta_index->list_count) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "invalid delta list number %u of %u", save_info.index, delta_index->list_count); @@ -1027,7 +1027,7 @@ static int restore_delta_list_data(struct delta_index *delta_index, unsigned int result = uds_read_from_buffered_reader(buffered_reader, data, save_info.byte_count); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read delta list data"); } @@ -1102,7 +1102,7 @@ static int flush_delta_list(struct delta_zone *zone, u32 flush_index) result = uds_write_to_buffered_writer(zone->buffered_writer, buffer, sizeof(buffer)); if (result != UDS_SUCCESS) { - uds_log_warning_strerror(result, "failed to write delta list memory"); + vdo_log_warning_strerror(result, "failed to write delta list memory"); return result; } @@ -1110,7 +1110,7 @@ static int flush_delta_list(struct delta_zone *zone, u32 flush_index) zone->memory + get_delta_list_byte_start(delta_list), get_delta_list_byte_size(delta_list)); if (result != UDS_SUCCESS) - uds_log_warning_strerror(result, "failed to write delta list memory"); + vdo_log_warning_strerror(result, "failed to write delta list memory"); return result; } @@ -1144,7 +1144,7 @@ int uds_start_saving_delta_index(const struct delta_index *delta_index, result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); if (result != UDS_SUCCESS) - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to write delta index header"); for (i = 0; i < delta_zone->list_count; i++) { @@ -1156,7 +1156,7 @@ int uds_start_saving_delta_index(const struct delta_index *delta_index, result = uds_write_to_buffered_writer(buffered_writer, data, sizeof(data)); if (result != UDS_SUCCESS) - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to write delta list size"); } @@ -1197,7 +1197,7 @@ int uds_write_guard_delta_list(struct buffered_writer *buffered_writer) result = uds_write_to_buffered_writer(buffered_writer, buffer, sizeof(buffer)); if (result != UDS_SUCCESS) - uds_log_warning_strerror(result, "failed to write guard delta list"); + vdo_log_warning_strerror(result, "failed to write guard delta list"); return UDS_SUCCESS; } @@ -1378,7 +1378,7 @@ noinline int uds_next_delta_index_entry(struct delta_index_entry *delta_entry) * This is not an assertion because uds_validate_chapter_index_page() wants to * handle this error. */ - uds_log_warning("Decoded past the end of the delta list"); + vdo_log_warning("Decoded past the end of the delta list"); return UDS_CORRUPT_DATA; } @@ -1959,7 +1959,7 @@ u32 uds_get_delta_index_page_count(u32 entry_count, u32 list_count, u32 mean_del void uds_log_delta_index_entry(struct delta_index_entry *delta_entry) { - uds_log_ratelimit(uds_log_info, + vdo_log_ratelimit(vdo_log_info, "List 0x%X Key 0x%X Offset 0x%X%s%s List_size 0x%X%s", delta_entry->list_number, delta_entry->key, delta_entry->offset, delta_entry->at_end ? " end" : "", diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index 74fd44c20e5c..627adc24af3b 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -231,7 +231,7 @@ static int __must_check compute_sizes(const struct uds_configuration *config, result = uds_compute_volume_index_save_blocks(config, sls->block_size, &sls->volume_index_blocks); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "cannot compute index save size"); + return vdo_log_error_strerror(result, "cannot compute index save size"); sls->page_map_blocks = DIV_ROUND_UP(uds_compute_index_page_map_save_size(geometry), @@ -255,13 +255,13 @@ int uds_compute_index_size(const struct uds_parameters *parameters, u64 *index_s struct save_layout_sizes sizes; if (index_size == NULL) { - uds_log_error("Missing output size pointer"); + vdo_log_error("Missing output size pointer"); return -EINVAL; } result = uds_make_configuration(parameters, &index_config); if (result != UDS_SUCCESS) { - uds_log_error_strerror(result, "cannot compute index size"); + vdo_log_error_strerror(result, "cannot compute index size"); return uds_status_to_errno(result); } @@ -648,7 +648,7 @@ static int discard_index_state_data(struct index_layout *layout) } if (saved_result != UDS_SUCCESS) { - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "%s: cannot destroy all index saves", __func__); } @@ -755,18 +755,18 @@ static int __must_check write_uds_index_config(struct index_layout *layout, result = open_layout_writer(layout, &layout->config, offset, &writer); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "failed to open config region"); + return vdo_log_error_strerror(result, "failed to open config region"); result = uds_write_config_contents(writer, config, layout->super.version); if (result != UDS_SUCCESS) { uds_free_buffered_writer(writer); - return uds_log_error_strerror(result, "failed to write config region"); + return vdo_log_error_strerror(result, "failed to write config region"); } result = uds_flush_buffered_writer(writer); if (result != UDS_SUCCESS) { uds_free_buffered_writer(writer); - return uds_log_error_strerror(result, "cannot flush config writer"); + return vdo_log_error_strerror(result, "cannot flush config writer"); } uds_free_buffered_writer(writer); @@ -873,7 +873,7 @@ static int find_latest_uds_index_save_slot(struct index_layout *layout, } if (latest == NULL) { - uds_log_error("No valid index save found"); + vdo_log_error("No valid index save found"); return UDS_INDEX_NOT_SAVED_CLEANLY; } @@ -1145,7 +1145,7 @@ static int __must_check load_region_table(struct buffered_reader *reader, result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "cannot read region table header"); + return vdo_log_error_strerror(result, "cannot read region table header"); decode_u64_le(buffer, &offset, &header.magic); decode_u64_le(buffer, &offset, &header.region_blocks); @@ -1158,7 +1158,7 @@ static int __must_check load_region_table(struct buffered_reader *reader, return UDS_NO_INDEX; if (header.version != 1) { - return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, "unknown region table version %hu", header.version); } @@ -1178,7 +1178,7 @@ static int __must_check load_region_table(struct buffered_reader *reader, sizeof(region_buffer)); if (result != UDS_SUCCESS) { vdo_free(table); - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "cannot read region table layouts"); } @@ -1209,7 +1209,7 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, result = uds_read_from_buffered_reader(reader, buffer, saved_size); if (result != UDS_SUCCESS) { vdo_free(buffer); - return uds_log_error_strerror(result, "cannot read region table header"); + return vdo_log_error_strerror(result, "cannot read region table header"); } memcpy(&super->magic_label, buffer, MAGIC_SIZE); @@ -1236,19 +1236,19 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, vdo_free(buffer); if (memcmp(super->magic_label, LAYOUT_MAGIC, MAGIC_SIZE) != 0) - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "unknown superblock magic label"); if ((super->version < SUPER_VERSION_MINIMUM) || (super->version == 4) || (super->version == 5) || (super->version == 6) || (super->version > SUPER_VERSION_MAXIMUM)) { - return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, "unknown superblock version number %u", super->version); } if (super->volume_offset < super->start_offset) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "inconsistent offsets (start %llu, volume %llu)", (unsigned long long) super->start_offset, (unsigned long long) super->volume_offset); @@ -1256,13 +1256,13 @@ static int __must_check read_super_block_data(struct buffered_reader *reader, /* Sub-indexes are no longer used but the layout retains this field. */ if (super->index_count != 1) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "invalid subindex count %u", super->index_count); } if (generate_primary_nonce(super->nonce_info, sizeof(super->nonce_info)) != super->nonce) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "inconsistent superblock nonce"); } @@ -1273,15 +1273,15 @@ static int __must_check verify_region(struct layout_region *lr, u64 start_block, enum region_kind kind, unsigned int instance) { if (lr->start_block != start_block) - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "incorrect layout region offset"); if (lr->kind != kind) - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "incorrect layout region kind"); if (lr->instance != instance) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "incorrect layout region instance"); } @@ -1323,7 +1323,7 @@ static int __must_check verify_sub_index(struct index_layout *layout, u64 start_ next_block -= layout->super.volume_offset; if (next_block != start_block + sil->sub_index.block_count) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "sub index region does not span all saves"); } @@ -1368,7 +1368,7 @@ static int __must_check reconstitute_layout(struct index_layout *layout, return result; if (++next_block != (first_block + layout->total_blocks)) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "layout table does not span total blocks"); } @@ -1388,19 +1388,19 @@ static int __must_check load_super_block(struct index_layout *layout, size_t blo if (table->header.type != RH_TYPE_SUPER) { vdo_free(table); - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "not a superblock region table"); } result = read_super_block_data(reader, layout, table->header.payload); if (result != UDS_SUCCESS) { vdo_free(table); - return uds_log_error_strerror(result, "unknown superblock format"); + return vdo_log_error_strerror(result, "unknown superblock format"); } if (super->block_size != block_size) { vdo_free(table); - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "superblock saved block_size %u differs from supplied block_size %zu", super->block_size, block_size); } @@ -1421,14 +1421,14 @@ static int __must_check read_index_save_data(struct buffered_reader *reader, size_t offset = 0; if (saved_size != sizeof(buffer)) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "unexpected index save data size %zu", saved_size); } result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "cannot read index save data"); + return vdo_log_error_strerror(result, "cannot read index save data"); decode_u64_le(buffer, &offset, &isl->save_data.timestamp); decode_u64_le(buffer, &offset, &isl->save_data.nonce); @@ -1436,7 +1436,7 @@ static int __must_check read_index_save_data(struct buffered_reader *reader, offset += sizeof(u32); if (isl->save_data.version > 1) { - return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, "unknown index save version number %u", isl->save_data.version); } @@ -1446,7 +1446,7 @@ static int __must_check read_index_save_data(struct buffered_reader *reader, if ((file_version.signature != INDEX_STATE_VERSION_301.signature) || (file_version.version_id != INDEX_STATE_VERSION_301.version_id)) { - return uds_log_error_strerror(UDS_UNSUPPORTED_VERSION, + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, "index state version %d,%d is unsupported", file_version.signature, file_version.version_id); @@ -1523,7 +1523,7 @@ static int __must_check reconstruct_index_save(struct index_save_layout *isl, next_block += isl->free_space.block_count; if (next_block != last_block) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "index save layout table incomplete"); } @@ -1539,7 +1539,7 @@ static int __must_check load_index_save(struct index_save_layout *isl, result = load_region_table(reader, &table); if (result != UDS_SUCCESS) { - return uds_log_error_strerror(result, "cannot read index save %u header", + return vdo_log_error_strerror(result, "cannot read index save %u header", instance); } @@ -1547,7 +1547,7 @@ static int __must_check load_index_save(struct index_save_layout *isl, u64 region_blocks = table->header.region_blocks; vdo_free(table); - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "unexpected index save %u region block count %llu", instance, (unsigned long long) region_blocks); @@ -1561,7 +1561,7 @@ static int __must_check load_index_save(struct index_save_layout *isl, if (table->header.type != RH_TYPE_SAVE) { - uds_log_error_strerror(UDS_CORRUPT_DATA, + vdo_log_error_strerror(UDS_CORRUPT_DATA, "unexpected index save %u header type %u", instance, table->header.type); vdo_free(table); @@ -1571,7 +1571,7 @@ static int __must_check load_index_save(struct index_save_layout *isl, result = read_index_save_data(reader, isl, table->header.payload); if (result != UDS_SUCCESS) { vdo_free(table); - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "unknown index save %u data format", instance); } @@ -1579,7 +1579,7 @@ static int __must_check load_index_save(struct index_save_layout *isl, result = reconstruct_index_save(isl, table); vdo_free(table); if (result != UDS_SUCCESS) { - return uds_log_error_strerror(result, "cannot reconstruct index save %u", + return vdo_log_error_strerror(result, "cannot reconstruct index save %u", instance); } @@ -1598,7 +1598,7 @@ static int __must_check load_sub_index_regions(struct index_layout *layout) result = open_region_reader(layout, &isl->index_save, &reader); if (result != UDS_SUCCESS) { - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "cannot get reader for index 0 save %u", j); return result; @@ -1626,12 +1626,12 @@ static int __must_check verify_uds_index_config(struct index_layout *layout, offset = layout->super.volume_offset - layout->super.start_offset; result = open_layout_reader(layout, &layout->config, offset, &reader); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "failed to open config reader"); + return vdo_log_error_strerror(result, "failed to open config reader"); result = uds_validate_config_contents(reader, config); if (result != UDS_SUCCESS) { uds_free_buffered_reader(reader); - return uds_log_error_strerror(result, "failed to read config region"); + return vdo_log_error_strerror(result, "failed to read config region"); } uds_free_buffered_reader(reader); @@ -1646,7 +1646,7 @@ static int load_index_layout(struct index_layout *layout, struct uds_configurati result = uds_make_buffered_reader(layout->factory, layout->offset / UDS_BLOCK_SIZE, 1, &reader); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, "unable to read superblock"); + return vdo_log_error_strerror(result, "unable to read superblock"); result = load_super_block(layout, UDS_BLOCK_SIZE, layout->offset / UDS_BLOCK_SIZE, reader); @@ -1675,7 +1675,7 @@ static int create_layout_factory(struct index_layout *layout, writable_size = uds_get_writable_size(factory) & -UDS_BLOCK_SIZE; if (writable_size < config->size + config->offset) { uds_put_io_factory(factory); - uds_log_error("index storage (%zu) is smaller than the requested size %zu", + vdo_log_error("index storage (%zu) is smaller than the requested size %zu", writable_size, config->size + config->offset); return -ENOSPC; } @@ -1708,7 +1708,7 @@ int uds_make_index_layout(struct uds_configuration *config, bool new_layout, } if (layout->factory_size < sizes.total_size) { - uds_log_error("index storage (%zu) is smaller than the required size %llu", + vdo_log_error("index storage (%zu) is smaller than the required size %llu", layout->factory_size, (unsigned long long) sizes.total_size); uds_free_index_layout(layout); diff --git a/drivers/md/dm-vdo/indexer/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c index c5d1b9995846..00b44e07d0c1 100644 --- a/drivers/md/dm-vdo/indexer/index-page-map.c +++ b/drivers/md/dm-vdo/indexer/index-page-map.c @@ -167,7 +167,7 @@ int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader * decode_u16_le(buffer, &offset, &map->entries[i]); vdo_free(buffer); - uds_log_debug("read index page map, last update %llu", + vdo_log_debug("read index page map, last update %llu", (unsigned long long) map->last_update); return UDS_SUCCESS; } diff --git a/drivers/md/dm-vdo/indexer/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c index 9eae00548095..aee0914d604a 100644 --- a/drivers/md/dm-vdo/indexer/index-session.c +++ b/drivers/md/dm-vdo/indexer/index-session.c @@ -104,7 +104,7 @@ int uds_launch_request(struct uds_request *request) int result; if (request->callback == NULL) { - uds_log_error("missing required callback"); + vdo_log_error("missing required callback"); return -EINVAL; } @@ -116,7 +116,7 @@ int uds_launch_request(struct uds_request *request) case UDS_UPDATE: break; default: - uds_log_error("received invalid callback type"); + vdo_log_error("received invalid callback type"); return -EINVAL; } @@ -244,7 +244,7 @@ static int __must_check make_empty_index_session(struct uds_index_session **inde int uds_create_index_session(struct uds_index_session **session) { if (session == NULL) { - uds_log_error("missing session pointer"); + vdo_log_error("missing session pointer"); return -EINVAL; } @@ -257,10 +257,10 @@ static int __must_check start_loading_index_session(struct uds_index_session *in mutex_lock(&index_session->request_mutex); if (index_session->state & IS_FLAG_SUSPENDED) { - uds_log_info("Index session is suspended"); + vdo_log_info("Index session is suspended"); result = -EBUSY; } else if (index_session->state != 0) { - uds_log_info("Index is already loaded"); + vdo_log_info("Index is already loaded"); result = -EBUSY; } else { index_session->state |= IS_FLAG_LOADING; @@ -290,7 +290,7 @@ static int initialize_index_session(struct uds_index_session *index_session, result = uds_make_configuration(&index_session->parameters, &config); if (result != UDS_SUCCESS) { - uds_log_error_strerror(result, "Failed to allocate config"); + vdo_log_error_strerror(result, "Failed to allocate config"); return result; } @@ -298,7 +298,7 @@ static int initialize_index_session(struct uds_index_session *index_session, result = uds_make_index(config, open_type, &index_session->load_context, enter_callback_stage, &index_session->index); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "Failed to make index"); + vdo_log_error_strerror(result, "Failed to make index"); else uds_log_configuration(config); @@ -332,15 +332,15 @@ int uds_open_index(enum uds_open_index_type open_type, char name[BDEVNAME_SIZE]; if (parameters == NULL) { - uds_log_error("missing required parameters"); + vdo_log_error("missing required parameters"); return -EINVAL; } if (parameters->bdev == NULL) { - uds_log_error("missing required block device"); + vdo_log_error("missing required block device"); return -EINVAL; } if (session == NULL) { - uds_log_error("missing required session pointer"); + vdo_log_error("missing required session pointer"); return -EINVAL; } @@ -350,11 +350,11 @@ int uds_open_index(enum uds_open_index_type open_type, session->parameters = *parameters; format_dev_t(name, parameters->bdev->bd_dev); - uds_log_info("%s: %s", get_open_type_string(open_type), name); + vdo_log_info("%s: %s", get_open_type_string(open_type), name); result = initialize_index_session(session, open_type); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "Failed %s", + vdo_log_error_strerror(result, "Failed %s", get_open_type_string(open_type)); finish_loading_index_session(session, result); @@ -426,7 +426,7 @@ int uds_suspend_index_session(struct uds_index_session *session, bool save) if ((session->state & IS_FLAG_WAITING) || (session->state & IS_FLAG_DESTROYING)) { no_work = true; - uds_log_info("Index session is already changing state"); + vdo_log_info("Index session is already changing state"); result = -EBUSY; } else if (session->state & IS_FLAG_SUSPENDED) { no_work = true; @@ -485,7 +485,7 @@ int uds_resume_index_session(struct uds_index_session *session, mutex_lock(&session->request_mutex); if (session->state & IS_FLAG_WAITING) { - uds_log_info("Index session is already changing state"); + vdo_log_info("Index session is already changing state"); no_work = true; result = -EBUSY; } else if (!(session->state & IS_FLAG_SUSPENDED)) { @@ -562,7 +562,7 @@ static int save_and_free_index(struct uds_index_session *index_session) if (!suspended) { result = uds_save_index(index); if (result != UDS_SUCCESS) - uds_log_warning_strerror(result, + vdo_log_warning_strerror(result, "ignoring error from save_index"); } uds_free_index(index); @@ -598,7 +598,7 @@ int uds_close_index(struct uds_index_session *index_session) } if (index_session->state & IS_FLAG_SUSPENDED) { - uds_log_info("Index session is suspended"); + vdo_log_info("Index session is suspended"); result = -EBUSY; } else if ((index_session->state & IS_FLAG_DESTROYING) || !(index_session->state & IS_FLAG_LOADED)) { @@ -611,10 +611,10 @@ int uds_close_index(struct uds_index_session *index_session) if (result != UDS_SUCCESS) return uds_status_to_errno(result); - uds_log_debug("Closing index"); + vdo_log_debug("Closing index"); wait_for_no_requests_in_progress(index_session); result = save_and_free_index(index_session); - uds_log_debug("Closed index"); + vdo_log_debug("Closed index"); mutex_lock(&index_session->request_mutex); index_session->state &= ~IS_FLAG_CLOSING; @@ -629,7 +629,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) int result; bool load_pending = false; - uds_log_debug("Destroying index session"); + vdo_log_debug("Destroying index session"); /* Wait for any current index state change to complete. */ mutex_lock(&index_session->request_mutex); @@ -641,7 +641,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) if (index_session->state & IS_FLAG_DESTROYING) { mutex_unlock(&index_session->request_mutex); - uds_log_info("Index session is already closing"); + vdo_log_info("Index session is already closing"); return -EBUSY; } @@ -672,7 +672,7 @@ int uds_destroy_index_session(struct uds_index_session *index_session) result = save_and_free_index(index_session); uds_request_queue_finish(index_session->callback_queue); index_session->callback_queue = NULL; - uds_log_debug("Destroyed index session"); + vdo_log_debug("Destroyed index session"); vdo_free(index_session); return uds_status_to_errno(result); } @@ -710,7 +710,7 @@ int uds_get_index_session_stats(struct uds_index_session *index_session, struct uds_index_stats *stats) { if (stats == NULL) { - uds_log_error("received a NULL index stats pointer"); + vdo_log_error("received a NULL index stats pointer"); return -EINVAL; } diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c index 221af95ca2a4..1ba767144426 100644 --- a/drivers/md/dm-vdo/indexer/index.c +++ b/drivers/md/dm-vdo/indexer/index.c @@ -188,7 +188,7 @@ static int finish_previous_chapter(struct uds_index *index, u64 current_chapter_ mutex_unlock(&writer->mutex); if (result != UDS_SUCCESS) - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "Writing of previous open chapter failed"); return UDS_SUCCESS; @@ -258,7 +258,7 @@ static int open_next_chapter(struct index_zone *zone) unsigned int finished_zones; u32 expire_chapters; - uds_log_debug("closing chapter %llu of zone %u after %u entries (%u short)", + vdo_log_debug("closing chapter %llu of zone %u after %u entries (%u short)", (unsigned long long) zone->newest_virtual_chapter, zone->id, zone->open_chapter->size, zone->open_chapter->capacity - zone->open_chapter->size); @@ -315,7 +315,7 @@ static int dispatch_index_zone_control_request(struct uds_request *request) return handle_chapter_closed(zone, message->virtual_chapter); default: - uds_log_error("invalid message type: %d", message->type); + vdo_log_error("invalid message type: %d", message->type); return UDS_INVALID_ARGUMENT; } } @@ -600,7 +600,7 @@ static int dispatch_index_request(struct uds_index *index, struct uds_request *r break; default: - result = uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + result = vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, "invalid request type: %d", request->type); break; @@ -618,7 +618,7 @@ static void execute_zone_request(struct uds_request *request) if (request->zone_message.type != UDS_MESSAGE_NONE) { result = dispatch_index_zone_control_request(request); if (result != UDS_SUCCESS) { - uds_log_error_strerror(result, "error executing message: %d", + vdo_log_error_strerror(result, "error executing message: %d", request->zone_message.type); } @@ -678,7 +678,7 @@ static void close_chapters(void *arg) struct chapter_writer *writer = arg; struct uds_index *index = writer->index; - uds_log_debug("chapter writer starting"); + vdo_log_debug("chapter writer starting"); mutex_lock(&writer->mutex); for (;;) { while (writer->zones_to_write < index->zone_count) { @@ -688,7 +688,7 @@ static void close_chapters(void *arg) * open chapter, so we can exit now. */ mutex_unlock(&writer->mutex); - uds_log_debug("chapter writer stopping"); + vdo_log_debug("chapter writer stopping"); return; } uds_wait_cond(&writer->cond, &writer->mutex); @@ -711,7 +711,7 @@ static void close_chapters(void *arg) index->has_saved_open_chapter = false; result = uds_discard_open_chapter(index->layout); if (result == UDS_SUCCESS) - uds_log_debug("Discarding saved open chapter"); + vdo_log_debug("Discarding saved open chapter"); } result = uds_close_open_chapter(writer->chapters, index->zone_count, @@ -818,7 +818,7 @@ static int load_index(struct uds_index *index) last_save_chapter = ((index->last_save != NO_LAST_SAVE) ? index->last_save : 0); - uds_log_info("loaded index from chapter %llu through chapter %llu", + vdo_log_info("loaded index from chapter %llu through chapter %llu", (unsigned long long) index->oldest_virtual_chapter, (unsigned long long) last_save_chapter); @@ -843,7 +843,7 @@ static int rebuild_index_page_map(struct uds_index *index, u64 vcn) index_page_number, &chapter_index_page); if (result != UDS_SUCCESS) { - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "failed to read index page %u in chapter %u", index_page_number, chapter); } @@ -851,7 +851,7 @@ static int rebuild_index_page_map(struct uds_index *index, u64 vcn) lowest_delta_list = chapter_index_page->lowest_list_number; highest_delta_list = chapter_index_page->highest_list_number; if (lowest_delta_list != expected_list_number) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "chapter %u index page %u is corrupt", chapter, index_page_number); } @@ -980,7 +980,7 @@ static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse) u32 physical_chapter; if (check_for_suspend(index)) { - uds_log_info("Replay interrupted by index shutdown at chapter %llu", + vdo_log_info("Replay interrupted by index shutdown at chapter %llu", (unsigned long long) virtual); return -EBUSY; } @@ -992,7 +992,7 @@ static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse) result = rebuild_index_page_map(index, virtual); if (result != UDS_SUCCESS) { - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "could not rebuild index page map for chapter %u", physical_chapter); } @@ -1005,7 +1005,7 @@ static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse) result = uds_get_volume_record_page(index->volume, physical_chapter, record_page_number, &record_page); if (result != UDS_SUCCESS) { - return uds_log_error_strerror(result, "could not get page %d", + return vdo_log_error_strerror(result, "could not get page %d", record_page_number); } @@ -1034,7 +1034,7 @@ static int replay_volume(struct uds_index *index) u64 upto_virtual = index->newest_virtual_chapter; bool will_be_sparse; - uds_log_info("Replaying volume from chapter %llu through chapter %llu", + vdo_log_info("Replaying volume from chapter %llu through chapter %llu", (unsigned long long) from_virtual, (unsigned long long) upto_virtual); @@ -1064,7 +1064,7 @@ static int replay_volume(struct uds_index *index) new_map_update = index->volume->index_page_map->last_update; if (new_map_update != old_map_update) { - uds_log_info("replay changed index page map update from %llu to %llu", + vdo_log_info("replay changed index page map update from %llu to %llu", (unsigned long long) old_map_update, (unsigned long long) new_map_update); } @@ -1084,7 +1084,7 @@ static int rebuild_index(struct uds_index *index) result = uds_find_volume_chapter_boundaries(index->volume, &lowest, &highest, &is_empty); if (result != UDS_SUCCESS) { - return uds_log_fatal_strerror(result, + return vdo_log_fatal_strerror(result, "cannot rebuild index: unknown volume chapter boundaries"); } @@ -1194,7 +1194,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op result = make_index_zone(index, z); if (result != UDS_SUCCESS) { uds_free_index(index); - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "Could not create index zone"); } } @@ -1203,7 +1203,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op result = uds_make_volume_index(config, nonce, &index->volume_index); if (result != UDS_SUCCESS) { uds_free_index(index); - return uds_log_error_strerror(result, "could not make volume index"); + return vdo_log_error_strerror(result, "could not make volume index"); } index->load_context = load_context; @@ -1229,14 +1229,14 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op break; case -ENOMEM: /* We should not try a rebuild for this error. */ - uds_log_error_strerror(result, "index could not be loaded"); + vdo_log_error_strerror(result, "index could not be loaded"); break; default: - uds_log_error_strerror(result, "index could not be loaded"); + vdo_log_error_strerror(result, "index could not be loaded"); if (open_type == UDS_LOAD) { result = rebuild_index(index); if (result != UDS_SUCCESS) { - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "index could not be rebuilt"); } } @@ -1246,7 +1246,7 @@ int uds_make_index(struct uds_configuration *config, enum uds_open_index_type op if (result != UDS_SUCCESS) { uds_free_index(index); - return uds_log_error_strerror(result, "fatal error in %s()", __func__); + return vdo_log_error_strerror(result, "fatal error in %s()", __func__); } for (z = 0; z < index->zone_count; z++) { @@ -1320,16 +1320,16 @@ int uds_save_index(struct uds_index *index) index->prev_save = index->last_save; index->last_save = ((index->newest_virtual_chapter == 0) ? NO_LAST_SAVE : index->newest_virtual_chapter - 1); - uds_log_info("beginning save (vcn %llu)", (unsigned long long) index->last_save); + vdo_log_info("beginning save (vcn %llu)", (unsigned long long) index->last_save); result = uds_save_index_state(index->layout, index); if (result != UDS_SUCCESS) { - uds_log_info("save index failed"); + vdo_log_info("save index failed"); index->last_save = index->prev_save; } else { index->has_saved_open_chapter = true; index->need_to_save = false; - uds_log_info("finished save (vcn %llu)", + vdo_log_info("finished save (vcn %llu)", (unsigned long long) index->last_save); } diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c index 0dcf6d596653..515765d35794 100644 --- a/drivers/md/dm-vdo/indexer/io-factory.c +++ b/drivers/md/dm-vdo/indexer/io-factory.c @@ -365,7 +365,7 @@ void uds_free_buffered_writer(struct buffered_writer *writer) flush_previous_buffer(writer); result = -dm_bufio_write_dirty_buffers(writer->client); if (result != UDS_SUCCESS) - uds_log_warning_strerror(result, "%s: failed to sync storage", __func__); + vdo_log_warning_strerror(result, "%s: failed to sync storage", __func__); dm_bufio_client_destroy(writer->client); uds_put_io_factory(writer->factory); diff --git a/drivers/md/dm-vdo/indexer/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c index 46b7bc1ac324..4a67bcadaae0 100644 --- a/drivers/md/dm-vdo/indexer/open-chapter.c +++ b/drivers/md/dm-vdo/indexer/open-chapter.c @@ -259,14 +259,14 @@ static int fill_delta_chapter_index(struct open_chapter_zone **chapter_zones, overflow_count++; break; default: - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "failed to build open chapter index"); return result; } } if (overflow_count > 0) - uds_log_warning("Failed to add %d entries to chapter index", + vdo_log_warning("Failed to add %d entries to chapter index", overflow_count); return UDS_SUCCESS; @@ -417,7 +417,7 @@ int uds_load_open_chapter(struct uds_index *index, struct buffered_reader *reade return result; if (memcmp(OPEN_CHAPTER_VERSION, version, sizeof(version)) != 0) { - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "Invalid open chapter version: %.*s", (int) sizeof(version), version); } diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c index e2b0600d82b9..12f954a0c532 100644 --- a/drivers/md/dm-vdo/indexer/volume-index.c +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -225,13 +225,13 @@ static int compute_volume_sub_index_parameters(const struct uds_configuration *c params->address_bits = bits_per(address_count - 1); params->chapter_bits = bits_per(rounded_chapters - 1); if ((u32) params->list_count != params->list_count) { - return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, "cannot initialize volume index with %llu delta lists", (unsigned long long) params->list_count); } if (params->address_bits > 31) { - return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, "cannot initialize volume index with %u address bits", params->address_bits); } @@ -568,7 +568,7 @@ int uds_put_volume_index_record(struct volume_index_record *record, u64 virtual_ u64 low = get_zone_for_record(record)->virtual_chapter_low; u64 high = get_zone_for_record(record)->virtual_chapter_high; - return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, "cannot put record into chapter number %llu that is out of the valid range %llu to %llu", (unsigned long long) virtual_chapter, (unsigned long long) low, @@ -590,7 +590,7 @@ int uds_put_volume_index_record(struct volume_index_record *record, u64 virtual_ record->is_found = true; break; case UDS_OVERFLOW: - uds_log_ratelimit(uds_log_warning_strerror, UDS_OVERFLOW, + vdo_log_ratelimit(vdo_log_warning_strerror, UDS_OVERFLOW, "Volume index entry dropped due to overflow condition"); uds_log_delta_index_entry(&record->delta_entry); break; @@ -606,7 +606,7 @@ int uds_remove_volume_index_record(struct volume_index_record *record) int result; if (!record->is_found) - return uds_log_warning_strerror(UDS_BAD_STATE, + return vdo_log_warning_strerror(UDS_BAD_STATE, "illegal operation on new record"); /* Mark the record so that it cannot be used again */ @@ -644,7 +644,7 @@ static void set_volume_sub_index_zone_open_chapter(struct volume_sub_index *sub_ 1 + (used_bits - sub_index->max_zone_bits) / sub_index->chapter_zone_bits; if (expire_count == 1) { - uds_log_ratelimit(uds_log_info, + vdo_log_ratelimit(vdo_log_info, "zone %u: At chapter %llu, expiring chapter %llu early", zone_number, (unsigned long long) virtual_chapter, @@ -662,7 +662,7 @@ static void set_volume_sub_index_zone_open_chapter(struct volume_sub_index *sub_ zone->virtual_chapter_high - zone->virtual_chapter_low; zone->virtual_chapter_low = zone->virtual_chapter_high; } - uds_log_ratelimit(uds_log_info, + vdo_log_ratelimit(vdo_log_info, "zone %u: At chapter %llu, expiring chapters %llu to %llu early", zone_number, (unsigned long long) virtual_chapter, @@ -713,14 +713,14 @@ int uds_set_volume_index_record_chapter(struct volume_index_record *record, int result; if (!record->is_found) - return uds_log_warning_strerror(UDS_BAD_STATE, + return vdo_log_warning_strerror(UDS_BAD_STATE, "illegal operation on new record"); if (!is_virtual_chapter_indexed(record, virtual_chapter)) { u64 low = get_zone_for_record(record)->virtual_chapter_low; u64 high = get_zone_for_record(record)->virtual_chapter_high; - return uds_log_warning_strerror(UDS_INVALID_ARGUMENT, + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, "cannot set chapter number %llu that is out of the valid range %llu to %llu", (unsigned long long) virtual_chapter, (unsigned long long) low, @@ -820,7 +820,7 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, result = uds_read_from_buffered_reader(readers[i], buffer, sizeof(buffer)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read volume index header"); } @@ -839,14 +839,14 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, result = UDS_CORRUPT_DATA; if (memcmp(header.magic, MAGIC_START_5, MAGIC_SIZE) != 0) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "volume index file had bad magic number"); } if (sub_index->volume_nonce == 0) { sub_index->volume_nonce = header.volume_nonce; } else if (header.volume_nonce != sub_index->volume_nonce) { - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "volume index volume nonce incorrect"); } @@ -857,7 +857,7 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, u64 low = header.virtual_chapter_low; u64 high = header.virtual_chapter_high; - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "Inconsistent volume index zone files: Chapter range is [%llu,%llu], chapter range %d is [%llu,%llu]", (unsigned long long) virtual_chapter_low, (unsigned long long) virtual_chapter_high, @@ -873,7 +873,7 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, result = uds_read_from_buffered_reader(readers[i], decoded, sizeof(u64)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read volume index flush ranges"); } @@ -891,7 +891,7 @@ static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, result = uds_start_restoring_delta_index(&sub_index->delta_index, readers, reader_count); if (result != UDS_SUCCESS) - return uds_log_warning_strerror(result, "restoring delta index failed"); + return vdo_log_warning_strerror(result, "restoring delta index failed"); return UDS_SUCCESS; } @@ -916,7 +916,7 @@ static int start_restoring_volume_index(struct volume_index *volume_index, result = uds_read_from_buffered_reader(buffered_readers[i], buffer, sizeof(buffer)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to read volume index header"); } @@ -931,13 +931,13 @@ static int start_restoring_volume_index(struct volume_index *volume_index, result = UDS_CORRUPT_DATA; if (memcmp(header.magic, MAGIC_START_6, MAGIC_SIZE) != 0) - return uds_log_warning_strerror(UDS_CORRUPT_DATA, + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, "volume index file had bad magic number"); if (i == 0) { volume_index->sparse_sample_rate = header.sparse_sample_rate; } else if (volume_index->sparse_sample_rate != header.sparse_sample_rate) { - uds_log_warning_strerror(UDS_CORRUPT_DATA, + vdo_log_warning_strerror(UDS_CORRUPT_DATA, "Inconsistent sparse sample rate in delta index zone files: %u vs. %u", volume_index->sparse_sample_rate, header.sparse_sample_rate); @@ -1031,7 +1031,7 @@ static int start_saving_volume_sub_index(const struct volume_sub_index *sub_inde result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); if (result != UDS_SUCCESS) - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to write volume index header"); for (i = 0; i < list_count; i++) { @@ -1041,7 +1041,7 @@ static int start_saving_volume_sub_index(const struct volume_sub_index *sub_inde result = uds_write_to_buffered_writer(buffered_writer, encoded, sizeof(u64)); if (result != UDS_SUCCESS) { - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to write volume index flush ranges"); } } @@ -1074,7 +1074,7 @@ static int start_saving_volume_index(const struct volume_index *volume_index, result = uds_write_to_buffered_writer(writer, buffer, offset); if (result != UDS_SUCCESS) { - uds_log_warning_strerror(result, "failed to write volume index header"); + vdo_log_warning_strerror(result, "failed to write volume index header"); return result; } @@ -1264,7 +1264,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non &volume_index->vi_non_hook); if (result != UDS_SUCCESS) { uds_free_volume_index(volume_index); - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "Error creating non hook volume index"); } @@ -1272,7 +1272,7 @@ int uds_make_volume_index(const struct uds_configuration *config, u64 volume_non &volume_index->vi_hook); if (result != UDS_SUCCESS) { uds_free_volume_index(volume_index); - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "Error creating hook volume index"); } diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c index 701f2220d803..655453bb276b 100644 --- a/drivers/md/dm-vdo/indexer/volume.c +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -357,7 +357,7 @@ static void enqueue_page_read(struct volume *volume, struct uds_request *request { /* Mark the page as queued, so that chapter invalidation knows to cancel a read. */ while (!enqueue_read(&volume->page_cache, request, physical_page)) { - uds_log_debug("Read queue full, waiting for reads to finish"); + vdo_log_debug("Read queue full, waiting for reads to finish"); uds_wait_cond(&volume->read_threads_read_done_cond, &volume->read_threads_mutex); } @@ -431,7 +431,7 @@ static int init_chapter_index_page(const struct volume *volume, u8 *index_page, return result; if (result != UDS_SUCCESS) { - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "Reading chapter index page for chapter %u page %u", chapter, index_page_number); } @@ -445,14 +445,14 @@ static int init_chapter_index_page(const struct volume *volume, u8 *index_page, (highest_list == chapter_index_page->highest_list_number)) return UDS_SUCCESS; - uds_log_warning("Index page map updated to %llu", + vdo_log_warning("Index page map updated to %llu", (unsigned long long) volume->index_page_map->last_update); - uds_log_warning("Page map expects that chapter %u page %u has range %u to %u, but chapter index page has chapter %llu with range %u to %u", + vdo_log_warning("Page map expects that chapter %u page %u has range %u to %u, but chapter index page has chapter %llu with range %u to %u", chapter, index_page_number, lowest_list, highest_list, (unsigned long long) ci_virtual, chapter_index_page->lowest_list_number, chapter_index_page->highest_list_number); - return uds_log_error_strerror(UDS_CORRUPT_DATA, + return vdo_log_error_strerror(UDS_CORRUPT_DATA, "index page map mismatch with chapter index"); } @@ -547,7 +547,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) int result; if (entry->invalid) { - uds_log_debug("Requeuing requests for invalid page"); + vdo_log_debug("Requeuing requests for invalid page"); return UDS_SUCCESS; } @@ -558,7 +558,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) mutex_lock(&volume->read_threads_mutex); if (IS_ERR(page_data)) { result = -PTR_ERR(page_data); - uds_log_warning_strerror(result, + vdo_log_warning_strerror(result, "error reading physical page %u from volume", page_number); cancel_page_in_cache(&volume->page_cache, page_number, page); @@ -566,7 +566,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) } if (entry->invalid) { - uds_log_warning("Page %u invalidated after read", page_number); + vdo_log_warning("Page %u invalidated after read", page_number); cancel_page_in_cache(&volume->page_cache, page_number, page); return UDS_SUCCESS; } @@ -574,7 +574,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) if (!is_record_page(volume->geometry, page_number)) { result = initialize_index_page(volume, page_number, page); if (result != UDS_SUCCESS) { - uds_log_warning("Error initializing chapter index page"); + vdo_log_warning("Error initializing chapter index page"); cancel_page_in_cache(&volume->page_cache, page_number, page); return result; } @@ -582,7 +582,7 @@ static int process_entry(struct volume *volume, struct queued_read *entry) result = put_page_in_cache(&volume->page_cache, page_number, page); if (result != UDS_SUCCESS) { - uds_log_warning("Error putting page %u in cache", page_number); + vdo_log_warning("Error putting page %u in cache", page_number); cancel_page_in_cache(&volume->page_cache, page_number, page); return result; } @@ -624,7 +624,7 @@ static void read_thread_function(void *arg) { struct volume *volume = arg; - uds_log_debug("reader starting"); + vdo_log_debug("reader starting"); mutex_lock(&volume->read_threads_mutex); while (true) { struct queued_read *queue_entry; @@ -638,7 +638,7 @@ static void read_thread_function(void *arg) release_queued_requests(volume, queue_entry, result); } mutex_unlock(&volume->read_threads_mutex); - uds_log_debug("reader done"); + vdo_log_debug("reader done"); } static void get_page_and_index(struct page_cache *cache, u32 physical_page, @@ -701,7 +701,7 @@ static int read_page_locked(struct volume *volume, u32 physical_page, page_data = dm_bufio_read(volume->client, physical_page, &page->buffer); if (IS_ERR(page_data)) { result = -PTR_ERR(page_data); - uds_log_warning_strerror(result, + vdo_log_warning_strerror(result, "error reading physical page %u from volume", physical_page); cancel_page_in_cache(&volume->page_cache, physical_page, page); @@ -712,7 +712,7 @@ static int read_page_locked(struct volume *volume, u32 physical_page, result = initialize_index_page(volume, physical_page, page); if (result != UDS_SUCCESS) { if (volume->lookup_mode != LOOKUP_FOR_REBUILD) - uds_log_warning("Corrupt index page %u", physical_page); + vdo_log_warning("Corrupt index page %u", physical_page); cancel_page_in_cache(&volume->page_cache, physical_page, page); return result; } @@ -720,7 +720,7 @@ static int read_page_locked(struct volume *volume, u32 physical_page, result = put_page_in_cache(&volume->page_cache, physical_page, page); if (result != UDS_SUCCESS) { - uds_log_warning("Error putting page %u in cache", physical_page); + vdo_log_warning("Error putting page %u in cache", physical_page); cancel_page_in_cache(&volume->page_cache, physical_page, page); return result; } @@ -947,7 +947,7 @@ int uds_read_chapter_index_from_volume(const struct volume *volume, u64 virtual_ &volume_buffers[i]); if (IS_ERR(index_page)) { result = -PTR_ERR(index_page); - uds_log_warning_strerror(result, + vdo_log_warning_strerror(result, "error reading physical page %u", physical_page); return result; @@ -1039,7 +1039,7 @@ static void invalidate_page(struct page_cache *cache, u32 physical_page) wait_for_pending_searches(cache, page->physical_page); clear_cache_page(cache, page); } else if (queue_index > -1) { - uds_log_debug("setting pending read to invalid"); + vdo_log_debug("setting pending read to invalid"); cache->read_queue[queue_index].invalid = true; } } @@ -1051,7 +1051,7 @@ void uds_forget_chapter(struct volume *volume, u64 virtual_chapter) u32 first_page = map_to_physical_page(volume->geometry, physical_chapter, 0); u32 i; - uds_log_debug("forgetting chapter %llu", (unsigned long long) virtual_chapter); + vdo_log_debug("forgetting chapter %llu", (unsigned long long) virtual_chapter); mutex_lock(&volume->read_threads_mutex); for (i = 0; i < volume->geometry->pages_per_chapter; i++) invalidate_page(&volume->page_cache, first_page + i); @@ -1077,14 +1077,14 @@ static int donate_index_page_locked(struct volume *volume, u32 physical_chapter, physical_chapter, index_page_number, &page->index_page); if (result != UDS_SUCCESS) { - uds_log_warning("Error initialize chapter index page"); + vdo_log_warning("Error initialize chapter index page"); cancel_page_in_cache(&volume->page_cache, physical_page, page); return result; } result = put_page_in_cache(&volume->page_cache, physical_page, page); if (result != UDS_SUCCESS) { - uds_log_warning("Error putting page %u in cache", physical_page); + vdo_log_warning("Error putting page %u in cache", physical_page); cancel_page_in_cache(&volume->page_cache, physical_page, page); return result; } @@ -1112,7 +1112,7 @@ static int write_index_pages(struct volume *volume, u32 physical_chapter_number, page_data = dm_bufio_new(volume->client, physical_page, &page_buffer); if (IS_ERR(page_data)) { - return uds_log_warning_strerror(-PTR_ERR(page_data), + return vdo_log_warning_strerror(-PTR_ERR(page_data), "failed to prepare index page"); } @@ -1122,14 +1122,14 @@ static int write_index_pages(struct volume *volume, u32 physical_chapter_number, &lists_packed); if (result != UDS_SUCCESS) { dm_bufio_release(page_buffer); - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to pack index page"); } dm_bufio_mark_buffer_dirty(page_buffer); if (lists_packed == 0) { - uds_log_debug("no delta lists packed on chapter %u page %u", + vdo_log_debug("no delta lists packed on chapter %u page %u", physical_chapter_number, index_page_number); } else { delta_list_number += lists_packed; @@ -1221,14 +1221,14 @@ static int write_record_pages(struct volume *volume, u32 physical_chapter_number page_data = dm_bufio_new(volume->client, physical_page, &page_buffer); if (IS_ERR(page_data)) { - return uds_log_warning_strerror(-PTR_ERR(page_data), + return vdo_log_warning_strerror(-PTR_ERR(page_data), "failed to prepare record page"); } result = encode_record_page(volume, next_record, page_data); if (result != UDS_SUCCESS) { dm_bufio_release(page_buffer); - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to encode record page %u", record_page_number); } @@ -1259,7 +1259,7 @@ int uds_write_chapter(struct volume *volume, struct open_chapter_index *chapter_ result = -dm_bufio_write_dirty_buffers(volume->client); if (result != UDS_SUCCESS) - uds_log_error_strerror(result, "cannot sync chapter to volume"); + vdo_log_error_strerror(result, "cannot sync chapter to volume"); return result; } @@ -1286,7 +1286,7 @@ static void probe_chapter(struct volume *volume, u32 chapter_number, return; if (page->virtual_chapter_number == BAD_CHAPTER) { - uds_log_error("corrupt index page in chapter %u", + vdo_log_error("corrupt index page in chapter %u", chapter_number); return; } @@ -1294,14 +1294,14 @@ static void probe_chapter(struct volume *volume, u32 chapter_number, if (vcn == BAD_CHAPTER) { vcn = page->virtual_chapter_number; } else if (page->virtual_chapter_number != vcn) { - uds_log_error("inconsistent chapter %u index page %u: expected vcn %llu, got vcn %llu", + vdo_log_error("inconsistent chapter %u index page %u: expected vcn %llu, got vcn %llu", chapter_number, i, (unsigned long long) vcn, (unsigned long long) page->virtual_chapter_number); return; } if (expected_list_number != page->lowest_list_number) { - uds_log_error("inconsistent chapter %u index page %u: expected list number %u, got list number %u", + vdo_log_error("inconsistent chapter %u index page %u: expected list number %u, got list number %u", chapter_number, i, expected_list_number, page->lowest_list_number); return; @@ -1314,7 +1314,7 @@ static void probe_chapter(struct volume *volume, u32 chapter_number, } if (chapter_number != uds_map_to_physical_chapter(geometry, vcn)) { - uds_log_error("chapter %u vcn %llu is out of phase (%u)", chapter_number, + vdo_log_error("chapter %u vcn %llu is out of phase (%u)", chapter_number, (unsigned long long) vcn, geometry->chapters_per_volume); return; } @@ -1431,7 +1431,7 @@ static int find_chapter_limits(struct volume *volume, u32 chapter_limit, u64 *lo probe_chapter(volume, right_chapter, &highest); if (bad_chapters++ >= MAX_BAD_CHAPTERS) { - uds_log_error("too many bad chapters in volume: %u", + vdo_log_error("too many bad chapters in volume: %u", bad_chapters); return UDS_CORRUPT_DATA; } @@ -1555,7 +1555,7 @@ int uds_make_volume(const struct uds_configuration *config, struct index_layout result = uds_copy_index_geometry(config->geometry, &volume->geometry); if (result != UDS_SUCCESS) { uds_free_volume(volume); - return uds_log_warning_strerror(result, + return vdo_log_warning_strerror(result, "failed to allocate geometry: error"); } geometry = volume->geometry; diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c index a909a11204c1..3aa438f84ea1 100644 --- a/drivers/md/dm-vdo/int-map.c +++ b/drivers/md/dm-vdo/int-map.c @@ -381,7 +381,7 @@ static int resize_buckets(struct int_map *map) /* Re-initialize the map to be empty and 50% larger. */ size_t new_capacity = map->capacity / 2 * 3; - uds_log_info("%s: attempting resize from %zu to %zu, current size=%zu", + vdo_log_info("%s: attempting resize from %zu to %zu, current size=%zu", __func__, map->capacity, new_capacity, map->size); result = allocate_buckets(map, new_capacity); if (result != VDO_SUCCESS) { diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 61bb48068c3a..9a3716bb3c05 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -408,7 +408,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter * Clean up the partially initialized bio-queue entirely and indicate that * initialization failed. */ - uds_log_error("bio map initialization failed %d", result); + vdo_log_error("bio map initialization failed %d", result); vdo_cleanup_io_submitter(io_submitter); vdo_free_io_submitter(io_submitter); return result; @@ -423,7 +423,7 @@ int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_inter * initialization failed. */ vdo_int_map_free(vdo_forget(bio_queue_data->map)); - uds_log_error("bio queue initialization failed %d", result); + vdo_log_error("bio queue initialization failed %d", result); vdo_cleanup_io_submitter(io_submitter); vdo_free_io_submitter(io_submitter); return result; diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 80f1e4c62ac6..3f7dc2cb6b98 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -16,14 +16,14 @@ #include "thread-device.h" #include "thread-utils.h" -int vdo_log_level = UDS_LOG_DEFAULT; +int vdo_log_level = VDO_LOG_DEFAULT; -int uds_get_log_level(void) +int vdo_get_log_level(void) { int log_level_latch = READ_ONCE(vdo_log_level); - if (unlikely(log_level_latch > UDS_LOG_MAX)) { - log_level_latch = UDS_LOG_DEFAULT; + if (unlikely(log_level_latch > VDO_LOG_MAX)) { + log_level_latch = VDO_LOG_DEFAULT; WRITE_ONCE(vdo_log_level, log_level_latch); } return log_level_latch; @@ -54,7 +54,7 @@ static void emit_log_message_to_kernel(int priority, const char *fmt, ...) va_list args; struct va_format vaf; - if (priority > uds_get_log_level()) + if (priority > vdo_get_log_level()) return; va_start(args, fmt); @@ -62,22 +62,22 @@ static void emit_log_message_to_kernel(int priority, const char *fmt, ...) vaf.va = &args; switch (priority) { - case UDS_LOG_EMERG: - case UDS_LOG_ALERT: - case UDS_LOG_CRIT: + case VDO_LOG_EMERG: + case VDO_LOG_ALERT: + case VDO_LOG_CRIT: pr_crit("%pV", &vaf); break; - case UDS_LOG_ERR: + case VDO_LOG_ERR: pr_err("%pV", &vaf); break; - case UDS_LOG_WARNING: + case VDO_LOG_WARNING: pr_warn("%pV", &vaf); break; - case UDS_LOG_NOTICE: - case UDS_LOG_INFO: + case VDO_LOG_NOTICE: + case VDO_LOG_INFO: pr_info("%pV", &vaf); break; - case UDS_LOG_DEBUG: + case VDO_LOG_DEBUG: pr_debug("%pV", &vaf); break; default: @@ -150,7 +150,7 @@ static void emit_log_message(int priority, const char *module, const char *prefi } /* - * uds_log_embedded_message() - Log a message embedded within another message. + * vdo_log_embedded_message() - Log a message embedded within another message. * @priority: the priority at which to log the message * @module: the name of the module doing the logging * @prefix: optional string prefix to message, may be NULL @@ -158,7 +158,7 @@ static void emit_log_message(int priority, const char *module, const char *prefi * @args1: arguments for message first part (required) * @fmt2: format of message second part */ -void uds_log_embedded_message(int priority, const char *module, const char *prefix, +void vdo_log_embedded_message(int priority, const char *module, const char *prefix, const char *fmt1, va_list args1, const char *fmt2, ...) { va_list args1_copy; @@ -168,7 +168,7 @@ void uds_log_embedded_message(int priority, const char *module, const char *pref va_start(args2, fmt2); if (module == NULL) - module = UDS_LOGGING_MODULE_NAME; + module = VDO_LOGGING_MODULE_NAME; if (prefix == NULL) prefix = ""; @@ -191,41 +191,41 @@ void uds_log_embedded_message(int priority, const char *module, const char *pref va_end(args2); } -int uds_vlog_strerror(int priority, int errnum, const char *module, const char *format, +int vdo_vlog_strerror(int priority, int errnum, const char *module, const char *format, va_list args) { - char errbuf[UDS_MAX_ERROR_MESSAGE_SIZE]; + char errbuf[VDO_MAX_ERROR_MESSAGE_SIZE]; const char *message = uds_string_error(errnum, errbuf, sizeof(errbuf)); - uds_log_embedded_message(priority, module, NULL, format, args, ": %s (%d)", + vdo_log_embedded_message(priority, module, NULL, format, args, ": %s (%d)", message, errnum); return errnum; } -int __uds_log_strerror(int priority, int errnum, const char *module, const char *format, ...) +int __vdo_log_strerror(int priority, int errnum, const char *module, const char *format, ...) { va_list args; va_start(args, format); - uds_vlog_strerror(priority, errnum, module, format, args); + vdo_vlog_strerror(priority, errnum, module, format, args); va_end(args); return errnum; } -void uds_log_backtrace(int priority) +void vdo_log_backtrace(int priority) { - if (priority > uds_get_log_level()) + if (priority > vdo_get_log_level()) return; dump_stack(); } -void __uds_log_message(int priority, const char *module, const char *format, ...) +void __vdo_log_message(int priority, const char *module, const char *format, ...) { va_list args; va_start(args, format); - uds_log_embedded_message(priority, module, NULL, format, args, "%s", ""); + vdo_log_embedded_message(priority, module, NULL, format, args, "%s", ""); va_end(args); } @@ -233,7 +233,7 @@ void __uds_log_message(int priority, const char *module, const char *format, ... * Sleep or delay a few milliseconds in an attempt to allow the log buffers to be flushed lest they * be overrun. */ -void uds_pause_for_logger(void) +void vdo_pause_for_logger(void) { fsleep(4000); } diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h index a8cdf46b6fc1..ae6ad691c027 100644 --- a/drivers/md/dm-vdo/logger.h +++ b/drivers/md/dm-vdo/logger.h @@ -3,8 +3,8 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_LOGGER_H -#define UDS_LOGGER_H +#ifndef VDO_LOGGER_H +#define VDO_LOGGER_H #include #include @@ -14,26 +14,26 @@ /* Custom logging utilities for UDS */ enum { - UDS_LOG_EMERG = LOGLEVEL_EMERG, - UDS_LOG_ALERT = LOGLEVEL_ALERT, - UDS_LOG_CRIT = LOGLEVEL_CRIT, - UDS_LOG_ERR = LOGLEVEL_ERR, - UDS_LOG_WARNING = LOGLEVEL_WARNING, - UDS_LOG_NOTICE = LOGLEVEL_NOTICE, - UDS_LOG_INFO = LOGLEVEL_INFO, - UDS_LOG_DEBUG = LOGLEVEL_DEBUG, + VDO_LOG_EMERG = LOGLEVEL_EMERG, + VDO_LOG_ALERT = LOGLEVEL_ALERT, + VDO_LOG_CRIT = LOGLEVEL_CRIT, + VDO_LOG_ERR = LOGLEVEL_ERR, + VDO_LOG_WARNING = LOGLEVEL_WARNING, + VDO_LOG_NOTICE = LOGLEVEL_NOTICE, + VDO_LOG_INFO = LOGLEVEL_INFO, + VDO_LOG_DEBUG = LOGLEVEL_DEBUG, - UDS_LOG_MAX = UDS_LOG_DEBUG, - UDS_LOG_DEFAULT = UDS_LOG_INFO, + VDO_LOG_MAX = VDO_LOG_DEBUG, + VDO_LOG_DEFAULT = VDO_LOG_INFO, }; extern int vdo_log_level; #define DM_MSG_PREFIX "vdo" -#define UDS_LOGGING_MODULE_NAME DM_NAME ": " DM_MSG_PREFIX +#define VDO_LOGGING_MODULE_NAME DM_NAME ": " DM_MSG_PREFIX /* Apply a rate limiter to a log method call. */ -#define uds_log_ratelimit(log_fn, ...) \ +#define vdo_log_ratelimit(log_fn, ...) \ do { \ static DEFINE_RATELIMIT_STATE(_rs, \ DEFAULT_RATELIMIT_INTERVAL, \ @@ -43,58 +43,58 @@ extern int vdo_log_level; } \ } while (0) -int uds_get_log_level(void); +int vdo_get_log_level(void); -void uds_log_embedded_message(int priority, const char *module, const char *prefix, +void vdo_log_embedded_message(int priority, const char *module, const char *prefix, const char *fmt1, va_list args1, const char *fmt2, ...) __printf(4, 0) __printf(6, 7); -void uds_log_backtrace(int priority); +void vdo_log_backtrace(int priority); /* All log functions will preserve the caller's value of errno. */ -#define uds_log_strerror(priority, errnum, ...) \ - __uds_log_strerror(priority, errnum, UDS_LOGGING_MODULE_NAME, __VA_ARGS__) +#define vdo_log_strerror(priority, errnum, ...) \ + __vdo_log_strerror(priority, errnum, VDO_LOGGING_MODULE_NAME, __VA_ARGS__) -int __uds_log_strerror(int priority, int errnum, const char *module, +int __vdo_log_strerror(int priority, int errnum, const char *module, const char *format, ...) __printf(4, 5); -int uds_vlog_strerror(int priority, int errnum, const char *module, const char *format, +int vdo_vlog_strerror(int priority, int errnum, const char *module, const char *format, va_list args) __printf(4, 0); /* Log an error prefixed with the string associated with the errnum. */ -#define uds_log_error_strerror(errnum, ...) \ - uds_log_strerror(UDS_LOG_ERR, errnum, __VA_ARGS__) +#define vdo_log_error_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_ERR, errnum, __VA_ARGS__) -#define uds_log_debug_strerror(errnum, ...) \ - uds_log_strerror(UDS_LOG_DEBUG, errnum, __VA_ARGS__) +#define vdo_log_debug_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_DEBUG, errnum, __VA_ARGS__) -#define uds_log_info_strerror(errnum, ...) \ - uds_log_strerror(UDS_LOG_INFO, errnum, __VA_ARGS__) +#define vdo_log_info_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_INFO, errnum, __VA_ARGS__) -#define uds_log_warning_strerror(errnum, ...) \ - uds_log_strerror(UDS_LOG_WARNING, errnum, __VA_ARGS__) +#define vdo_log_warning_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_WARNING, errnum, __VA_ARGS__) -#define uds_log_fatal_strerror(errnum, ...) \ - uds_log_strerror(UDS_LOG_CRIT, errnum, __VA_ARGS__) +#define vdo_log_fatal_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_CRIT, errnum, __VA_ARGS__) -#define uds_log_message(priority, ...) \ - __uds_log_message(priority, UDS_LOGGING_MODULE_NAME, __VA_ARGS__) +#define vdo_log_message(priority, ...) \ + __vdo_log_message(priority, VDO_LOGGING_MODULE_NAME, __VA_ARGS__) -void __uds_log_message(int priority, const char *module, const char *format, ...) +void __vdo_log_message(int priority, const char *module, const char *format, ...) __printf(3, 4); -#define uds_log_debug(...) uds_log_message(UDS_LOG_DEBUG, __VA_ARGS__) +#define vdo_log_debug(...) vdo_log_message(VDO_LOG_DEBUG, __VA_ARGS__) -#define uds_log_info(...) uds_log_message(UDS_LOG_INFO, __VA_ARGS__) +#define vdo_log_info(...) vdo_log_message(VDO_LOG_INFO, __VA_ARGS__) -#define uds_log_warning(...) uds_log_message(UDS_LOG_WARNING, __VA_ARGS__) +#define vdo_log_warning(...) vdo_log_message(VDO_LOG_WARNING, __VA_ARGS__) -#define uds_log_error(...) uds_log_message(UDS_LOG_ERR, __VA_ARGS__) +#define vdo_log_error(...) vdo_log_message(VDO_LOG_ERR, __VA_ARGS__) -#define uds_log_fatal(...) uds_log_message(UDS_LOG_CRIT, __VA_ARGS__) +#define vdo_log_fatal(...) vdo_log_message(VDO_LOG_CRIT, __VA_ARGS__) -void uds_pause_for_logger(void); -#endif /* UDS_LOGGER_H */ +void vdo_pause_for_logger(void); +#endif /* VDO_LOGGER_H */ diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index 300f9d2d2d5c..258bc55e419b 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -363,8 +363,8 @@ struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone) */ void vdo_dump_logical_zone(const struct logical_zone *zone) { - uds_log_info("logical_zone %u", zone->zone_number); - uds_log_info(" flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu", + vdo_log_info("logical_zone %u", zone->zone_number); + vdo_log_info(" flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu", (unsigned long long) READ_ONCE(zone->flush_generation), (unsigned long long) READ_ONCE(zone->oldest_active_generation), (unsigned long long) READ_ONCE(zone->notification_generation), diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c index 62bb717c4c50..185f259c7245 100644 --- a/drivers/md/dm-vdo/memory-alloc.c +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -150,7 +150,7 @@ static void remove_vmalloc_block(void *ptr) if (block != NULL) vdo_free(block); else - uds_log_info("attempting to remove ptr %px not found in vmalloc list", ptr); + vdo_log_info("attempting to remove ptr %px not found in vmalloc list", ptr); } /* @@ -284,7 +284,7 @@ int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) memalloc_noio_restore(noio_flags); if (unlikely(p == NULL)) { - uds_log_error("Could not allocate %zu bytes for %s in %u msecs", + vdo_log_error("Could not allocate %zu bytes for %s in %u msecs", size, what, jiffies_to_msecs(jiffies - start_time)); return -ENOMEM; } @@ -391,7 +391,7 @@ void vdo_memory_exit(void) VDO_ASSERT_LOG_ONLY(memory_stats.vmalloc_bytes == 0, "vmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", memory_stats.vmalloc_bytes, memory_stats.vmalloc_blocks); - uds_log_debug("peak usage %zd bytes", memory_stats.peak_bytes); + vdo_log_debug("peak usage %zd bytes", memory_stats.peak_bytes); } void vdo_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used) @@ -426,13 +426,13 @@ void vdo_report_memory_usage(void) peak_usage = memory_stats.peak_bytes; spin_unlock_irqrestore(&memory_stats.lock, flags); total_bytes = kmalloc_bytes + vmalloc_bytes; - uds_log_info("current module memory tracking (actual allocation sizes, not requested):"); - uds_log_info(" %llu bytes in %llu kmalloc blocks", + vdo_log_info("current module memory tracking (actual allocation sizes, not requested):"); + vdo_log_info(" %llu bytes in %llu kmalloc blocks", (unsigned long long) kmalloc_bytes, (unsigned long long) kmalloc_blocks); - uds_log_info(" %llu bytes in %llu vmalloc blocks", + vdo_log_info(" %llu bytes in %llu vmalloc blocks", (unsigned long long) vmalloc_bytes, (unsigned long long) vmalloc_blocks); - uds_log_info(" total %llu bytes, peak usage %llu bytes", + vdo_log_info(" total %llu bytes, peak usage %llu bytes", (unsigned long long) total_bytes, (unsigned long long) peak_usage); } diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index 18c9d2af8aed..2802cf92922b 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -421,7 +421,7 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) result = vdo_allocate(1, struct vdo_statistics, __func__, &stats); if (result != VDO_SUCCESS) { - uds_log_error("Cannot allocate memory to write VDO statistics"); + vdo_log_error("Cannot allocate memory to write VDO statistics"); return result; } diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c index 4d45243161a6..16cf29b4c90a 100644 --- a/drivers/md/dm-vdo/packer.c +++ b/drivers/md/dm-vdo/packer.c @@ -748,7 +748,7 @@ static void dump_packer_bin(const struct packer_bin *bin, bool canceled) /* Don't dump empty bins. */ return; - uds_log_info(" %sBin slots_used=%u free_space=%zu", + vdo_log_info(" %sBin slots_used=%u free_space=%zu", (canceled ? "Canceled" : ""), bin->slots_used, bin->free_space); /* @@ -767,8 +767,8 @@ void vdo_dump_packer(const struct packer *packer) { struct packer_bin *bin; - uds_log_info("packer"); - uds_log_info(" flushGeneration=%llu state %s packer_bin_count=%llu", + vdo_log_info("packer"); + vdo_log_info(" flushGeneration=%llu state %s packer_bin_count=%llu", (unsigned long long) packer->flush_generation, vdo_get_admin_state_code(&packer->state)->name, (unsigned long long) packer->size); diff --git a/drivers/md/dm-vdo/permassert.c b/drivers/md/dm-vdo/permassert.c index 6fe49c4b7e51..bf9eccea1cb3 100644 --- a/drivers/md/dm-vdo/permassert.c +++ b/drivers/md/dm-vdo/permassert.c @@ -15,10 +15,10 @@ int vdo_assertion_failed(const char *expression_string, const char *file_name, va_start(args, format); - uds_log_embedded_message(UDS_LOG_ERR, UDS_LOGGING_MODULE_NAME, "assertion \"", + vdo_log_embedded_message(VDO_LOG_ERR, VDO_LOGGING_MODULE_NAME, "assertion \"", format, args, "\" (%s) failed at %s:%d", expression_string, file_name, line_number); - uds_log_backtrace(UDS_LOG_ERR); + vdo_log_backtrace(VDO_LOG_ERR); va_end(args); diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c index 6678f472fb44..2fee3a7c1191 100644 --- a/drivers/md/dm-vdo/physical-zone.c +++ b/drivers/md/dm-vdo/physical-zone.c @@ -163,7 +163,7 @@ static void release_pbn_lock_provisional_reference(struct pbn_lock *lock, result = vdo_release_block_reference(allocator, locked_pbn); if (result != VDO_SUCCESS) { - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "Failed to release reference to %s physical block %llu", lock->implementation->release_reason, (unsigned long long) locked_pbn); @@ -294,7 +294,7 @@ static int __must_check borrow_pbn_lock_from_pool(struct pbn_lock_pool *pool, idle_pbn_lock *idle; if (pool->borrowed >= pool->capacity) - return uds_log_error_strerror(VDO_LOCK_ERROR, + return vdo_log_error_strerror(VDO_LOCK_ERROR, "no free PBN locks left to borrow"); pool->borrowed += 1; @@ -499,7 +499,7 @@ static int allocate_and_lock_block(struct allocation *allocation) if (lock->holder_count > 0) { /* This block is already locked, which should be impossible. */ - return uds_log_error_strerror(VDO_LOCK_ERROR, + return vdo_log_error_strerror(VDO_LOCK_ERROR, "Newly allocated block %llu was spuriously locked (holder_count=%u)", (unsigned long long) allocation->pbn, lock->holder_count); diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c index 6df373b88042..ee6321a3e523 100644 --- a/drivers/md/dm-vdo/recovery-journal.c +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -804,7 +804,7 @@ void vdo_free_recovery_journal(struct recovery_journal *journal) "journal being freed has no active tail blocks"); } else if (!vdo_is_state_saved(&journal->state) && !list_empty(&journal->active_tail_blocks)) { - uds_log_warning("journal being freed has uncommitted entries"); + vdo_log_warning("journal being freed has uncommitted entries"); } for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) { @@ -1305,7 +1305,7 @@ static void handle_write_error(struct vdo_completion *completion) struct recovery_journal *journal = block->journal; vio_record_metadata_io_error(as_vio(completion)); - uds_log_error_strerror(completion->result, + vdo_log_error_strerror(completion->result, "cannot write recovery journal block %llu", (unsigned long long) block->sequence_number); enter_journal_read_only_mode(journal, completion->result); @@ -1719,7 +1719,7 @@ vdo_get_recovery_journal_statistics(const struct recovery_journal *journal) */ static void dump_recovery_block(const struct recovery_journal_block *block) { - uds_log_info(" sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters", + vdo_log_info(" sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters", (unsigned long long) block->sequence_number, block->entry_count, (block->committing ? "committing" : "waiting"), vdo_waitq_num_waiters(&block->entry_waiters), @@ -1736,8 +1736,8 @@ void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal const struct recovery_journal_block *block; struct recovery_journal_statistics stats = vdo_get_recovery_journal_statistics(journal); - uds_log_info("Recovery Journal"); - uds_log_info(" block_map_head=%llu slab_journal_head=%llu last_write_acknowledged=%llu tail=%llu block_map_reap_head=%llu slab_journal_reap_head=%llu disk_full=%llu slab_journal_commits_requested=%llu entry_waiters=%zu", + vdo_log_info("Recovery Journal"); + vdo_log_info(" block_map_head=%llu slab_journal_head=%llu last_write_acknowledged=%llu tail=%llu block_map_reap_head=%llu slab_journal_reap_head=%llu disk_full=%llu slab_journal_commits_requested=%llu entry_waiters=%zu", (unsigned long long) journal->block_map_head, (unsigned long long) journal->slab_journal_head, (unsigned long long) journal->last_write_acknowledged, @@ -1747,16 +1747,16 @@ void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal (unsigned long long) stats.disk_full, (unsigned long long) stats.slab_journal_commits_requested, vdo_waitq_num_waiters(&journal->entry_waiters)); - uds_log_info(" entries: started=%llu written=%llu committed=%llu", + vdo_log_info(" entries: started=%llu written=%llu committed=%llu", (unsigned long long) stats.entries.started, (unsigned long long) stats.entries.written, (unsigned long long) stats.entries.committed); - uds_log_info(" blocks: started=%llu written=%llu committed=%llu", + vdo_log_info(" blocks: started=%llu written=%llu committed=%llu", (unsigned long long) stats.blocks.started, (unsigned long long) stats.blocks.written, (unsigned long long) stats.blocks.committed); - uds_log_info(" active blocks:"); + vdo_log_info(" active blocks:"); list_for_each_entry(block, &journal->active_tail_blocks, list_node) dump_recovery_block(block); } diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index c7abb8078336..defc9359f10e 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -265,13 +265,13 @@ static void finish_repair(struct vdo_completion *completion) free_repair_completion(vdo_forget(repair)); if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { - uds_log_info("Read-only rebuild complete"); + vdo_log_info("Read-only rebuild complete"); vdo_launch_completion(parent); return; } /* FIXME: shouldn't this say either "recovery" or "repair"? */ - uds_log_info("Rebuild complete"); + vdo_log_info("Rebuild complete"); /* * Now that we've freed the repair completion and its vast array of journal entries, we @@ -291,9 +291,9 @@ static void abort_repair(struct vdo_completion *completion) struct repair_completion *repair = as_repair_completion(completion); if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) - uds_log_info("Read-only rebuild aborted"); + vdo_log_info("Read-only rebuild aborted"); else - uds_log_warning("Recovery aborted"); + vdo_log_warning("Recovery aborted"); free_repair_completion(vdo_forget(repair)); vdo_continue_completion(parent, result); @@ -329,10 +329,10 @@ static void drain_slab_depot(struct vdo_completion *completion) prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { - uds_log_info("Saving rebuilt state"); + vdo_log_info("Saving rebuilt state"); operation = VDO_ADMIN_STATE_REBUILDING; } else { - uds_log_info("Replayed %zu journal entries into slab journals", + vdo_log_info("Replayed %zu journal entries into slab journals", repair->entries_added_to_slab_journals); operation = VDO_ADMIN_STATE_RECOVERING; } @@ -350,7 +350,7 @@ static void flush_block_map_updates(struct vdo_completion *completion) { vdo_assert_on_admin_thread(completion->vdo, __func__); - uds_log_info("Flushing block map changes"); + vdo_log_info("Flushing block map changes"); prepare_repair_completion(as_repair_completion(completion), drain_slab_depot, VDO_ZONE_TYPE_ADMIN); vdo_drain_block_map(completion->vdo->block_map, VDO_ADMIN_STATE_RECOVERING, @@ -449,7 +449,7 @@ static bool process_slot(struct block_map_page *page, struct vdo_completion *com if (result == VDO_SUCCESS) return true; - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "Could not adjust reference count for PBN %llu, slot %u mapped to PBN %llu", (unsigned long long) vdo_get_block_map_page_pbn(page), slot, (unsigned long long) mapping.pbn); @@ -615,7 +615,7 @@ static int process_entry(physical_block_number_t pbn, struct vdo_completion *com int result; if ((pbn == VDO_ZERO_BLOCK) || !vdo_is_physical_data_block(depot, pbn)) { - return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, "PBN %llu out of range", (unsigned long long) pbn); } @@ -623,7 +623,7 @@ static int process_entry(physical_block_number_t pbn, struct vdo_completion *com result = vdo_adjust_reference_count_for_rebuild(depot, pbn, VDO_JOURNAL_BLOCK_MAP_REMAPPING); if (result != VDO_SUCCESS) { - return uds_log_error_strerror(result, + return vdo_log_error_strerror(result, "Could not adjust reference count for block map tree PBN %llu", (unsigned long long) pbn); } @@ -758,7 +758,7 @@ static int validate_recovery_journal_entry(const struct vdo *vdo, !vdo_is_valid_location(&entry->unmapping) || !vdo_is_physical_data_block(vdo->depot, entry->mapping.pbn) || !vdo_is_physical_data_block(vdo->depot, entry->unmapping.pbn)) { - return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, "Invalid entry: %s (%llu, %u) from %llu to %llu is not within bounds", vdo_get_journal_operation_name(entry->operation), (unsigned long long) entry->slot.pbn, @@ -772,7 +772,7 @@ static int validate_recovery_journal_entry(const struct vdo *vdo, (entry->mapping.pbn == VDO_ZERO_BLOCK) || (entry->unmapping.state != VDO_MAPPING_STATE_UNMAPPED) || (entry->unmapping.pbn != VDO_ZERO_BLOCK))) { - return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, "Invalid entry: %s (%llu, %u) from %llu to %llu is not a valid tree mapping", vdo_get_journal_operation_name(entry->operation), (unsigned long long) entry->slot.pbn, @@ -875,7 +875,7 @@ void vdo_replay_into_slab_journals(struct block_allocator *allocator, void *cont .entry_count = 0, }; - uds_log_info("Replaying entries into slab journals for zone %u", + vdo_log_info("Replaying entries into slab journals for zone %u", allocator->zone_number); completion->parent = repair; add_slab_journal_entries(completion); @@ -907,7 +907,7 @@ static void flush_block_map(struct vdo_completion *completion) vdo_assert_on_admin_thread(completion->vdo, __func__); - uds_log_info("Flushing block map changes"); + vdo_log_info("Flushing block map changes"); prepare_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); operation = (vdo_state_requires_read_only_rebuild(completion->vdo->load_state) ? VDO_ADMIN_STATE_REBUILDING : @@ -1107,7 +1107,7 @@ static void recover_block_map(struct vdo_completion *completion) vdo_state_requires_read_only_rebuild(vdo->load_state); if (repair->block_map_entry_count == 0) { - uds_log_info("Replaying 0 recovery entries into block map"); + vdo_log_info("Replaying 0 recovery entries into block map"); vdo_free(vdo_forget(repair->journal_data)); launch_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); return; @@ -1124,7 +1124,7 @@ static void recover_block_map(struct vdo_completion *completion) }; min_heapify_all(&repair->replay_heap, &repair_min_heap); - uds_log_info("Replaying %zu recovery entries into block map", + vdo_log_info("Replaying %zu recovery entries into block map", repair->block_map_entry_count); repair->current_entry = &repair->entries[repair->block_map_entry_count - 1]; @@ -1437,7 +1437,7 @@ static int validate_heads(struct repair_completion *repair) return VDO_SUCCESS; - return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, "Journal tail too early. block map head: %llu, slab journal head: %llu, tail: %llu", (unsigned long long) repair->block_map_head, (unsigned long long) repair->slab_journal_head, @@ -1571,7 +1571,7 @@ static int parse_journal_for_recovery(struct repair_completion *repair) header = get_recovery_journal_block_header(journal, repair->journal_data, i); if (header.metadata_type == VDO_METADATA_RECOVERY_JOURNAL) { /* This is an old format block, so we need to upgrade */ - uds_log_error_strerror(VDO_UNSUPPORTED_VERSION, + vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, "Recovery journal is in the old format, a read-only rebuild is required."); vdo_enter_read_only_mode(repair->completion.vdo, VDO_UNSUPPORTED_VERSION); @@ -1628,7 +1628,7 @@ static int parse_journal_for_recovery(struct repair_completion *repair) if (result != VDO_SUCCESS) return result; - uds_log_info("Highest-numbered recovery journal block has sequence number %llu, and the highest-numbered usable block is %llu", + vdo_log_info("Highest-numbered recovery journal block has sequence number %llu, and the highest-numbered usable block is %llu", (unsigned long long) repair->highest_tail, (unsigned long long) repair->tail); @@ -1656,7 +1656,7 @@ static void finish_journal_load(struct vdo_completion *completion) if (++repair->vios_complete != repair->vio_count) return; - uds_log_info("Finished reading recovery journal"); + vdo_log_info("Finished reading recovery journal"); uninitialize_vios(repair); prepare_repair_completion(repair, recover_block_map, VDO_ZONE_TYPE_LOGICAL); vdo_continue_completion(&repair->completion, parse_journal(repair)); @@ -1701,12 +1701,12 @@ void vdo_repair(struct vdo_completion *parent) vdo_assert_on_admin_thread(vdo, __func__); if (vdo->load_state == VDO_FORCE_REBUILD) { - uds_log_warning("Rebuilding reference counts to clear read-only mode"); + vdo_log_warning("Rebuilding reference counts to clear read-only mode"); vdo->states.vdo.read_only_recoveries++; } else if (vdo->load_state == VDO_REBUILD_FOR_UPGRADE) { - uds_log_warning("Rebuilding reference counts for upgrade"); + vdo_log_warning("Rebuilding reference counts for upgrade"); } else { - uds_log_warning("Device was dirty, rebuilding reference counts"); + vdo_log_warning("Device was dirty, rebuilding reference counts"); } result = vdo_allocate_extended(struct repair_completion, page_count, diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index 00746de09c12..dc9f3d3c3995 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -568,7 +568,7 @@ static void release_journal_locks(struct vdo_waiter *waiter, void *context) * Don't bother logging what might be lots of errors if we are already in * read-only mode. */ - uds_log_error_strerror(result, "failed slab summary update %llu", + vdo_log_error_strerror(result, "failed slab summary update %llu", (unsigned long long) journal->summarized); } @@ -702,7 +702,7 @@ static void complete_write(struct vdo_completion *completion) if (result != VDO_SUCCESS) { vio_record_metadata_io_error(as_vio(completion)); - uds_log_error_strerror(result, "cannot write slab journal block %llu", + vdo_log_error_strerror(result, "cannot write slab journal block %llu", (unsigned long long) committed); vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); check_if_slab_drained(journal->slab); @@ -1020,7 +1020,7 @@ static void finish_summary_update(struct vdo_waiter *waiter, void *context) slab->active_count--; if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { - uds_log_error_strerror(result, "failed to update slab summary"); + vdo_log_error_strerror(result, "failed to update slab summary"); vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); } @@ -1440,7 +1440,7 @@ static int increment_for_data(struct vdo_slab *slab, struct reference_block *blo default: /* Single or shared */ if (*counter_ptr >= MAXIMUM_REFERENCE_COUNT) { - return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Incrementing a block already having 254 references (slab %u, offset %u)", slab->slab_number, block_number); } @@ -1473,7 +1473,7 @@ static int decrement_for_data(struct vdo_slab *slab, struct reference_block *blo { switch (old_status) { case RS_FREE: - return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Decrementing free block at offset %u in slab %u", block_number, slab->slab_number); @@ -1537,7 +1537,7 @@ static int increment_for_block_map(struct vdo_slab *slab, struct reference_block switch (old_status) { case RS_FREE: if (normal_operation) { - return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Incrementing unallocated block map block (slab %u, offset %u)", slab->slab_number, block_number); } @@ -1552,7 +1552,7 @@ static int increment_for_block_map(struct vdo_slab *slab, struct reference_block case RS_PROVISIONAL: if (!normal_operation) - return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Block map block had provisional reference during replay (slab %u, offset %u)", slab->slab_number, block_number); @@ -1562,7 +1562,7 @@ static int increment_for_block_map(struct vdo_slab *slab, struct reference_block return VDO_SUCCESS; default: - return uds_log_error_strerror(VDO_REF_COUNT_INVALID, + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Incrementing a block map block which is already referenced %u times (slab %u, offset %u)", *counter_ptr, slab->slab_number, block_number); @@ -2219,7 +2219,7 @@ static void unpack_reference_block(struct packed_reference_block *packed, block->commit_points[i])) { size_t block_index = block - block->slab->reference_blocks; - uds_log_warning("Torn write detected in sector %u of reference block %zu of slab %u", + vdo_log_warning("Torn write detected in sector %u of reference block %zu of slab %u", i, block_index, block->slab->slab_number); } } @@ -2698,9 +2698,9 @@ static void finish_scrubbing(struct slab_scrubber *scrubber, int result) * thread does not yet know about. */ if (prior_state == VDO_DIRTY) - uds_log_info("VDO commencing normal operation"); + vdo_log_info("VDO commencing normal operation"); else if (prior_state == VDO_RECOVERING) - uds_log_info("Exiting recovery mode"); + vdo_log_info("Exiting recovery mode"); } /* @@ -2790,7 +2790,7 @@ static int apply_block_entries(struct packed_slab_journal_block *block, if (entry.sbn > max_sbn) { /* This entry is out of bounds. */ - return uds_log_error_strerror(VDO_CORRUPT_JOURNAL, + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, "vdo_slab journal entry (%llu, %u) had invalid offset %u in slab (size %u blocks)", (unsigned long long) block_number, entry_point.entry_count, @@ -2799,7 +2799,7 @@ static int apply_block_entries(struct packed_slab_journal_block *block, result = replay_reference_count_change(slab, &entry_point, entry); if (result != VDO_SUCCESS) { - uds_log_error_strerror(result, + vdo_log_error_strerror(result, "vdo_slab journal entry (%llu, %u) (%s of offset %u) could not be applied in slab %u", (unsigned long long) block_number, entry_point.entry_count, @@ -2857,7 +2857,7 @@ static void apply_journal_entries(struct vdo_completion *completion) (header.has_block_map_increments && (header.entry_count > journal->full_entries_per_block))) { /* The block is not what we expect it to be. */ - uds_log_error("vdo_slab journal block for slab %u was invalid", + vdo_log_error("vdo_slab journal block for slab %u was invalid", slab->slab_number); abort_scrubbing(scrubber, VDO_CORRUPT_JOURNAL); return; @@ -3580,22 +3580,22 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) struct slab_iterator iterator = get_slab_iterator(allocator); const struct slab_scrubber *scrubber = &allocator->scrubber; - uds_log_info("block_allocator zone %u", allocator->zone_number); + vdo_log_info("block_allocator zone %u", allocator->zone_number); while (iterator.next != NULL) { struct vdo_slab *slab = next_slab(&iterator); struct slab_journal *journal = &slab->journal; if (slab->reference_blocks != NULL) { /* Terse because there are a lot of slabs to dump and syslog is lossy. */ - uds_log_info("slab %u: P%u, %llu free", slab->slab_number, + vdo_log_info("slab %u: P%u, %llu free", slab->slab_number, slab->priority, (unsigned long long) slab->free_blocks); } else { - uds_log_info("slab %u: status %s", slab->slab_number, + vdo_log_info("slab %u: status %s", slab->slab_number, status_to_string(slab->status)); } - uds_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", + vdo_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", vdo_waitq_num_waiters(&journal->entry_waiters), uds_bool_to_string(journal->waiting_to_commit), uds_bool_to_string(journal->updating_slab_summary), @@ -3614,7 +3614,7 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) if (slab->counters != NULL) { /* Terse because there are a lot of slabs to dump and syslog is lossy. */ - uds_log_info(" slab: free=%u/%u blocks=%u dirty=%zu active=%zu journal@(%llu,%u)", + vdo_log_info(" slab: free=%u/%u blocks=%u dirty=%zu active=%zu journal@(%llu,%u)", slab->free_blocks, slab->block_count, slab->reference_block_count, vdo_waitq_num_waiters(&slab->dirty_blocks), @@ -3622,7 +3622,7 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) (unsigned long long) slab->slab_journal_point.sequence_number, slab->slab_journal_point.entry_count); } else { - uds_log_info(" no counters"); + vdo_log_info(" no counters"); } /* @@ -3631,11 +3631,11 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) */ if (pause_counter++ == 31) { pause_counter = 0; - uds_pause_for_logger(); + vdo_pause_for_logger(); } } - uds_log_info("slab_scrubber slab_count %u waiters %zu %s%s", + vdo_log_info("slab_scrubber slab_count %u waiters %zu %s%s", READ_ONCE(scrubber->slab_count), vdo_waitq_num_waiters(&scrubber->waiters), vdo_get_admin_state_code(&scrubber->admin_state)->name, @@ -4109,7 +4109,7 @@ static int allocate_components(struct slab_depot *depot, slab_count = vdo_compute_slab_count(depot->first_block, depot->last_block, depot->slab_size_shift); if (thread_config->physical_zone_count > slab_count) { - return uds_log_error_strerror(VDO_BAD_CONFIGURATION, + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, "%u physical zones exceeds slab count %u", thread_config->physical_zone_count, slab_count); @@ -4167,7 +4167,7 @@ int vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo, block_count_t slab_size = state.slab_config.slab_blocks; if (!is_power_of_2(slab_size)) { - return uds_log_error_strerror(UDS_INVALID_ARGUMENT, + return vdo_log_error_strerror(UDS_INVALID_ARGUMENT, "slab size must be a power of two"); } slab_size_shift = ilog2(slab_size); @@ -4676,7 +4676,7 @@ int vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, new_state.last_block, depot->slab_size_shift); if (new_slab_count <= depot->slab_count) - return uds_log_error_strerror(VDO_INCREMENT_TOO_SMALL, + return vdo_log_error_strerror(VDO_INCREMENT_TOO_SMALL, "Depot can only grow"); if (new_slab_count == depot->new_slab_count) { /* Check it out, we've already got all the new slabs allocated! */ @@ -5092,8 +5092,8 @@ void vdo_get_slab_depot_statistics(const struct slab_depot *depot, */ void vdo_dump_slab_depot(const struct slab_depot *depot) { - uds_log_info("vdo slab depot"); - uds_log_info(" zone_count=%u old_zone_count=%u slabCount=%u active_release_request=%llu new_release_request=%llu", + vdo_log_info("vdo slab depot"); + vdo_log_info(" zone_count=%u old_zone_count=%u slabCount=%u active_release_request=%llu new_release_request=%llu", (unsigned int) depot->zone_count, (unsigned int) depot->old_zone_count, READ_ONCE(depot->slab_count), (unsigned long long) depot->active_release_request, diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index 42e87b2344bc..918e46e7121f 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -87,8 +87,8 @@ int vdo_register_status_codes(void) */ int vdo_status_to_errno(int error) { - char error_name[UDS_MAX_ERROR_NAME_SIZE]; - char error_message[UDS_MAX_ERROR_MESSAGE_SIZE]; + char error_name[VDO_MAX_ERROR_NAME_SIZE]; + char error_message[VDO_MAX_ERROR_MESSAGE_SIZE]; /* 0 is success, negative a system error code */ if (likely(error <= 0)) @@ -103,7 +103,7 @@ int vdo_status_to_errno(int error) case VDO_READ_ONLY: return -EIO; default: - uds_log_info("%s: mapping internal status code %d (%s: %s) to EIO", + vdo_log_info("%s: mapping internal status code %d (%s: %s) to EIO", __func__, error, uds_string_error_name(error, error_name, sizeof(error_name)), uds_string_error(error, error_message, sizeof(error_message))); diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index c822df86f731..bd620be61c1d 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -84,7 +84,7 @@ int vdo_create_thread(void (*thread_function)(void *), void *thread_data, result = vdo_allocate(1, struct thread, __func__, &thread); if (result != VDO_SUCCESS) { - uds_log_warning("Error allocating memory for %s", name); + vdo_log_warning("Error allocating memory for %s", name); return result; } diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c index 28e6352c758e..fff847767755 100644 --- a/drivers/md/dm-vdo/vdo.c +++ b/drivers/md/dm-vdo/vdo.c @@ -304,7 +304,7 @@ static int __must_check read_geometry_block(struct vdo *vdo) result = blk_status_to_errno(vio->bio->bi_status); free_vio(vdo_forget(vio)); if (result != 0) { - uds_log_error_strerror(result, "synchronous read failed"); + vdo_log_error_strerror(result, "synchronous read failed"); vdo_free(block); return -EIO; } @@ -493,7 +493,7 @@ static int initialize_vdo(struct vdo *vdo, struct device_config *config, return result; } - uds_log_info("zones: %d logical, %d physical, %d hash; total threads: %d", + vdo_log_info("zones: %d logical, %d physical, %d hash; total threads: %d", config->thread_counts.logical_zones, config->thread_counts.physical_zones, config->thread_counts.hash_zones, vdo->thread_config.thread_count); @@ -841,7 +841,7 @@ int vdo_synchronous_flush(struct vdo *vdo) atomic64_inc(&vdo->stats.flush_out); if (result != 0) { - uds_log_error_strerror(result, "synchronous flush failed"); + vdo_log_error_strerror(result, "synchronous flush failed"); result = -EIO; } @@ -928,7 +928,7 @@ static void handle_save_error(struct vdo_completion *completion) container_of(as_vio(completion), struct vdo_super_block, vio); vio_record_metadata_io_error(&super_block->vio); - uds_log_error_strerror(completion->result, "super block save failed"); + vdo_log_error_strerror(completion->result, "super block save failed"); /* * Mark the super block as unwritable so that we won't attempt to write it again. This * avoids the case where a growth attempt fails writing the super block with the new size, @@ -1154,7 +1154,7 @@ static void make_thread_read_only(struct vdo_completion *completion) thread->is_read_only = true; listener = thread->listeners; if (thread_id == 0) - uds_log_error_strerror(READ_ONCE(notifier->read_only_error), + vdo_log_error_strerror(READ_ONCE(notifier->read_only_error), "Unrecoverable error, entering read-only mode"); } else { /* We've just finished notifying a listener */ @@ -1329,7 +1329,7 @@ void vdo_enter_recovery_mode(struct vdo *vdo) if (vdo_in_read_only_mode(vdo)) return; - uds_log_info("Entering recovery mode"); + vdo_log_info("Entering recovery mode"); vdo_set_state(vdo, VDO_RECOVERING); } @@ -1382,7 +1382,7 @@ static void set_compression_callback(struct vdo_completion *completion) } } - uds_log_info("compression is %s", (*enable ? "enabled" : "disabled")); + vdo_log_info("compression is %s", (*enable ? "enabled" : "disabled")); *enable = was_enabled; complete_synchronous_action(completion); } diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c index b1e4e604c2c3..b291578f726f 100644 --- a/drivers/md/dm-vdo/vio.c +++ b/drivers/md/dm-vdo/vio.c @@ -131,7 +131,7 @@ int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, */ result = vdo_allocate(1, struct vio, __func__, &vio); if (result != VDO_SUCCESS) { - uds_log_error("metadata vio allocation failure %d", result); + vdo_log_error("metadata vio allocation failure %d", result); return result; } @@ -225,7 +225,7 @@ int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, bytes_added = bio_add_page(bio, page, bytes, offset); if (bytes_added != bytes) { - return uds_log_error_strerror(VDO_BIO_CREATION_FAILED, + return vdo_log_error_strerror(VDO_BIO_CREATION_FAILED, "Could only add %i bytes to bio", bytes_added); } @@ -258,18 +258,18 @@ void update_vio_error_stats(struct vio *vio, const char *format, ...) case VDO_NO_SPACE: atomic64_inc(&vdo->stats.no_space_error_count); - priority = UDS_LOG_DEBUG; + priority = VDO_LOG_DEBUG; break; default: - priority = UDS_LOG_ERR; + priority = VDO_LOG_ERR; } if (!__ratelimit(&error_limiter)) return; va_start(args, format); - uds_vlog_strerror(priority, vio->completion.result, UDS_LOGGING_MODULE_NAME, + vdo_vlog_strerror(priority, vio->completion.result, VDO_LOGGING_MODULE_NAME, format, args); va_end(args); } From 900d337b4677b958c4139e307a9eebb72503d044 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 14 Feb 2024 09:34:46 -0600 Subject: [PATCH 0960/1038] dm vdo string-utils: change from uds_ to vdo_ namespace Signed-off-by: Mike Snitzer Signed-off-by: Chung Chung Signed-off-by: Matthew Sakai --- drivers/md/dm-vdo/dm-vdo-target.c | 2 +- drivers/md/dm-vdo/errors.c | 16 ++++++++-------- drivers/md/dm-vdo/logical-zone.c | 2 +- drivers/md/dm-vdo/slab-depot.c | 6 +++--- drivers/md/dm-vdo/string-utils.c | 2 +- drivers/md/dm-vdo/string-utils.h | 10 +++++----- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 4908996f5224..b7b3eec39522 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -340,7 +340,7 @@ static int join_strings(char **substring_array, size_t array_length, char separa current_position = &output[0]; for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++) { - current_position = uds_append_to_buffer(current_position, + current_position = vdo_append_to_buffer(current_position, output + string_length, "%s", substring_array[i]); *current_position = separator; diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c index 8b2d22381274..6f89eb1c63a3 100644 --- a/drivers/md/dm-vdo/errors.c +++ b/drivers/md/dm-vdo/errors.c @@ -154,19 +154,19 @@ const char *uds_string_error(int errnum, char *buf, size_t buflen) block_name = get_error_info(errnum, &info); if (block_name != NULL) { if (info != NULL) { - buffer = uds_append_to_buffer(buffer, buf_end, "%s: %s", + buffer = vdo_append_to_buffer(buffer, buf_end, "%s: %s", block_name, info->message); } else { - buffer = uds_append_to_buffer(buffer, buf_end, "Unknown %s %d", + buffer = vdo_append_to_buffer(buffer, buf_end, "Unknown %s %d", block_name, errnum); } } else if (info != NULL) { - buffer = uds_append_to_buffer(buffer, buf_end, "%s", info->message); + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", info->message); } else { const char *tmp = system_string_error(errnum, buffer, buf_end - buffer); if (tmp != buffer) - buffer = uds_append_to_buffer(buffer, buf_end, "%s", tmp); + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", tmp); else buffer += strlen(tmp); } @@ -188,19 +188,19 @@ const char *uds_string_error_name(int errnum, char *buf, size_t buflen) block_name = get_error_info(errnum, &info); if (block_name != NULL) { if (info != NULL) { - buffer = uds_append_to_buffer(buffer, buf_end, "%s", info->name); + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", info->name); } else { - buffer = uds_append_to_buffer(buffer, buf_end, "%s %d", + buffer = vdo_append_to_buffer(buffer, buf_end, "%s %d", block_name, errnum); } } else if (info != NULL) { - buffer = uds_append_to_buffer(buffer, buf_end, "%s", info->name); + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", info->name); } else { const char *tmp; tmp = system_string_error(errnum, buffer, buf_end - buffer); if (tmp != buffer) - buffer = uds_append_to_buffer(buffer, buf_end, "%s", tmp); + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", tmp); else buffer += strlen(tmp); } diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c index 258bc55e419b..026f031ffc9e 100644 --- a/drivers/md/dm-vdo/logical-zone.c +++ b/drivers/md/dm-vdo/logical-zone.c @@ -368,6 +368,6 @@ void vdo_dump_logical_zone(const struct logical_zone *zone) (unsigned long long) READ_ONCE(zone->flush_generation), (unsigned long long) READ_ONCE(zone->oldest_active_generation), (unsigned long long) READ_ONCE(zone->notification_generation), - uds_bool_to_string(READ_ONCE(zone->notifying)), + vdo_bool_to_string(READ_ONCE(zone->notifying)), (unsigned long long) READ_ONCE(zone->ios_in_flush_generation)); } diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c index dc9f3d3c3995..46e4721e5b4f 100644 --- a/drivers/md/dm-vdo/slab-depot.c +++ b/drivers/md/dm-vdo/slab-depot.c @@ -3597,8 +3597,8 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) vdo_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", vdo_waitq_num_waiters(&journal->entry_waiters), - uds_bool_to_string(journal->waiting_to_commit), - uds_bool_to_string(journal->updating_slab_summary), + vdo_bool_to_string(journal->waiting_to_commit), + vdo_bool_to_string(journal->updating_slab_summary), (unsigned long long) journal->head, (unsigned long long) journal->unreapable, (unsigned long long) journal->tail, @@ -3606,7 +3606,7 @@ void vdo_dump_block_allocator(const struct block_allocator *allocator) (unsigned long long) journal->summarized, (unsigned long long) journal->last_summarized, (unsigned long long) journal->recovery_lock, - uds_bool_to_string(journal->recovery_lock != 0)); + vdo_bool_to_string(journal->recovery_lock != 0)); /* * Given the frequency with which the locks are just a tiny bit off, it might be * worth dumping all the locks, but that might be too much logging. diff --git a/drivers/md/dm-vdo/string-utils.c b/drivers/md/dm-vdo/string-utils.c index 6cdf018cdaf0..71e44b4683ea 100644 --- a/drivers/md/dm-vdo/string-utils.c +++ b/drivers/md/dm-vdo/string-utils.c @@ -5,7 +5,7 @@ #include "string-utils.h" -char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) +char *vdo_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) { va_list args; size_t n; diff --git a/drivers/md/dm-vdo/string-utils.h b/drivers/md/dm-vdo/string-utils.h index 8275af582cf7..96eecd38b1c2 100644 --- a/drivers/md/dm-vdo/string-utils.h +++ b/drivers/md/dm-vdo/string-utils.h @@ -3,21 +3,21 @@ * Copyright 2023 Red Hat */ -#ifndef UDS_STRING_UTILS_H -#define UDS_STRING_UTILS_H +#ifndef VDO_STRING_UTILS_H +#define VDO_STRING_UTILS_H #include #include /* Utilities related to string manipulation */ -static inline const char *uds_bool_to_string(bool value) +static inline const char *vdo_bool_to_string(bool value) { return value ? "true" : "false"; } /* Append a formatted string to the end of a buffer. */ -char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) +char *vdo_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) __printf(3, 4); -#endif /* UDS_STRING_UTILS_H */ +#endif /* VDO_STRING_UTILS_H */ From d0464d8287364bc754790128fbe95e38837c760b Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 1 Mar 2024 16:54:40 +0800 Subject: [PATCH 0961/1038] dm vdo block-map: Remove stray semicolon Remove the unnecessary semicolon at the end of the for statement. Signed-off-by: Yang Li Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/block-map.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c index e79156dc7cc9..a0a7c1bd634e 100644 --- a/drivers/md/dm-vdo/block-map.c +++ b/drivers/md/dm-vdo/block-map.c @@ -2705,7 +2705,7 @@ void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, cursor->waiter.callback = launch_cursor; acquire_vio_from_pool(cursors->pool, &cursor->waiter); - }; + } } /** From 7eb30fe18fe9eae235f71ecd5c1d24eaeb21a036 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Fri, 1 Mar 2024 18:29:05 -0500 Subject: [PATCH 0962/1038] dm vdo: remove vdo_perform_once Remove obsolete function vdo_perform_once. Instead, initialize necessary module state when loading the module. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dm-vdo-target.c | 2 ++ drivers/md/dm-vdo/status-codes.c | 30 ++++++------------------------ drivers/md/dm-vdo/thread-utils.c | 28 +--------------------------- drivers/md/dm-vdo/thread-utils.h | 4 +--- 4 files changed, 10 insertions(+), 54 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index b7b3eec39522..9128e6cd0844 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -34,6 +34,7 @@ #include "string-utils.h" #include "thread-device.h" #include "thread-registry.h" +#include "thread-utils.h" #include "types.h" #include "vdo.h" #include "vio.h" @@ -2872,6 +2873,7 @@ static int __init vdo_init(void) /* Memory tracking must be initialized first for accurate accounting. */ vdo_memory_init(); + vdo_initialize_threads_mutex(); vdo_initialize_thread_device_registry(); vdo_initialize_device_registry_once(); vdo_log_info("loaded version %s", CURRENT_VERSION); diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index 918e46e7121f..d3493450b169 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -40,10 +40,11 @@ const struct error_info vdo_status_list[] = { { "VDO_INVALID_ADMIN_STATE", "Invalid operation for current state" }, }; -static atomic_t vdo_status_codes_registered = ATOMIC_INIT(0); -static int status_code_registration_result; - -static void do_status_code_registration(void) +/** + * vdo_register_status_codes() - Register the VDO status codes. + * Return: A success or error code. + */ +int vdo_register_status_codes(void) { int result; @@ -53,26 +54,7 @@ static void do_status_code_registration(void) result = uds_register_error_block("VDO Status", VDO_STATUS_CODE_BASE, VDO_STATUS_CODE_BLOCK_END, vdo_status_list, sizeof(vdo_status_list)); - /* - * The following test handles cases where libvdo is statically linked against both the test - * modules and the test driver (because multiple instances of this module call their own - * copy of this function once each, resulting in multiple calls to register_error_block - * which is shared in libuds). - */ - if (result == UDS_DUPLICATE_NAME) - result = UDS_SUCCESS; - - status_code_registration_result = (result == UDS_SUCCESS) ? VDO_SUCCESS : result; -} - -/** - * vdo_register_status_codes() - Register the VDO status codes if needed. - * Return: A success or error code. - */ -int vdo_register_status_codes(void) -{ - vdo_perform_once(&vdo_status_codes_registered, do_status_code_registration); - return status_code_registration_result; + return (result == UDS_SUCCESS) ? VDO_SUCCESS : result; } /** diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c index bd620be61c1d..ec08478dd013 100644 --- a/drivers/md/dm-vdo/thread-utils.c +++ b/drivers/md/dm-vdo/thread-utils.c @@ -17,7 +17,6 @@ static struct hlist_head thread_list; static struct mutex thread_mutex; -static atomic_t thread_once = ATOMIC_INIT(0); struct thread { void (*thread_function)(void *thread_data); @@ -27,31 +26,7 @@ struct thread { struct completion thread_done; }; -#define ONCE_NOT_DONE 0 -#define ONCE_IN_PROGRESS 1 -#define ONCE_COMPLETE 2 - -/* Run a function once only, and record that fact in the atomic value. */ -void vdo_perform_once(atomic_t *once, void (*function)(void)) -{ - for (;;) { - switch (atomic_cmpxchg(once, ONCE_NOT_DONE, ONCE_IN_PROGRESS)) { - case ONCE_NOT_DONE: - function(); - atomic_set_release(once, ONCE_COMPLETE); - return; - case ONCE_IN_PROGRESS: - cond_resched(); - break; - case ONCE_COMPLETE: - return; - default: - return; - } - } -} - -static void thread_init(void) +void vdo_initialize_threads_mutex(void) { mutex_init(&thread_mutex); } @@ -62,7 +37,6 @@ static int thread_starter(void *arg) struct thread *thread = arg; thread->thread_task = current; - vdo_perform_once(&thread_once, thread_init); mutex_lock(&thread_mutex); hlist_add_head(&thread->thread_links, &thread_list); mutex_unlock(&thread_mutex); diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h index f3619a581c5e..687ab43e2cee 100644 --- a/drivers/md/dm-vdo/thread-utils.h +++ b/drivers/md/dm-vdo/thread-utils.h @@ -12,11 +12,9 @@ struct thread; - +void vdo_initialize_threads_mutex(void); int __must_check vdo_create_thread(void (*thread_function)(void *), void *thread_data, const char *name, struct thread **new_thread); void vdo_join_threads(struct thread *thread); -void vdo_perform_once(atomic_t *once_state, void (*function) (void)); - #endif /* UDS_THREADS_H */ From 2a7f925bc25fc3591e6c7f2c1bd561004d744b6d Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Fri, 1 Mar 2024 18:19:40 -0500 Subject: [PATCH 0963/1038] dm vdo: remove meaningless version number constant Also remove related log messages. Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- drivers/md/dm-vdo/dm-vdo-target.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 9128e6cd0844..5a4b0a927f56 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -39,8 +39,6 @@ #include "vdo.h" #include "vio.h" -#define CURRENT_VERSION "8.3.0.65" - enum admin_phases { GROW_LOGICAL_PHASE_START, GROW_LOGICAL_PHASE_GROW_BLOCK_MAP, @@ -2863,8 +2861,6 @@ static void vdo_module_destroy(void) instances.count); vdo_free(instances.words); memset(&instances, 0, sizeof(struct instance_tracker)); - - vdo_log_info("unloaded version %s", CURRENT_VERSION); } static int __init vdo_init(void) @@ -2876,7 +2872,6 @@ static int __init vdo_init(void) vdo_initialize_threads_mutex(); vdo_initialize_thread_device_registry(); vdo_initialize_device_registry_once(); - vdo_log_info("loaded version %s", CURRENT_VERSION); /* Add VDO errors to the set of errors registered by the indexer. */ result = vdo_register_status_codes(); From fae1b0129327343b30be8f254da246a010810959 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Tue, 5 Mar 2024 08:39:13 +0000 Subject: [PATCH 0964/1038] slab: remove PARTIAL_NODE slab_state The PARTIAL_NODE slab_state has gone with SLAB removed, so just remove it. Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slab.h | 1 - tools/include/linux/slab.h | 1 - 2 files changed, 2 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index 6b879d8e8a7a..16c343942456 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -363,7 +363,6 @@ static inline int objs_per_slab(const struct kmem_cache *cache, enum slab_state { DOWN, /* No slab functionality yet */ PARTIAL, /* SLUB: kmem_cache_node available */ - PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */ UP, /* Slab caches usable but not all extras yet */ FULL /* Everything is working */ }; diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h index 311759ea25e9..51b25e9c4ec7 100644 --- a/tools/include/linux/slab.h +++ b/tools/include/linux/slab.h @@ -18,7 +18,6 @@ bool slab_is_available(void); enum slab_state { DOWN, PARTIAL, - PARTIAL_NODE, UP, FULL }; From e7a7681c859643f3f2476b2a28a494877fd89442 Mon Sep 17 00:00:00 2001 From: Qingliang Li Date: Fri, 1 Mar 2024 17:26:57 +0800 Subject: [PATCH 0965/1038] PM: sleep: wakeirq: fix wake irq warning in system suspend When driver uses pm_runtime_force_suspend() as the system suspend callback function and registers the wake irq with reverse enable ordering, the wake irq will be re-enabled when entering system suspend, triggering an 'Unbalanced enable for IRQ xxx' warning. In this scenario, the call sequence during system suspend is as follows: suspend_devices_and_enter() -> dpm_suspend_start() -> dpm_run_callback() -> pm_runtime_force_suspend() -> dev_pm_enable_wake_irq_check() -> dev_pm_enable_wake_irq_complete() -> suspend_enter() -> dpm_suspend_noirq() -> device_wakeup_arm_wake_irqs() -> dev_pm_arm_wake_irq() To fix this issue, complete the setting of WAKE_IRQ_DEDICATED_ENABLED flag in dev_pm_enable_wake_irq_complete() to avoid redundant irq enablement. Fixes: 8527beb12087 ("PM: sleep: wakeirq: fix wake irq arming") Reviewed-by: Dhruva Gole Signed-off-by: Qingliang Li Reviewed-by: Johan Hovold Cc: 5.16+ # 5.16+ Signed-off-by: Rafael J. Wysocki --- drivers/base/power/wakeirq.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 42171f766dcb..5a5a9e978e85 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -313,8 +313,10 @@ void dev_pm_enable_wake_irq_complete(struct device *dev) return; if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED && - wirq->status & WAKE_IRQ_DEDICATED_REVERSE) + wirq->status & WAKE_IRQ_DEDICATED_REVERSE) { enable_irq(wirq->irq); + wirq->status |= WAKE_IRQ_DEDICATED_ENABLED; + } } /** From e8d1b41e69d72c62865bebe8f441163ec00b3d44 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 5 Mar 2024 11:42:56 +0100 Subject: [PATCH 0966/1038] mmc: tmio: avoid concurrent runs of mmc_request_done() With the to-be-fixed commit, the reset_work handler cleared 'host->mrq' outside of the spinlock protected critical section. That leaves a small race window during execution of 'tmio_mmc_reset()' where the done_work handler could grab a pointer to the now invalid 'host->mrq'. Both would use it to call mmc_request_done() causing problems (see link below). However, 'host->mrq' cannot simply be cleared earlier inside the critical section. That would allow new mrqs to come in asynchronously while the actual reset of the controller still needs to be done. So, like 'tmio_mmc_set_ios()', an ERR_PTR is used to prevent new mrqs from coming in but still avoiding concurrency between work handlers. Reported-by: Dirk Behme Closes: https://lore.kernel.org/all/20240220061356.3001761-1-dirk.behme@de.bosch.com/ Fixes: df3ef2d3c92c ("mmc: protect the tmio_mmc driver against a theoretical race") Signed-off-by: Wolfram Sang Tested-by: Dirk Behme Reviewed-by: Dirk Behme Cc: stable@vger.kernel.org # 3.0+ Link: https://lore.kernel.org/r/20240305104423.3177-2-wsa+renesas@sang-engineering.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/tmio_mmc_core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c index be7f18fd4836..c253d176db69 100644 --- a/drivers/mmc/host/tmio_mmc_core.c +++ b/drivers/mmc/host/tmio_mmc_core.c @@ -259,6 +259,8 @@ static void tmio_mmc_reset_work(struct work_struct *work) else mrq->cmd->error = -ETIMEDOUT; + /* No new calls yet, but disallow concurrent tmio_mmc_done_work() */ + host->mrq = ERR_PTR(-EBUSY); host->cmd = NULL; host->data = NULL; From ae5004a40a262d329039b99b62bd3fe7645b66ad Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 26 Feb 2024 22:37:39 +0100 Subject: [PATCH 0967/1038] mmc: wmt-sdmmc: remove an incorrect release_mem_region() call in the .remove function This looks strange to call release_mem_region() in a remove function without any request_mem_region() in the probe or "struct resource" somewhere. So remove the corresponding code. Fixes: 3a96dff0f828 ("mmc: SD/MMC Host Controller for Wondermedia WM8505/WM8650") Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/bb0bb1ed1e18de55e8c0547625bde271e64b8c31.1708983064.git.christophe.jaillet@wanadoo.fr Signed-off-by: Ulf Hansson --- drivers/mmc/host/wmt-sdmmc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c index 77d5f1d24489..860380931b6c 100644 --- a/drivers/mmc/host/wmt-sdmmc.c +++ b/drivers/mmc/host/wmt-sdmmc.c @@ -883,7 +883,6 @@ static void wmt_mci_remove(struct platform_device *pdev) { struct mmc_host *mmc; struct wmt_mci_priv *priv; - struct resource *res; u32 reg_tmp; mmc = platform_get_drvdata(pdev); @@ -911,9 +910,6 @@ static void wmt_mci_remove(struct platform_device *pdev) clk_disable_unprepare(priv->clk_sdmmc); clk_put(priv->clk_sdmmc); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, resource_size(res)); - mmc_free_host(mmc); dev_info(&pdev->dev, "WMT MCI device removed\n"); From 88803989ff6d73155969df94cc84f1ecd28c4d9c Mon Sep 17 00:00:00 2001 From: Yang Xiwen Date: Thu, 29 Feb 2024 09:36:19 +0800 Subject: [PATCH 0968/1038] mmc: core: Use a struct device* as in-param to mmc_of_parse_clk_phase() Parsing dt usually happens very early, sometimes even before the struct mmc_host has been allocated (e.g. dw_mci_probe() and dw_mci_parse_dt() in dw_mmc.c). Looking at the source of mmc_of_parse_clk_phase(), it's actually not needed to have an initialized mmc_host, let's therefore pass a struct device* to it instead. Also update the only current user, sdhci-of-aspeed. Reviewed-by: Paul Menzel Acked-by: Andrew Jeffery Signed-off-by: Yang Xiwen Link: https://lore.kernel.org/r/20240229-b4-mmc-hi3798mv200-v7-1-10c03f316285@outlook.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/host.c | 4 +--- drivers/mmc/host/sdhci-of-aspeed.c | 2 +- include/linux/mmc/host.h | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 7cc9a33d28ca..3aa9dda6fd4f 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -234,10 +234,8 @@ static void mmc_of_parse_timing_phase(struct device *dev, const char *prop, } void -mmc_of_parse_clk_phase(struct mmc_host *host, struct mmc_clk_phase_map *map) +mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map) { - struct device *dev = host->parent; - mmc_of_parse_timing_phase(dev, "clk-phase-legacy", &map->phase[MMC_TIMING_LEGACY]); mmc_of_parse_timing_phase(dev, "clk-phase-mmc-hs", diff --git a/drivers/mmc/host/sdhci-of-aspeed.c b/drivers/mmc/host/sdhci-of-aspeed.c index 42d54532cabe..430c1f90037b 100644 --- a/drivers/mmc/host/sdhci-of-aspeed.c +++ b/drivers/mmc/host/sdhci-of-aspeed.c @@ -435,7 +435,7 @@ static int aspeed_sdhci_probe(struct platform_device *pdev) goto err_sdhci_add; if (dev->phase_desc) - mmc_of_parse_clk_phase(host->mmc, &dev->phase_map); + mmc_of_parse_clk_phase(&pdev->dev, &dev->phase_map); ret = sdhci_add_host(host); if (ret) diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 2f445c651742..5894bf912f7b 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -539,7 +539,7 @@ struct mmc_host *devm_mmc_alloc_host(struct device *dev, int extra); int mmc_add_host(struct mmc_host *); void mmc_remove_host(struct mmc_host *); void mmc_free_host(struct mmc_host *); -void mmc_of_parse_clk_phase(struct mmc_host *host, +void mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map); int mmc_of_parse(struct mmc_host *host); int mmc_of_parse_voltage(struct mmc_host *host, u32 *mask); From dab267cf0e568fec2f6d38bbfe9abac749539ca8 Mon Sep 17 00:00:00 2001 From: Yang Xiwen Date: Thu, 29 Feb 2024 09:36:20 +0800 Subject: [PATCH 0969/1038] mmc: dw_mmc-hi3798cv200: remove MODULE_ALIAS() The alias is not used and should be removed. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Yang Xiwen Link: https://lore.kernel.org/r/20240229-b4-mmc-hi3798mv200-v7-2-10c03f316285@outlook.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/dw_mmc-hi3798cv200.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/mmc/host/dw_mmc-hi3798cv200.c b/drivers/mmc/host/dw_mmc-hi3798cv200.c index e9470c50a348..61923a518369 100644 --- a/drivers/mmc/host/dw_mmc-hi3798cv200.c +++ b/drivers/mmc/host/dw_mmc-hi3798cv200.c @@ -201,4 +201,3 @@ module_platform_driver(dw_mci_hi3798cv200_driver); MODULE_DESCRIPTION("HiSilicon Hi3798CV200 Specific DW-MSHC Driver Extension"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:dwmmc_hi3798cv200"); From 832ff3126527339160990e99c1cbc7a5ddc0576c Mon Sep 17 00:00:00 2001 From: Yang Xiwen Date: Thu, 29 Feb 2024 09:36:21 +0800 Subject: [PATCH 0970/1038] dt-bindings: mmc: dw-mshc-hi3798cv200: convert to YAML convert the legacy txt binding to modern YAML and rename to hisilicon,hi3798cv200-dw-mshc.yaml. No semantic change. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Yang Xiwen Link: https://lore.kernel.org/r/20240229-b4-mmc-hi3798mv200-v7-3-10c03f316285@outlook.com Signed-off-by: Ulf Hansson --- .../bindings/mmc/hi3798cv200-dw-mshc.txt | 40 ---------- .../mmc/hisilicon,hi3798cv200-dw-mshc.yaml | 75 +++++++++++++++++++ 2 files changed, 75 insertions(+), 40 deletions(-) delete mode 100644 Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt create mode 100644 Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml diff --git a/Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt deleted file mode 100644 index a0693b7145f2..000000000000 --- a/Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt +++ /dev/null @@ -1,40 +0,0 @@ -* Hisilicon Hi3798CV200 specific extensions to the Synopsys Designware Mobile - Storage Host Controller - -Read synopsys-dw-mshc.txt for more details - -The Synopsys designware mobile storage host controller is used to interface -a SoC with storage medium such as eMMC or SD/MMC cards. This file documents -differences between the core Synopsys dw mshc controller properties described -by synopsys-dw-mshc.txt and the properties used by the Hisilicon Hi3798CV200 -specific extensions to the Synopsys Designware Mobile Storage Host Controller. - -Required Properties: -- compatible: Should contain "hisilicon,hi3798cv200-dw-mshc". -- clocks: A list of phandle + clock-specifier pairs for the clocks listed - in clock-names. -- clock-names: Should contain the following: - "ciu" - The ciu clock described in synopsys-dw-mshc.txt. - "biu" - The biu clock described in synopsys-dw-mshc.txt. - "ciu-sample" - Hi3798CV200 extended phase clock for ciu sampling. - "ciu-drive" - Hi3798CV200 extended phase clock for ciu driving. - -Example: - - emmc: mmc@9830000 { - compatible = "hisilicon,hi3798cv200-dw-mshc"; - reg = <0x9830000 0x10000>; - interrupts = ; - clocks = <&crg HISTB_MMC_CIU_CLK>, - <&crg HISTB_MMC_BIU_CLK>, - <&crg HISTB_MMC_SAMPLE_CLK>, - <&crg HISTB_MMC_DRV_CLK>; - clock-names = "ciu", "biu", "ciu-sample", "ciu-drive"; - fifo-depth = <256>; - clock-frequency = <200000000>; - cap-mmc-highspeed; - mmc-ddr-1_8v; - mmc-hs200-1_8v; - non-removable; - bus-width = <8>; - }; diff --git a/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml new file mode 100644 index 000000000000..f3dc973cb490 --- /dev/null +++ b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mmc/hisilicon,hi3798cv200-dw-mshc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hisilicon Hi3798CV200 SoC specific extensions to the Synopsys DWMMC controller + +maintainers: + - Yang Xiwen + +properties: + compatible: + enum: + - hisilicon,hi3798cv200-dw-mshc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: bus interface unit clock + - description: card interface unit clock + - description: card input sample phase clock + - description: controller output drive phase clock + + clock-names: + items: + - const: ciu + - const: biu + - const: ciu-sample + - const: ciu-drive + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +allOf: + - $ref: synopsys-dw-mshc-common.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + #include + + mmc@9830000 { + compatible = "hisilicon,hi3798cv200-dw-mshc"; + reg = <0x9830000 0x10000>; + interrupts = ; + clocks = <&crg HISTB_MMC_CIU_CLK>, + <&crg HISTB_MMC_BIU_CLK>, + <&crg HISTB_MMC_SAMPLE_CLK>, + <&crg HISTB_MMC_DRV_CLK>; + clock-names = "ciu", "biu", "ciu-sample", "ciu-drive"; + resets = <&crg 0xa0 4>; + reset-names = "reset"; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins_1 &emmc_pins_2 + &emmc_pins_3 &emmc_pins_4>; + fifo-depth = <256>; + clock-frequency = <200000000>; + cap-mmc-highspeed; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + non-removable; + bus-width = <8>; + }; From cddacdce8ffb168d6cc5ffd0bd5a55153e528135 Mon Sep 17 00:00:00 2001 From: Yang Xiwen Date: Thu, 29 Feb 2024 09:36:22 +0800 Subject: [PATCH 0971/1038] dt-bindings: mmc: hisilicon,hi3798cv200-dw-mshc: add Hi3798MV200 binding Add binding and an extra property for Hi3798MV200 DWMMC specific extension. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Yang Xiwen Link: https://lore.kernel.org/r/20240229-b4-mmc-hi3798mv200-v7-4-10c03f316285@outlook.com Signed-off-by: Ulf Hansson --- .../mmc/hisilicon,hi3798cv200-dw-mshc.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml index f3dc973cb490..41c9b22523e7 100644 --- a/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml +++ b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mmc/hisilicon,hi3798cv200-dw-mshc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Hisilicon Hi3798CV200 SoC specific extensions to the Synopsys DWMMC controller +title: Hisilicon HiSTB SoCs specific extensions to the Synopsys DWMMC controller maintainers: - Yang Xiwen @@ -13,6 +13,7 @@ properties: compatible: enum: - hisilicon,hi3798cv200-dw-mshc + - hisilicon,hi3798mv200-dw-mshc reg: maxItems: 1 @@ -34,6 +35,15 @@ properties: - const: ciu-sample - const: ciu-drive + hisilicon,sap-dll-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + DWMMC core on Hi3798MV2x SoCs has a delay-locked-loop(DLL) attached to card data input path. + It is integrated into CRG core on the SoC and has to be controlled during tuning. + items: + - description: A phandle pointed to the CRG syscon node + - description: Sample DLL register offset in CRG address space + required: - compatible - reg @@ -44,6 +54,18 @@ required: allOf: - $ref: synopsys-dw-mshc-common.yaml# + - if: + properties: + compatible: + contains: + const: hisilicon,hi3798mv200-dw-mshc + then: + required: + - hisilicon,sap-dll-reg + else: + properties: + hisilicon,sap-dll-reg: false + unevaluatedProperties: false examples: From 25d043841db17b7ad7d850caf4652bd8b3d580ed Mon Sep 17 00:00:00 2001 From: Yang Xiwen Date: Thu, 29 Feb 2024 09:36:23 +0800 Subject: [PATCH 0972/1038] mmc: dw_mmc: add support for hi3798mv200 Add support for Hi3798MV200 specific extension. Signed-off-by: Yang Xiwen Link: https://lore.kernel.org/r/20240229-b4-mmc-hi3798mv200-v7-5-10c03f316285@outlook.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/Kconfig | 9 + drivers/mmc/host/Makefile | 1 + drivers/mmc/host/dw_mmc-hi3798mv200.c | 251 ++++++++++++++++++++++++++ 3 files changed, 261 insertions(+) create mode 100644 drivers/mmc/host/dw_mmc-hi3798mv200.c diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 81f2c4e05287..aebc587f77a7 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -798,6 +798,15 @@ config MMC_DW_HI3798CV200 Synopsys DesignWare Memory Card Interface driver. Select this option for platforms based on HiSilicon Hi3798CV200 SoC. +config MMC_DW_HI3798MV200 + tristate "Hi3798MV200 specific extensions for Synopsys DW Memory Card Interface" + depends on MMC_DW + select MMC_DW_PLTFM + help + This selects support for HiSilicon Hi3798MV200 SoC specific extensions to the + Synopsys DesignWare Memory Card Interface driver. Select this option + for platforms based on HiSilicon Hi3798MV200 SoC. + config MMC_DW_K3 tristate "K3 specific extensions for Synopsys DW Memory Card Interface" depends on MMC_DW diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile index d0be4465f3ec..f53f86d200ac 100644 --- a/drivers/mmc/host/Makefile +++ b/drivers/mmc/host/Makefile @@ -51,6 +51,7 @@ obj-$(CONFIG_MMC_DW_PLTFM) += dw_mmc-pltfm.o obj-$(CONFIG_MMC_DW_BLUEFIELD) += dw_mmc-bluefield.o obj-$(CONFIG_MMC_DW_EXYNOS) += dw_mmc-exynos.o obj-$(CONFIG_MMC_DW_HI3798CV200) += dw_mmc-hi3798cv200.o +obj-$(CONFIG_MMC_DW_HI3798MV200) += dw_mmc-hi3798mv200.o obj-$(CONFIG_MMC_DW_K3) += dw_mmc-k3.o obj-$(CONFIG_MMC_DW_PCI) += dw_mmc-pci.o obj-$(CONFIG_MMC_DW_ROCKCHIP) += dw_mmc-rockchip.o diff --git a/drivers/mmc/host/dw_mmc-hi3798mv200.c b/drivers/mmc/host/dw_mmc-hi3798mv200.c new file mode 100644 index 000000000000..989ae8dda722 --- /dev/null +++ b/drivers/mmc/host/dw_mmc-hi3798mv200.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Modified from dw_mmc-hi3798cv200.c + * + * Copyright (c) 2024 Yang Xiwen + * Copyright (c) 2018 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "dw_mmc.h" +#include "dw_mmc-pltfm.h" + +#define SDMMC_TUNING_CTRL 0x118 +#define SDMMC_TUNING_FIND_EDGE BIT(5) + +#define ALL_INT_CLR 0x1ffff + +/* DLL ctrl reg */ +#define SAP_DLL_CTRL_DLLMODE BIT(16) + +struct dw_mci_hi3798mv200_priv { + struct clk *sample_clk; + struct clk *drive_clk; + struct regmap *crg_reg; + u32 sap_dll_offset; + struct mmc_clk_phase_map phase_map; +}; + +static void dw_mci_hi3798mv200_set_ios(struct dw_mci *host, struct mmc_ios *ios) +{ + struct dw_mci_hi3798mv200_priv *priv = host->priv; + struct mmc_clk_phase phase = priv->phase_map.phase[ios->timing]; + u32 val; + + val = mci_readl(host, ENABLE_SHIFT); + if (ios->timing == MMC_TIMING_MMC_DDR52 + || ios->timing == MMC_TIMING_UHS_DDR50) + val |= SDMMC_ENABLE_PHASE; + else + val &= ~SDMMC_ENABLE_PHASE; + mci_writel(host, ENABLE_SHIFT, val); + + val = mci_readl(host, DDR_REG); + if (ios->timing == MMC_TIMING_MMC_HS400) + val |= SDMMC_DDR_HS400; + else + val &= ~SDMMC_DDR_HS400; + mci_writel(host, DDR_REG, val); + + if (clk_set_rate(host->ciu_clk, ios->clock)) + dev_warn(host->dev, "Failed to set rate to %u\n", ios->clock); + else + /* + * CLK_MUX_ROUND_NEAREST is enabled for this clock + * The actual clock rate is not what we set, but a rounded value + * so we should get the rate once again + */ + host->bus_hz = clk_get_rate(host->ciu_clk); + + if (phase.valid) { + clk_set_phase(priv->drive_clk, phase.out_deg); + clk_set_phase(priv->sample_clk, phase.in_deg); + } else { + dev_warn(host->dev, + "The phase entry for timing mode %d is missing in device tree.\n", + ios->timing); + } +} + +static inline int dw_mci_hi3798mv200_enable_tuning(struct dw_mci_slot *slot) +{ + struct dw_mci_hi3798mv200_priv *priv = slot->host->priv; + + return regmap_clear_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE); +} + +static inline int dw_mci_hi3798mv200_disable_tuning(struct dw_mci_slot *slot) +{ + struct dw_mci_hi3798mv200_priv *priv = slot->host->priv; + + return regmap_set_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE); +} + +static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot, + u32 opcode) +{ + static const int degrees[] = { 0, 45, 90, 135, 180, 225, 270, 315 }; + struct dw_mci *host = slot->host; + struct dw_mci_hi3798mv200_priv *priv = host->priv; + int raise_point = -1, fall_point = -1, mid; + int err, prev_err = -1; + int found = 0; + int regval; + int i; + int ret; + + ret = dw_mci_hi3798mv200_enable_tuning(slot); + if (ret < 0) + return ret; + + for (i = 0; i < ARRAY_SIZE(degrees); i++) { + clk_set_phase(priv->sample_clk, degrees[i]); + mci_writel(host, RINTSTS, ALL_INT_CLR); + + /* + * HiSilicon implemented a tuning mechanism. + * It needs special interaction with the DLL. + * + * Treat edge(flip) found as an error too. + */ + err = mmc_send_tuning(slot->mmc, opcode, NULL); + regval = mci_readl(host, TUNING_CTRL); + if (err || (regval & SDMMC_TUNING_FIND_EDGE)) + err = 1; + else + found = 1; + + if (i > 0) { + if (err && !prev_err) + fall_point = i - 1; + if (!err && prev_err) + raise_point = i; + } + + if (raise_point != -1 && fall_point != -1) + goto tuning_out; + + prev_err = err; + err = 0; + } + +tuning_out: + ret = dw_mci_hi3798mv200_disable_tuning(slot); + if (ret < 0) + return ret; + + if (found) { + if (raise_point == -1) + raise_point = 0; + if (fall_point == -1) + fall_point = ARRAY_SIZE(degrees) - 1; + if (fall_point < raise_point) { + if ((raise_point + fall_point) > + (ARRAY_SIZE(degrees) - 1)) + mid = fall_point / 2; + else + mid = (raise_point + ARRAY_SIZE(degrees) - 1) / 2; + } else { + mid = (raise_point + fall_point) / 2; + } + + /* + * We don't care what timing we are tuning for, + * simply use the same phase for all timing needs tuning. + */ + priv->phase_map.phase[MMC_TIMING_MMC_HS200].in_deg = degrees[mid]; + priv->phase_map.phase[MMC_TIMING_MMC_HS400].in_deg = degrees[mid]; + priv->phase_map.phase[MMC_TIMING_UHS_SDR104].in_deg = degrees[mid]; + + clk_set_phase(priv->sample_clk, degrees[mid]); + dev_dbg(host->dev, "Tuning clk_sample[%d, %d], set[%d]\n", + raise_point, fall_point, degrees[mid]); + ret = 0; + } else { + dev_err(host->dev, "No valid clk_sample shift!\n"); + ret = -EINVAL; + } + + mci_writel(host, RINTSTS, ALL_INT_CLR); + + return ret; +} + +static int dw_mci_hi3798mv200_init(struct dw_mci *host) +{ + struct dw_mci_hi3798mv200_priv *priv; + struct device_node *np = host->dev->of_node; + int ret; + + priv = devm_kzalloc(host->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + mmc_of_parse_clk_phase(host->dev, &priv->phase_map); + + priv->sample_clk = devm_clk_get_enabled(host->dev, "ciu-sample"); + if (IS_ERR(priv->sample_clk)) + return dev_err_probe(host->dev, PTR_ERR(priv->sample_clk), + "failed to get enabled ciu-sample clock\n"); + + priv->drive_clk = devm_clk_get_enabled(host->dev, "ciu-drive"); + if (IS_ERR(priv->drive_clk)) + return dev_err_probe(host->dev, PTR_ERR(priv->drive_clk), + "failed to get enabled ciu-drive clock\n"); + + priv->crg_reg = syscon_regmap_lookup_by_phandle(np, "hisilicon,sap-dll-reg"); + if (IS_ERR(priv->crg_reg)) + return dev_err_probe(host->dev, PTR_ERR(priv->crg_reg), + "failed to get CRG reg\n"); + + ret = of_property_read_u32_index(np, "hisilicon,sap-dll-reg", 1, &priv->sap_dll_offset); + if (ret) + return dev_err_probe(host->dev, ret, "failed to get sample DLL register offset\n"); + + host->priv = priv; + return 0; +} + +static const struct dw_mci_drv_data hi3798mv200_data = { + .common_caps = MMC_CAP_CMD23, + .init = dw_mci_hi3798mv200_init, + .set_ios = dw_mci_hi3798mv200_set_ios, + .execute_tuning = dw_mci_hi3798mv200_execute_tuning_mix_mode, +}; + +static const struct of_device_id dw_mci_hi3798mv200_match[] = { + { .compatible = "hisilicon,hi3798mv200-dw-mshc" }, + {}, +}; + +static int dw_mci_hi3798mv200_probe(struct platform_device *pdev) +{ + return dw_mci_pltfm_register(pdev, &hi3798mv200_data); +} + +static void dw_mci_hi3798mv200_remove(struct platform_device *pdev) +{ + dw_mci_pltfm_remove(pdev); +} + +MODULE_DEVICE_TABLE(of, dw_mci_hi3798mv200_match); +static struct platform_driver dw_mci_hi3798mv200_driver = { + .probe = dw_mci_hi3798mv200_probe, + .remove_new = dw_mci_hi3798mv200_remove, + .driver = { + .name = "dwmmc_hi3798mv200", + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .of_match_table = dw_mci_hi3798mv200_match, + }, +}; +module_platform_driver(dw_mci_hi3798mv200_driver); + +MODULE_DESCRIPTION("HiSilicon Hi3798MV200 Specific DW-MSHC Driver Extension"); +MODULE_LICENSE("GPL"); From 67e90a7deacb1243d87567ec7b89e34ff639d97d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 4 Mar 2024 19:56:06 +0200 Subject: [PATCH 0973/1038] mmc: dw_mmc: Remove unused of_gpio.h of_gpio.h is deprecated and subject to remove. The driver doesn't use it, simply remove the unused header. Signed-off-by: Andy Shevchenko Reviewed-by: Sam Protsenko Link: https://lore.kernel.org/r/20240304175606.1200076-1-andriy.shevchenko@linux.intel.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/dw_mmc-exynos.c | 1 - drivers/mmc/host/dw_mmc.c | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c index 698408e8bad0..6dc057718d2c 100644 --- a/drivers/mmc/host/dw_mmc-exynos.c +++ b/drivers/mmc/host/dw_mmc-exynos.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c index 829af2c98a44..8e2d676b9239 100644 --- a/drivers/mmc/host/dw_mmc.c +++ b/drivers/mmc/host/dw_mmc.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include "dw_mmc.h" From a5f372a1bbaaf07b4028d0d2002c460958c46911 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 4 Mar 2024 20:48:30 +0200 Subject: [PATCH 0974/1038] mmc: mmc_spi: Don't mention DMA direction Since driver doesn't handle any DMA requests, drop any use of DMA bits, such as DMA direction. Instead, use MMC_DATA_WRITE flag directly. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20240304184830.1319526-1-andriy.shevchenko@linux.intel.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/mmc_spi.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c index b8dda8160c4e..922275de0593 100644 --- a/drivers/mmc/host/mmc_spi.c +++ b/drivers/mmc/host/mmc_spi.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -510,10 +509,7 @@ mmc_spi_command_send(struct mmc_spi_host *host, * so we explicitly initialize it to all ones on RX paths. */ static void -mmc_spi_setup_data_message( - struct mmc_spi_host *host, - bool multiple, - enum dma_data_direction direction) +mmc_spi_setup_data_message(struct mmc_spi_host *host, bool multiple, bool write) { struct spi_transfer *t; struct scratch *scratch = host->data; @@ -523,7 +519,7 @@ mmc_spi_setup_data_message( /* for reads, readblock() skips 0xff bytes before finding * the token; for writes, this transfer issues that token. */ - if (direction == DMA_TO_DEVICE) { + if (write) { t = &host->token; memset(t, 0, sizeof(*t)); t->len = 1; @@ -547,7 +543,7 @@ mmc_spi_setup_data_message( t = &host->crc; memset(t, 0, sizeof(*t)); t->len = 2; - if (direction == DMA_TO_DEVICE) { + if (write) { /* the actual CRC may get written later */ t->tx_buf = &scratch->crc_val; } else { @@ -570,10 +566,10 @@ mmc_spi_setup_data_message( * the next token (next data block, or STOP_TRAN). We can try to * minimize I/O ops by using a single read to collect end-of-busy. */ - if (multiple || direction == DMA_TO_DEVICE) { + if (multiple || write) { t = &host->early_status; memset(t, 0, sizeof(*t)); - t->len = (direction == DMA_TO_DEVICE) ? sizeof(scratch->status) : 1; + t->len = write ? sizeof(scratch->status) : 1; t->tx_buf = host->ones; t->rx_buf = scratch->status; t->cs_change = 1; @@ -777,15 +773,15 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, { struct spi_device *spi = host->spi; struct spi_transfer *t; - enum dma_data_direction direction = mmc_get_dma_dir(data); struct scatterlist *sg; unsigned n_sg; bool multiple = (data->blocks > 1); - const char *write_or_read = (direction == DMA_TO_DEVICE) ? "write" : "read"; + bool write = (data->flags & MMC_DATA_WRITE); + const char *write_or_read = write ? "write" : "read"; u32 clock_rate; unsigned long timeout; - mmc_spi_setup_data_message(host, multiple, direction); + mmc_spi_setup_data_message(host, multiple, write); t = &host->t; if (t->speed_hz) @@ -807,7 +803,7 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, /* allow pio too; we don't allow highmem */ kmap_addr = kmap(sg_page(sg)); - if (direction == DMA_TO_DEVICE) + if (write) t->tx_buf = kmap_addr + sg->offset; else t->rx_buf = kmap_addr + sg->offset; @@ -818,7 +814,7 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, dev_dbg(&spi->dev, " %s block, %d bytes\n", write_or_read, t->len); - if (direction == DMA_TO_DEVICE) + if (write) status = mmc_spi_writeblock(host, t, timeout); else status = mmc_spi_readblock(host, t, timeout); @@ -833,7 +829,9 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, } /* discard mappings */ - if (direction == DMA_FROM_DEVICE) + if (write) + /* nothing to do */; + else flush_dcache_page(sg_page(sg)); kunmap(sg_page(sg)); @@ -850,7 +848,7 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, * that can affect the STOP_TRAN logic. Complete (and current) * MMC specs should sort that out before Linux starts using CMD23. */ - if (direction == DMA_TO_DEVICE && multiple) { + if (write && multiple) { struct scratch *scratch = host->data; int tmp; const unsigned statlen = sizeof(scratch->status); From 0cd8fd15a75e82ef249e99b3d80e2012a729d028 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 5 Mar 2024 11:56:24 +0100 Subject: [PATCH 0975/1038] mmc: tmio: comment the ERR_PTR usage in this driver It is not super obvious why the driver sometimes uses an ERR_PTR for the current mrq. Explain why in comments. Signed-off-by: Wolfram Sang Link: https://lore.kernel.org/r/20240305105623.3379-2-wsa+renesas@sang-engineering.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/tmio_mmc_core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c index c253d176db69..93e912afd3ae 100644 --- a/drivers/mmc/host/tmio_mmc_core.c +++ b/drivers/mmc/host/tmio_mmc_core.c @@ -972,6 +972,7 @@ static void tmio_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) return; } + /* Disallow new mrqs and work handlers to run */ host->mrq = ERR_PTR(-EBUSY); spin_unlock_irqrestore(&host->lock, flags); @@ -1006,8 +1007,9 @@ static void tmio_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) "%s.%d: IOS interrupted: clk %u, mode %u", current->comm, task_pid_nr(current), ios->clock, ios->power_mode); - host->mrq = NULL; + /* Ready for new mrqs */ + host->mrq = NULL; host->clk_cache = ios->clock; mutex_unlock(&host->ios_lock); From 32abd250879a272fd96da48db59fde5ef71946cf Mon Sep 17 00:00:00 2001 From: Flavio Suligoi Date: Mon, 4 Mar 2024 12:00:22 +0100 Subject: [PATCH 0976/1038] thermal: core: Remove excess empty line from a comment The first and the third lines of the kerneldoc comment for: thermal_zone_device_set_polling() belong to the same sentences, so join them together. Signed-off-by: Flavio Suligoi [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index f7a7d43809e7..34a31bc72023 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -273,7 +273,6 @@ static int __init thermal_register_governors(void) /* * Zone update section: main control loop applied to each zone while monitoring - * * in polling mode. The monitoring is done using a workqueue. * Same update may be done on a zone by calling thermal_zone_device_update(). * From a51ab63b297ce9e26e3ffb9be896018a42d5f32f Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Thu, 8 Feb 2024 11:46:28 +0800 Subject: [PATCH 0977/1038] ACPI: CPPC: enable AMD CPPC V2 support for family 17h processors As there are some AMD processors which only support CPPC V2 firmware and BIOS implementation, the amd_pstate driver will be failed to load when system booting with below kernel warning message: [ 0.477523] amd_pstate: the _CPC object is not present in SBIOS or ACPI disabled To make the amd_pstate driver can be loaded on those TR40 processors, it needs to match x86_model from 0x30 to 0x7F for family 17H. With the change, the system can load amd_pstate driver as expected. Reviewed-by: Mario Limonciello Reported-by: Gino Badouri Closes: https://bugzilla.kernel.org/show_bug.cgi?id=218171 Fixes: fbd74d1689 ("ACPI: CPPC: Fix enabling CPPC on AMD systems with shared memory") Signed-off-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/acpi/cppc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 8d8752b44f11..ff8f25faca3d 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -20,7 +20,7 @@ bool cpc_supported_by_cpu(void) (boot_cpu_data.x86_model >= 0x20 && boot_cpu_data.x86_model <= 0x2f))) return true; else if (boot_cpu_data.x86 == 0x17 && - boot_cpu_data.x86_model >= 0x70 && boot_cpu_data.x86_model <= 0x7f) + boot_cpu_data.x86_model >= 0x30 && boot_cpu_data.x86_model <= 0x7f) return true; return boot_cpu_has(X86_FEATURE_CPPC); } From 8ae438f5ff168cd0e3ba6e3cef7240fa5d110528 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 4 Mar 2024 19:35:33 +0200 Subject: [PATCH 0978/1038] gpiolib: Deduplicate cleanup for-loop in gpiochip_add_data_with_key() There is no need to repeat for-loop twice in the error path in gpiochip_add_data_with_key(). Deduplicate it. While at it, rename loop variable to be more specific and avoid ambguity. It also properly unwinds the SRCU, i.e. in reversed order of allocating. Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index e2e583b40207..ce94e37bcbee 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -861,7 +861,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, struct lock_class_key *request_key) { struct gpio_device *gdev; - unsigned int i, j; + unsigned int desc_index; int base = 0; int ret = 0; @@ -965,8 +965,8 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, } } - for (i = 0; i < gc->ngpio; i++) - gdev->descs[i].gdev = gdev; + for (desc_index = 0; desc_index < gc->ngpio; desc_index++) + gdev->descs[desc_index].gdev = gdev; BLOCKING_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); @@ -992,19 +992,16 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_cleanup_gdev_srcu; - for (i = 0; i < gc->ngpio; i++) { - struct gpio_desc *desc = &gdev->descs[i]; + for (desc_index = 0; desc_index < gc->ngpio; desc_index++) { + struct gpio_desc *desc = &gdev->descs[desc_index]; ret = init_srcu_struct(&desc->srcu); - if (ret) { - for (j = 0; j < i; j++) - cleanup_srcu_struct(&gdev->descs[j].srcu); - goto err_free_gpiochip_mask; - } + if (ret) + goto err_cleanup_desc_srcu; - if (gc->get_direction && gpiochip_line_is_valid(gc, i)) { + if (gc->get_direction && gpiochip_line_is_valid(gc, desc_index)) { assign_bit(FLAG_IS_OUT, - &desc->flags, !gc->get_direction(gc, i)); + &desc->flags, !gc->get_direction(gc, desc_index)); } else { assign_bit(FLAG_IS_OUT, &desc->flags, !gc->direction_input); @@ -1061,9 +1058,8 @@ err_free_hogs: err_remove_of_chip: of_gpiochip_remove(gc); err_cleanup_desc_srcu: - for (i = 0; i < gdev->ngpio; i++) - cleanup_srcu_struct(&gdev->descs[i].srcu); -err_free_gpiochip_mask: + while (desc_index--) + cleanup_srcu_struct(&gdev->descs[desc_index].srcu); gpiochip_free_valid_mask(gc); err_cleanup_gdev_srcu: cleanup_srcu_struct(&gdev->srcu); From 297dc37a1e65f2f7378c2f755516d2c8c1db2c32 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Tue, 5 Mar 2024 08:21:24 +0800 Subject: [PATCH 0979/1038] selftest: gpio: remove obsolete gpio-mockup test With the removal of the ARCH_NR_GPIOS, the number of available GPIOs is effectively unlimited, causing the gpio-mockup module load failure test that overflowed the number of GPIOs to unexpectedly succeed, and so fail. The test is no longer relevant so remove it. Promote the "no lines defined" test so there is still one load failure test in the basic set. Fixes: 7b61212f2a07 ("gpiolib: Get rid of ARCH_NR_GPIOS") Reported-by: Pengfei Xu Reported-by: Yi Lai Closes: https://lore.kernel.org/linux-gpio/ZC6OHBjdwBdT4sSb@xpf.sh.intel.com/ Signed-off-by: Kent Gibson Acked-by: Christophe Leroy Signed-off-by: Bartosz Golaszewski --- tools/testing/selftests/gpio/gpio-mockup.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/gpio/gpio-mockup.sh b/tools/testing/selftests/gpio/gpio-mockup.sh index 0d6c5f7f95d2..fc2dd4c24d06 100755 --- a/tools/testing/selftests/gpio/gpio-mockup.sh +++ b/tools/testing/selftests/gpio/gpio-mockup.sh @@ -377,13 +377,10 @@ if [ "$full_test" ]; then insmod_test "0,32,32,44,-1,22,-1,31" 32 12 22 31 fi echo "2. Module load error tests" -echo "2.1 gpio overflow" -# Currently: The max number of gpio(1024) is defined in arm architecture. -insmod_test "-1,1024" +echo "2.1 no lines defined" +insmod_test "0,0" if [ "$full_test" ]; then - echo "2.2 no lines defined" - insmod_test "0,0" - echo "2.3 ignore range overlap" + echo "2.2 ignore range overlap" insmod_test "0,32,0,1" 32 insmod_test "0,32,1,5" 32 insmod_test "0,32,30,35" 32 From 4fe6207508d6525e87149348daeafde152ba672c Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 5 Mar 2024 16:00:45 -0300 Subject: [PATCH 0980/1038] regulator: core: make regulator_class constant Since commit 43a7206b0963 ("driver core: class: make class_register() take a const *"), the driver core allows for struct class to be in read-only memory, so move the regulator_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Link: https://msgid.link/r/20240305-class_cleanup-regulator-v1-1-4950345d6d8f@marliere.net Signed-off-by: Mark Brown --- drivers/regulator/core.c | 2 +- drivers/regulator/internal.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 17c98c5fa45f..d019ca6dee9b 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -5890,7 +5890,7 @@ static const struct dev_pm_ops __maybe_unused regulator_pm_ops = { }; #endif -struct class regulator_class = { +const struct class regulator_class = { .name = "regulator", .dev_release = regulator_dev_release, .dev_groups = regulator_dev_groups, diff --git a/drivers/regulator/internal.h b/drivers/regulator/internal.h index fb4433068d29..77a502141089 100644 --- a/drivers/regulator/internal.h +++ b/drivers/regulator/internal.h @@ -58,7 +58,7 @@ struct regulator { struct dentry *debugfs; }; -extern struct class regulator_class; +extern const struct class regulator_class; static inline struct regulator_dev *dev_to_rdev(struct device *dev) { From 6b8e288f49570ee2ba15a2a07c2ebf7ad2210422 Mon Sep 17 00:00:00 2001 From: He Rongguang Date: Mon, 4 Mar 2024 14:14:06 +0800 Subject: [PATCH 0981/1038] cpuidle: ACPI/intel: fix MWAIT hint target C-state computation According to x86 spec ([1] and [2]), MWAIT hint_address[7:4] plus 1 is the corresponding C-state, and 0xF means C0. ACPI C-state table usually only contains C1+, but nothing prevents ACPI firmware from presenting a C-state (maybe C1+) but using MWAIT address C0 (i.e., 0xF in ACPI FFH MWAIT hint address). And if this is the case, Linux erroneously treat this cstate as C16, while actually this should be valid C0 instead of C16, as per the specifications. Since ACPI firmware is out of Linux kernel scope, fix the kernel handling of 0xF ->(to) C0 in this situation. This is found when a tweaked ACPI C-state table is presented by Qemu to VM. Also modify the intel_idle case for code consistency. [1]. Intel SDM Vol 2, Table 4-11. MWAIT Hints Register (EAX): "Value of 0 means C1; 1 means C2 and so on Value of 01111B means C0". [2]. AMD manual Vol 3, MWAIT: "The processor C-state is EAX[7:4]+1, so to request C0 is to place the value F in EAX[7:4] and to request C1 is to place the value 0 in EAX[7:4].". Signed-off-by: He Rongguang [ rjw: Subject and changelog edits, whitespace fixups ] Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/acpi/cstate.c | 4 ++-- drivers/idle/intel_idle.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c index 401808b47af3..f3ffd0a3a012 100644 --- a/arch/x86/kernel/acpi/cstate.c +++ b/arch/x86/kernel/acpi/cstate.c @@ -131,8 +131,8 @@ static long acpi_processor_ffh_cstate_probe_cpu(void *_cx) cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx); /* Check whether this particular cx_type (in CST) is supported or not */ - cstate_type = ((cx->address >> MWAIT_SUBSTATE_SIZE) & - MWAIT_CSTATE_MASK) + 1; + cstate_type = (((cx->address >> MWAIT_SUBSTATE_SIZE) & + MWAIT_CSTATE_MASK) + 1) & MWAIT_CSTATE_MASK; edx_part = edx >> (cstate_type * MWAIT_SUBSTATE_SIZE); num_cstate_subtype = edx_part & MWAIT_SUBSTATE_MASK; diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index bcf1198e8991..e486027f8b07 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1934,7 +1934,8 @@ static void __init spr_idle_state_table_update(void) static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) { - unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; + unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & + MWAIT_CSTATE_MASK; unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & MWAIT_SUBSTATE_MASK; From 2f4a4d63a193be6fd530d180bb13c3592052904c Mon Sep 17 00:00:00 2001 From: Jarred White Date: Fri, 1 Mar 2024 11:25:59 -0800 Subject: [PATCH 0982/1038] ACPI: CPPC: Use access_width over bit_width for system memory accesses To align with ACPI 6.3+, since bit_width can be any 8-bit value, it cannot be depended on to be always on a clean 8b boundary. This was uncovered on the Cobalt 100 platform. SError Interrupt on CPU26, code 0xbe000011 -- SError CPU: 26 PID: 1510 Comm: systemd-udevd Not tainted 5.15.2.1-13 #1 Hardware name: MICROSOFT CORPORATION, BIOS MICROSOFT CORPORATION pstate: 62400009 (nZCv daif +PAN -UAO +TCO -DIT -SSBS BTYPE=--) pc : cppc_get_perf_caps+0xec/0x410 lr : cppc_get_perf_caps+0xe8/0x410 sp : ffff8000155ab730 x29: ffff8000155ab730 x28: ffff0080139d0038 x27: ffff0080139d0078 x26: 0000000000000000 x25: ffff0080139d0058 x24: 00000000ffffffff x23: ffff0080139d0298 x22: ffff0080139d0278 x21: 0000000000000000 x20: ffff00802b251910 x19: ffff0080139d0000 x18: ffffffffffffffff x17: 0000000000000000 x16: ffffdc7e111bad04 x15: ffff00802b251008 x14: ffffffffffffffff x13: ffff013f1fd63300 x12: 0000000000000006 x11: ffffdc7e128f4420 x10: 0000000000000000 x9 : ffffdc7e111badec x8 : ffff00802b251980 x7 : 0000000000000000 x6 : ffff0080139d0028 x5 : 0000000000000000 x4 : ffff0080139d0018 x3 : 00000000ffffffff x2 : 0000000000000008 x1 : ffff8000155ab7a0 x0 : 0000000000000000 Kernel panic - not syncing: Asynchronous SError Interrupt CPU: 26 PID: 1510 Comm: systemd-udevd Not tainted 5.15.2.1-13 #1 Hardware name: MICROSOFT CORPORATION, BIOS MICROSOFT CORPORATION Call trace: dump_backtrace+0x0/0x1e0 show_stack+0x24/0x30 dump_stack_lvl+0x8c/0xb8 dump_stack+0x18/0x34 panic+0x16c/0x384 add_taint+0x0/0xc0 arm64_serror_panic+0x7c/0x90 arm64_is_fatal_ras_serror+0x34/0xa4 do_serror+0x50/0x6c el1h_64_error_handler+0x40/0x74 el1h_64_error+0x7c/0x80 cppc_get_perf_caps+0xec/0x410 cppc_cpufreq_cpu_init+0x74/0x400 [cppc_cpufreq] cpufreq_online+0x2dc/0xa30 cpufreq_add_dev+0xc0/0xd4 subsys_interface_register+0x134/0x14c cpufreq_register_driver+0x1b0/0x354 cppc_cpufreq_init+0x1a8/0x1000 [cppc_cpufreq] do_one_initcall+0x50/0x250 do_init_module+0x60/0x27c load_module+0x2300/0x2570 __do_sys_finit_module+0xa8/0x114 __arm64_sys_finit_module+0x2c/0x3c invoke_syscall+0x78/0x100 el0_svc_common.constprop.0+0x180/0x1a0 do_el0_svc+0x84/0xa0 el0_svc+0x2c/0xc0 el0t_64_sync_handler+0xa4/0x12c el0t_64_sync+0x1a4/0x1a8 Instead, use access_width to determine the size and use the offset and width to shift and mask the bits to read/write out. Make sure to add a check for system memory since pcc redefines the access_width to subspace id. If access_width is not set, then fall back to using bit_width. Signed-off-by: Jarred White Reviewed-by: Easwar Hariharan Cc: 5.15+ # 5.15+ [ rjw: Subject and changelog edits, comment adjustments ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/cppc_acpi.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index d155a86a8614..b954ce3638a9 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -166,6 +166,13 @@ show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time); +/* Check for valid access_width, otherwise, fallback to using bit_width */ +#define GET_BIT_WIDTH(reg) ((reg)->access_width ? (8 << ((reg)->access_width - 1)) : (reg)->bit_width) + +/* Shift and apply the mask for CPC reads/writes */ +#define MASK_VAL(reg, val) ((val) >> ((reg)->bit_offset & \ + GENMASK(((reg)->bit_width), 0))) + static ssize_t show_feedback_ctrs(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -780,6 +787,7 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) } else if (gas_t->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { if (gas_t->address) { void __iomem *addr; + size_t access_width; if (!osc_cpc_flexible_adr_space_confirmed) { pr_debug("Flexible address space capability not supported\n"); @@ -787,7 +795,8 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) goto out_free; } - addr = ioremap(gas_t->address, gas_t->bit_width/8); + access_width = GET_BIT_WIDTH(gas_t) / 8; + addr = ioremap(gas_t->address, access_width); if (!addr) goto out_free; cpc_ptr->cpc_regs[i-2].sys_mem_vaddr = addr; @@ -983,6 +992,7 @@ int __weak cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) { void __iomem *vaddr = NULL; + int size; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); struct cpc_reg *reg = ®_res->cpc_entry.reg; @@ -994,7 +1004,7 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) *val = 0; if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) { - u32 width = 8 << (reg->access_width - 1); + u32 width = GET_BIT_WIDTH(reg); u32 val_u32; acpi_status status; @@ -1018,7 +1028,9 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) return acpi_os_read_memory((acpi_physical_address)reg->address, val, reg->bit_width); - switch (reg->bit_width) { + size = GET_BIT_WIDTH(reg); + + switch (size) { case 8: *val = readb_relaxed(vaddr); break; @@ -1037,18 +1049,22 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) return -EFAULT; } + if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) + *val = MASK_VAL(reg, *val); + return 0; } static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val) { int ret_val = 0; + int size; void __iomem *vaddr = NULL; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); struct cpc_reg *reg = ®_res->cpc_entry.reg; if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) { - u32 width = 8 << (reg->access_width - 1); + u32 width = GET_BIT_WIDTH(reg); acpi_status status; status = acpi_os_write_port((acpi_io_address)reg->address, @@ -1070,7 +1086,12 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val) return acpi_os_write_memory((acpi_physical_address)reg->address, val, reg->bit_width); - switch (reg->bit_width) { + size = GET_BIT_WIDTH(reg); + + if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) + val = MASK_VAL(reg, val); + + switch (size) { case 8: writeb_relaxed(val, vaddr); break; From 00efe7fcf9ceeff0808bca9460afb49e7ada6068 Mon Sep 17 00:00:00 2001 From: Sergey Kalinichev Date: Mon, 4 Mar 2024 12:26:38 +0300 Subject: [PATCH 0983/1038] ACPI: resource: Use IRQ override on Maibenben X565 Use ACPI IRQ override on Maibenben X565 laptop to make the internal keyboard work. Add a new entry to the irq1_edge_low_force_override structure, similar to the existing ones. Signed-off-by: Sergey Kalinichev Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 4fae9af32f69..59423fe9d0f2 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -623,6 +623,13 @@ static const struct dmi_system_id irq1_edge_low_force_override[] = { DMI_MATCH(DMI_BOARD_NAME, "X577"), }, }, + { + /* Maibenben X565 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "MAIBENBEN"), + DMI_MATCH(DMI_BOARD_NAME, "X565"), + }, + }, { } }; From 53b94f421d53d2a54a8ed42fbcba8b5dd39695fe Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 5 Mar 2024 12:32:09 +0100 Subject: [PATCH 0984/1038] thermal: intel: int340x_thermal: Use thermal zone accessor functions Make int340x_thermal use the dedicated accessor functions for the thermal zone device object address and the thermal zone type string. This is requisite for future thermal core improvements. Signed-off-by: Rafael J. Wysocki Reviewed-by: Srinivas Pandruvada --- drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index e8182f0929d3..400fde7cb3b1 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -58,7 +58,8 @@ static int int340x_thermal_set_trip_temp(struct thermal_zone_device *zone, static void int340x_thermal_critical(struct thermal_zone_device *zone) { - dev_dbg(&zone->device, "%s: critical temperature reached\n", zone->type); + dev_dbg(thermal_zone_device(zone), "%s: critical temperature reached\n", + thermal_zone_device_type(zone)); } static inline void *int_to_trip_priv(int i) From 80a38bfbbd5965c8bda73b20aa78d308739bbc31 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:44 +0000 Subject: [PATCH 0985/1038] spi: dt-bindings: introduce FIFO depth properties There are SPI IPs that can be configured by the integrator with a specific FIFO depth depending on the system's capabilities. For example, the samsung USI SPI IP can be configured by the integrator with a TX/RX FIFO from 8 byte to 256 bytes. Introduce the ``fifo-depth`` property for such instances of IPs where the same FIFO depth is used for both RX and TX. Introduce ``rx-fifo-depth`` and ``tx-fifo-depth`` properties for cases where the RX FIFO depth is different from the TX FIFO depth. Make the dedicated RX/TX properties dependent on each other and mutual exclusive with the other. Reviewed-by: Rob Herring Signed-off-by: Tudor Ambarus Reviewed-by: Conor Dooley Link: https://msgid.link/r/20240216070555.2483977-2-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- .../bindings/spi/spi-controller.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Documentation/devicetree/bindings/spi/spi-controller.yaml b/Documentation/devicetree/bindings/spi/spi-controller.yaml index 524f6fe8c27b..093150c0cb87 100644 --- a/Documentation/devicetree/bindings/spi/spi-controller.yaml +++ b/Documentation/devicetree/bindings/spi/spi-controller.yaml @@ -69,6 +69,21 @@ properties: Should be generally avoided and be replaced by spi-cs-high + ACTIVE_HIGH. + fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Size of the RX and TX data FIFOs in bytes. + + rx-fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Size of the RX data FIFO in bytes. + + tx-fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Size of the TX data FIFO in bytes. + num-cs: $ref: /schemas/types.yaml#/definitions/uint32 description: @@ -116,6 +131,10 @@ patternProperties: - compatible - reg +dependencies: + rx-fifo-depth: [ tx-fifo-depth ] + tx-fifo-depth: [ rx-fifo-depth ] + allOf: - if: not: @@ -129,6 +148,14 @@ allOf: properties: "#address-cells": const: 0 + - not: + required: + - fifo-depth + - rx-fifo-depth + - not: + required: + - fifo-depth + - tx-fifo-depth additionalProperties: true From ff8faa8a5c0f4c2da797cd22a163ee3cc8823b13 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:45 +0000 Subject: [PATCH 0986/1038] spi: s3c64xx: define a magic value Define a magic value, it will be used in the next patch as well. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-3-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 6f29dca68491..6ff3b25b6feb 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -78,6 +78,7 @@ #define S3C64XX_SPI_INT_RX_FIFORDY_EN (1<<1) #define S3C64XX_SPI_INT_TX_FIFORDY_EN (1<<0) +#define S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT 6 #define S3C64XX_SPI_ST_RX_OVERRUN_ERR (1<<5) #define S3C64XX_SPI_ST_RX_UNDERRUN_ERR (1<<4) #define S3C64XX_SPI_ST_TX_OVERRUN_ERR (1<<3) @@ -108,7 +109,8 @@ #define FIFO_LVL_MASK(i) ((i)->port_conf->fifo_lvl_mask[i->port_id]) #define S3C64XX_SPI_ST_TX_DONE(v, i) (((v) & \ (1 << (i)->port_conf->tx_st_done)) ? 1 : 0) -#define TX_FIFO_LVL(v, i) (((v) >> 6) & FIFO_LVL_MASK(i)) +#define TX_FIFO_LVL(v, i) (((v) >> S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT) & \ + FIFO_LVL_MASK(i)) #define RX_FIFO_LVL(v, i) (((v) >> (i)->port_conf->rx_lvl_offset) & \ FIFO_LVL_MASK(i)) #define FIFO_DEPTH(i) ((FIFO_LVL_MASK(i) >> 1) + 1) From d6911cf27e5c8491cbfedd4ae2d1ee74a3e685b4 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:46 +0000 Subject: [PATCH 0987/1038] spi: s3c64xx: allow full FIFO masks The driver is wrong because is using partial register field masks for the SPI_STATUS.{RX, TX}_FIFO_LVL register fields. We see s3c64xx_spi_port_config.fifo_lvl_mask with different values for different instances of the same IP. Take s5pv210_spi_port_config for example, it defines: .fifo_lvl_mask = { 0x1ff, 0x7F }, fifo_lvl_mask is used to determine the FIFO depth of the instance of the IP. In this case, the integrator uses a 256 bytes FIFO for the first SPI instance of the IP, and a 64 bytes FIFO for the second instance. While the first mask reflects the SPI_STATUS.{RX, TX}_FIFO_LVL register fields, the second one is two bits short. Using partial field masks is misleading and can hide problems of the driver's logic. Allow platforms to specify the full FIFO mask, regardless of the FIFO depth. Introduce {rx, tx}_fifomask to represent the SPI_STATUS.{RX, TX}_FIFO_LVL register fields. It's a shifted mask defining the field's length and position. We'll be able to deprecate the use of @rx_lvl_offset, as the shift value can be determined from the mask. The existing compatibles shall start using {rx, tx}_fifomask so that they use the full field mask and to avoid shifting the mask to position, and then shifting it back to zero in the {TX, RX}_FIFO_LVL macros. @rx_lvl_offset will be deprecated in a further patch, after we have the infrastructure to deprecate @fifo_lvl_mask as well. No functional change intended. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-4-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 40 +++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 6ff3b25b6feb..338ca3f03ea5 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -3,6 +3,7 @@ // Copyright (c) 2009 Samsung Electronics Co., Ltd. // Jaswinder Singh +#include #include #include #include @@ -109,10 +110,10 @@ #define FIFO_LVL_MASK(i) ((i)->port_conf->fifo_lvl_mask[i->port_id]) #define S3C64XX_SPI_ST_TX_DONE(v, i) (((v) & \ (1 << (i)->port_conf->tx_st_done)) ? 1 : 0) -#define TX_FIFO_LVL(v, i) (((v) >> S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT) & \ - FIFO_LVL_MASK(i)) -#define RX_FIFO_LVL(v, i) (((v) >> (i)->port_conf->rx_lvl_offset) & \ - FIFO_LVL_MASK(i)) +#define TX_FIFO_LVL(v, sdd) (((v) & (sdd)->tx_fifomask) >> \ + __ffs((sdd)->tx_fifomask)) +#define RX_FIFO_LVL(v, sdd) (((v) & (sdd)->rx_fifomask) >> \ + __ffs((sdd)->rx_fifomask)) #define FIFO_DEPTH(i) ((FIFO_LVL_MASK(i) >> 1) + 1) #define S3C64XX_SPI_MAX_TRAILCNT 0x3ff @@ -136,6 +137,10 @@ struct s3c64xx_spi_dma_data { * struct s3c64xx_spi_port_config - SPI Controller hardware info * @fifo_lvl_mask: Bit-mask for {TX|RX}_FIFO_LVL bits in SPI_STATUS register. * @rx_lvl_offset: Bit offset of RX_FIFO_LVL bits in SPI_STATUS regiter. + * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. + * @tx_fifomask: SPI_STATUS.TX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. * @tx_st_done: Bit offset of TX_DONE bit in SPI_STATUS regiter. * @clk_div: Internal clock divider * @quirks: Bitmask of known quirks @@ -154,6 +159,8 @@ struct s3c64xx_spi_dma_data { struct s3c64xx_spi_port_config { int fifo_lvl_mask[MAX_SPI_PORTS]; int rx_lvl_offset; + u32 rx_fifomask; + u32 tx_fifomask; int tx_st_done; int quirks; int clk_div; @@ -184,6 +191,10 @@ struct s3c64xx_spi_port_config { * @tx_dma: Local transmit DMA data (e.g. chan and direction) * @port_conf: Local SPI port configuration data * @port_id: Port identification number + * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. + * @tx_fifomask: SPI_STATUS.TX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. */ struct s3c64xx_spi_driver_data { void __iomem *regs; @@ -203,6 +214,8 @@ struct s3c64xx_spi_driver_data { struct s3c64xx_spi_dma_data tx_dma; const struct s3c64xx_spi_port_config *port_conf; unsigned int port_id; + u32 rx_fifomask; + u32 tx_fifomask; }; static void s3c64xx_flush_fifo(struct s3c64xx_spi_driver_data *sdd) @@ -1183,6 +1196,23 @@ static inline const struct s3c64xx_spi_port_config *s3c64xx_spi_get_port_config( return (const struct s3c64xx_spi_port_config *)platform_get_device_id(pdev)->driver_data; } +static void s3c64xx_spi_set_fifomask(struct s3c64xx_spi_driver_data *sdd) +{ + const struct s3c64xx_spi_port_config *port_conf = sdd->port_conf; + + if (port_conf->rx_fifomask) + sdd->rx_fifomask = port_conf->rx_fifomask; + else + sdd->rx_fifomask = FIFO_LVL_MASK(sdd) << + port_conf->rx_lvl_offset; + + if (port_conf->tx_fifomask) + sdd->tx_fifomask = port_conf->tx_fifomask; + else + sdd->tx_fifomask = FIFO_LVL_MASK(sdd) << + S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT; +} + static int s3c64xx_spi_probe(struct platform_device *pdev) { struct resource *mem_res; @@ -1231,6 +1261,8 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->port_id = pdev->id; } + s3c64xx_spi_set_fifomask(sdd); + sdd->cur_bpw = 8; sdd->tx_dma.direction = DMA_MEM_TO_DEV; From c6e776ab6abdfce5a1edcde7a22c639e76499939 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:47 +0000 Subject: [PATCH 0988/1038] spi: s3c64xx: determine the fifo depth only once Determine the FIFO depth only once, at probe time. ``sdd->fifo_depth`` can be set later on with the FIFO depth specified in the device tree. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-5-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 338ca3f03ea5..72572e23cde5 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -191,6 +191,7 @@ struct s3c64xx_spi_port_config { * @tx_dma: Local transmit DMA data (e.g. chan and direction) * @port_conf: Local SPI port configuration data * @port_id: Port identification number + * @fifo_depth: depth of the FIFO. * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's * length and position. * @tx_fifomask: SPI_STATUS.TX_FIFO_LVL mask. Shifted mask defining the field's @@ -214,6 +215,7 @@ struct s3c64xx_spi_driver_data { struct s3c64xx_spi_dma_data tx_dma; const struct s3c64xx_spi_port_config *port_conf; unsigned int port_id; + unsigned int fifo_depth; u32 rx_fifomask; u32 tx_fifomask; }; @@ -424,7 +426,7 @@ static bool s3c64xx_spi_can_dma(struct spi_controller *host, struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); if (sdd->rx_dma.ch && sdd->tx_dma.ch) - return xfer->len > FIFO_DEPTH(sdd); + return xfer->len > sdd->fifo_depth; return false; } @@ -548,7 +550,7 @@ static u32 s3c64xx_spi_wait_for_timeout(struct s3c64xx_spi_driver_data *sdd, void __iomem *regs = sdd->regs; unsigned long val = 1; u32 status; - u32 max_fifo = FIFO_DEPTH(sdd); + u32 max_fifo = sdd->fifo_depth; if (timeout_ms) val = msecs_to_loops(timeout_ms); @@ -655,7 +657,7 @@ static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd, * For any size less than the fifo size the below code is * executed atleast once. */ - loops = xfer->len / FIFO_DEPTH(sdd); + loops = xfer->len / sdd->fifo_depth; buf = xfer->rx_buf; do { /* wait for data to be received in the fifo */ @@ -792,7 +794,7 @@ static int s3c64xx_spi_transfer_one(struct spi_controller *host, struct spi_transfer *xfer) { struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); - const unsigned int fifo_len = FIFO_DEPTH(sdd); + const unsigned int fifo_len = sdd->fifo_depth; const void *tx_buf = NULL; void *rx_buf = NULL; int target_len = 0, origin_len = 0; @@ -1261,6 +1263,8 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->port_id = pdev->id; } + sdd->fifo_depth = FIFO_DEPTH(sdd); + s3c64xx_spi_set_fifomask(sdd); sdd->cur_bpw = 8; @@ -1352,7 +1356,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) dev_dbg(&pdev->dev, "Samsung SoC SPI Driver loaded for Bus SPI-%d with %d Targets attached\n", sdd->port_id, host->num_chipselect); dev_dbg(&pdev->dev, "\tIOmem=[%pR]\tFIFO %dbytes\n", - mem_res, FIFO_DEPTH(sdd)); + mem_res, sdd->fifo_depth); pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); From 414d7b8c9147db7dc34c0e2bae2e2361b922dc07 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:48 +0000 Subject: [PATCH 0989/1038] spi: s3c64xx: retrieve the FIFO depth from the device tree There are SoCs that configure different FIFO depths for their instances of the SPI IP. See the fifo_lvl_mask defined for exynos4_spi_port_config for example: .fifo_lvl_mask = { 0x1ff, 0x7F, 0x7F }, The first instance of the IP is configured with 256 bytes FIFOs, whereas the last two are configured with 64 bytes FIFOs. Instead of mangling with the .fifo_lvl_mask and its dependency of the DT alias ID, allow such SoCs to determine the FIFO depth via the ``fifo-depth`` DT property. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-6-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 72572e23cde5..b1c63f75021d 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1263,7 +1263,9 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->port_id = pdev->id; } - sdd->fifo_depth = FIFO_DEPTH(sdd); + if (of_property_read_u32(pdev->dev.of_node, "fifo-depth", + &sdd->fifo_depth)) + sdd->fifo_depth = FIFO_DEPTH(sdd); s3c64xx_spi_set_fifomask(sdd); From 82b98fb8cd33db7793e3e695c44e4e75bca03b3e Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:49 +0000 Subject: [PATCH 0990/1038] spi: s3c64xx: allow FIFO depth to be determined from the compatible There are SoCs that use the same FIFO depth for all the instances of the SPI IP. See the fifo_lvl_mask defined for gs101 for example: .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, Instead of specifying the FIFO depth with the same value for all 16 nodes in this case, allow such SoCs to infer the FIFO depth from the compatible. There are other SoCs than can benefit of this, see: {gs101, fsd, exynos850, s3c641, s3c2443}_spi_port_config. The FIFO depth inferred from the compatible has a higher precedence than the one that might be specified via device tree, the driver shall know better. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-7-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index b1c63f75021d..68f95c04d092 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -137,6 +137,7 @@ struct s3c64xx_spi_dma_data { * struct s3c64xx_spi_port_config - SPI Controller hardware info * @fifo_lvl_mask: Bit-mask for {TX|RX}_FIFO_LVL bits in SPI_STATUS register. * @rx_lvl_offset: Bit offset of RX_FIFO_LVL bits in SPI_STATUS regiter. + * @fifo_depth: depth of the FIFO. * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's * length and position. * @tx_fifomask: SPI_STATUS.TX_FIFO_LVL mask. Shifted mask defining the field's @@ -159,6 +160,7 @@ struct s3c64xx_spi_dma_data { struct s3c64xx_spi_port_config { int fifo_lvl_mask[MAX_SPI_PORTS]; int rx_lvl_offset; + unsigned int fifo_depth; u32 rx_fifomask; u32 tx_fifomask; int tx_st_done; @@ -1263,8 +1265,10 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->port_id = pdev->id; } - if (of_property_read_u32(pdev->dev.of_node, "fifo-depth", - &sdd->fifo_depth)) + if (sdd->port_conf->fifo_depth) + sdd->fifo_depth = sdd->port_conf->fifo_depth; + else if (of_property_read_u32(pdev->dev.of_node, "fifo-depth", + &sdd->fifo_depth)) sdd->fifo_depth = FIFO_DEPTH(sdd); s3c64xx_spi_set_fifomask(sdd); From e08433e095dda8b5e44c376648dbf65c6fb6771a Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:50 +0000 Subject: [PATCH 0991/1038] spi: s3c64xx: let the SPI core determine the bus number Let the core determine the bus number, either by getting the alias ID (as the driver forces now), or by allocating a dynamic bus number when the alias is absent. Prepare the driver to allow dt aliases to be absent. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-8-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 68f95c04d092..ac47755beb02 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1279,7 +1279,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->rx_dma.direction = DMA_DEV_TO_MEM; host->dev.of_node = pdev->dev.of_node; - host->bus_num = sdd->port_id; + host->bus_num = -1; host->setup = s3c64xx_spi_setup; host->cleanup = s3c64xx_spi_cleanup; host->prepare_transfer_hardware = s3c64xx_spi_prepare_transfer; @@ -1360,7 +1360,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) } dev_dbg(&pdev->dev, "Samsung SoC SPI Driver loaded for Bus SPI-%d with %d Targets attached\n", - sdd->port_id, host->num_chipselect); + host->bus_num, host->num_chipselect); dev_dbg(&pdev->dev, "\tIOmem=[%pR]\tFIFO %dbytes\n", mem_res, sdd->fifo_depth); From 2cda3623ff4f002877a81f4e7a4c3401fd98aa2d Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:51 +0000 Subject: [PATCH 0992/1038] spi: s3c64xx: introduce s3c64xx_spi_set_port_id() Prepare driver to get rid of the of alias ID dependency. Split the port_id logic into a dedicated method. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-9-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index ac47755beb02..40de325bd094 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1200,6 +1200,27 @@ static inline const struct s3c64xx_spi_port_config *s3c64xx_spi_get_port_config( return (const struct s3c64xx_spi_port_config *)platform_get_device_id(pdev)->driver_data; } +static int s3c64xx_spi_set_port_id(struct platform_device *pdev, + struct s3c64xx_spi_driver_data *sdd) +{ + int ret; + + if (pdev->dev.of_node) { + ret = of_alias_get_id(pdev->dev.of_node, "spi"); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Failed to get alias id\n"); + sdd->port_id = ret; + } else { + if (pdev->id < 0) + return dev_err_probe(&pdev->dev, -EINVAL, + "Negative platform ID is not allowed\n"); + sdd->port_id = pdev->id; + } + + return 0; +} + static void s3c64xx_spi_set_fifomask(struct s3c64xx_spi_driver_data *sdd) { const struct s3c64xx_spi_port_config *port_conf = sdd->port_conf; @@ -1252,18 +1273,10 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->host = host; sdd->cntrlr_info = sci; sdd->pdev = pdev; - if (pdev->dev.of_node) { - ret = of_alias_get_id(pdev->dev.of_node, "spi"); - if (ret < 0) - return dev_err_probe(&pdev->dev, ret, - "Failed to get alias id\n"); - sdd->port_id = ret; - } else { - if (pdev->id < 0) - return dev_err_probe(&pdev->dev, -EINVAL, - "Negative platform ID is not allowed\n"); - sdd->port_id = pdev->id; - } + + ret = s3c64xx_spi_set_port_id(pdev, sdd); + if (ret) + return ret; if (sdd->port_conf->fifo_depth) sdd->fifo_depth = sdd->port_conf->fifo_depth; From ea3fba7c41babda225fea324a72d171be9ff6de6 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:52 +0000 Subject: [PATCH 0993/1038] spi: s3c64xx: get rid of the OF alias ID dependency Compatibles that set ``port_conf->{rx, tx}_fifomask`` are now safe to get rid of the OF alias ID dependency. Let the driver probe even without the alias for these. With this we also protect the FIFO_LVL_MASK calls from s3c64xx_spi_set_fifomask(). Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-10-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 40de325bd094..d2d1c9767145 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1203,8 +1203,12 @@ static inline const struct s3c64xx_spi_port_config *s3c64xx_spi_get_port_config( static int s3c64xx_spi_set_port_id(struct platform_device *pdev, struct s3c64xx_spi_driver_data *sdd) { + const struct s3c64xx_spi_port_config *port_conf = sdd->port_conf; int ret; + if (port_conf->rx_fifomask && port_conf->tx_fifomask) + return 0; + if (pdev->dev.of_node) { ret = of_alias_get_id(pdev->dev.of_node, "spi"); if (ret < 0) From ad0adac84d42b693295f4bde407d9f20c9a694ab Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:53 +0000 Subject: [PATCH 0994/1038] spi: s3c64xx: deprecate fifo_lvl_mask, rx_lvl_offset and port_id Deprecate fifo_lvl_mask, rx_lvl_offset and port_id. One shall use {rx, tx}_fifomask instead. Add messages to each port configuration. Suggested-by: Sam Protsenko Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-11-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index d2d1c9767145..128f4a7c4bd9 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -135,8 +135,8 @@ struct s3c64xx_spi_dma_data { /** * struct s3c64xx_spi_port_config - SPI Controller hardware info - * @fifo_lvl_mask: Bit-mask for {TX|RX}_FIFO_LVL bits in SPI_STATUS register. - * @rx_lvl_offset: Bit offset of RX_FIFO_LVL bits in SPI_STATUS regiter. + * @fifo_lvl_mask: [DEPRECATED] use @{rx, tx}_fifomask instead. + * @rx_lvl_offset: [DEPRECATED] use @{rx,tx}_fifomask instead. * @fifo_depth: depth of the FIFO. * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's * length and position. @@ -192,7 +192,7 @@ struct s3c64xx_spi_port_config { * @rx_dma: Local receive DMA data (e.g. chan and direction) * @tx_dma: Local transmit DMA data (e.g. chan and direction) * @port_conf: Local SPI port configuration data - * @port_id: Port identification number + * @port_id: [DEPRECATED] use @{rx,tx}_fifomask instead. * @fifo_depth: depth of the FIFO. * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's * length and position. @@ -1508,7 +1508,9 @@ static const struct dev_pm_ops s3c64xx_spi_pm = { }; static const struct s3c64xx_spi_port_config s3c2443_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 13, .tx_st_done = 21, .clk_div = 2, @@ -1516,14 +1518,18 @@ static const struct s3c64xx_spi_port_config s3c2443_spi_port_config = { }; static const struct s3c64xx_spi_port_config s3c6410_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f, 0x7F }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 13, .tx_st_done = 21, .clk_div = 2, }; static const struct s3c64xx_spi_port_config s5pv210_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7F }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1531,7 +1537,9 @@ static const struct s3c64xx_spi_port_config s5pv210_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos4_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7F, 0x7F }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1541,7 +1549,9 @@ static const struct s3c64xx_spi_port_config exynos4_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos7_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7F, 0x7F, 0x7F, 0x7F, 0x1ff}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1551,7 +1561,9 @@ static const struct s3c64xx_spi_port_config exynos7_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos5433_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x1ff}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1562,7 +1574,9 @@ static const struct s3c64xx_spi_port_config exynos5433_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos850_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 4, @@ -1573,8 +1587,10 @@ static const struct s3c64xx_spi_port_config exynos850_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynosautov9_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 4, @@ -1586,7 +1602,9 @@ static const struct s3c64xx_spi_port_config exynosautov9_spi_port_config = { }; static const struct s3c64xx_spi_port_config fsd_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1597,8 +1615,10 @@ static const struct s3c64xx_spi_port_config fsd_spi_port_config = { }; static const struct s3c64xx_spi_port_config gs101_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 4, From e8b16c7a420420a994f68c181abc4a82dcca0616 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:54 +0000 Subject: [PATCH 0995/1038] spi: s3c64xx: switch gs101 to new port config data Drop the fifo_lvl_mask and rx_lvl_offset and switch to the new port config data. Advantages of the change: - drop dependency on the OF alias ID. - FIFO depth is inferred from the compatible. GS101 integrates 16 SPI IPs, all with 64 bytes FIFO depths. - use full mask for SPI_STATUS.{RX, TX}_FIFO_LVL fields. Using partial masks is misleading and can hide problems of the driver logic. S3C64XX_SPI_ST_TX_FIFO_RDY_V2 was defined based on the USI's SPI_VERSION.USI_IP_VERSION register field, which has value 2 at reset. MAX_SPI_PORTS is updated to reflect the maximum number of ports for the rest of the compatibles. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-12-tudor.ambarus@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 128f4a7c4bd9..784786407d2e 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -20,7 +20,7 @@ #include #include -#define MAX_SPI_PORTS 16 +#define MAX_SPI_PORTS 12 #define S3C64XX_SPI_QUIRK_CS_AUTO (1 << 1) #define AUTOSUSPEND_TIMEOUT 2000 @@ -79,6 +79,8 @@ #define S3C64XX_SPI_INT_RX_FIFORDY_EN (1<<1) #define S3C64XX_SPI_INT_TX_FIFORDY_EN (1<<0) +#define S3C64XX_SPI_ST_RX_FIFO_RDY_V2 GENMASK(23, 15) +#define S3C64XX_SPI_ST_TX_FIFO_RDY_V2 GENMASK(14, 6) #define S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT 6 #define S3C64XX_SPI_ST_RX_OVERRUN_ERR (1<<5) #define S3C64XX_SPI_ST_RX_UNDERRUN_ERR (1<<4) @@ -1615,11 +1617,9 @@ static const struct s3c64xx_spi_port_config fsd_spi_port_config = { }; static const struct s3c64xx_spi_port_config gs101_spi_port_config = { - /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ - .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, - /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ - .rx_lvl_offset = 15, + .fifo_depth = 64, + .rx_fifomask = S3C64XX_SPI_ST_RX_FIFO_RDY_V2, + .tx_fifomask = S3C64XX_SPI_ST_TX_FIFO_RDY_V2, .tx_st_done = 25, .clk_div = 4, .high_speed = true, From 7ad288208d24e42047e5bf0b88271684a32aa967 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 16 Feb 2024 07:05:55 +0000 Subject: [PATCH 0996/1038] spi: s3c64xx: switch exynos850 to new port config data Exynos850 has the same version of USI SPI (v2.1) as GS101. Drop the fifo_lvl_mask and rx_lvl_offset and switch to the new port config data. Backward compatibility with DT is not broken because when alises are set: - the SPI core will set the bus number according to the alias ID - the FIFO depth is always the same size for exynos850 (64 bytes) no matter the alias ID number. Advantages of the change: - drop dependency on the OF alias ID. - FIFO depth is inferred from the compatible. Exynos850 integrates 3 SPI IPs, all with 64 bytes FIFO depths. - use full mask for SPI_STATUS.{RX, TX}_FIFO_LVL fields. Using partial masks is misleading and can hide problems of the driver logic. Just compiled tested. Signed-off-by: Tudor Ambarus Link: https://msgid.link/r/20240216070555.2483977-13-tudor.ambarus@linaro.org Tested-by: Sam Protsenko Signed-off-by: Mark Brown --- drivers/spi/spi-s3c64xx.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 784786407d2e..9fcbe040cb2f 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1576,10 +1576,9 @@ static const struct s3c64xx_spi_port_config exynos5433_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos850_spi_port_config = { - /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ - .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f }, - /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ - .rx_lvl_offset = 15, + .fifo_depth = 64, + .rx_fifomask = S3C64XX_SPI_ST_RX_FIFO_RDY_V2, + .tx_fifomask = S3C64XX_SPI_ST_TX_FIFO_RDY_V2, .tx_st_done = 25, .clk_div = 4, .high_speed = true, From ad86f7e959dc1814c3b2bcbeb08c3c02214110a7 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 14:56:59 +0100 Subject: [PATCH 0997/1038] firmware: arm_scmi: Populate perf commands rate_limit Arm SCMI spec. v3.2, s4.5.3.4 PERFORMANCE_DOMAIN_ATTRIBUTES defines a per-domain rate_limit for performance requests: """ Rate Limit in microseconds, indicating the minimum time required between successive requests. A value of 0 indicates that this field is not supported by the platform. This field does not apply to FastChannels. """" The field is first defined in SCMI v1.0. Add support to fetch this value and advertise it through a rate_limit_get() callback. Signed-off-by: Pierre Gondois Reviewed-by: Cristian Marussi Acked-by: Sudeep Holla Signed-off-by: Viresh Kumar --- drivers/firmware/arm_scmi/perf.c | 21 +++++++++++++++++++++ include/linux/scmi_protocol.h | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 8ea2a7b3d35d..3269415ff77d 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -153,6 +153,7 @@ struct perf_dom_info { bool perf_fastchannels; bool level_indexing_mode; u32 opp_count; + u32 rate_limit_us; u32 sustained_freq_khz; u32 sustained_perf_level; unsigned long mult_factor; @@ -266,6 +267,8 @@ scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph, if (PROTOCOL_REV_MAJOR(version) >= 0x4) dom_info->level_indexing_mode = SUPPORTS_LEVEL_INDEXING(flags); + dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) & + GENMASK(19, 0); dom_info->sustained_freq_khz = le32_to_cpu(attr->sustained_freq_khz); dom_info->sustained_perf_level = @@ -842,6 +845,23 @@ scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph, return dom->opp[dom->opp_count - 1].trans_latency_us * 1000; } +static int +scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit) +{ + struct perf_dom_info *dom; + + if (!rate_limit) + return -EINVAL; + + dom = scmi_perf_domain_lookup(ph, domain); + if (IS_ERR(dom)) + return PTR_ERR(dom); + + *rate_limit = dom->rate_limit_us; + return 0; +} + static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain, unsigned long freq, bool poll) { @@ -957,6 +977,7 @@ static const struct scmi_perf_proto_ops perf_proto_ops = { .level_set = scmi_perf_level_set, .level_get = scmi_perf_level_get, .transition_latency_get = scmi_dvfs_transition_latency_get, + .rate_limit_get = scmi_dvfs_rate_limit_get, .device_opps_add = scmi_dvfs_device_opps_add, .freq_set = scmi_dvfs_freq_set, .freq_get = scmi_dvfs_freq_get, diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index f2f05fb42d28..acd956ffcb84 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -128,6 +128,8 @@ struct scmi_perf_domain_info { * @level_set: sets the performance level of a domain * @level_get: gets the performance level of a domain * @transition_latency_get: gets the DVFS transition latency for a given device + * @rate_limit_get: gets the minimum time (us) required between successive + * requests * @device_opps_add: adds all the OPPs for a given device * @freq_set: sets the frequency for a given device using sustained frequency * to sustained performance level mapping @@ -154,6 +156,8 @@ struct scmi_perf_proto_ops { u32 *level, bool poll); int (*transition_latency_get)(const struct scmi_protocol_handle *ph, u32 domain); + int (*rate_limit_get)(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit); int (*device_opps_add)(const struct scmi_protocol_handle *ph, struct device *dev, u32 domain); int (*freq_set)(const struct scmi_protocol_handle *ph, u32 domain, From 2441caa84aac8abf1be9e20db3e6bb921e74c8a2 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 14:57:00 +0100 Subject: [PATCH 0998/1038] firmware: arm_scmi: Populate fast channel rate_limit Arm SCMI spec. v3.2, s4.5.3.12 PERFORMANCE_DESCRIBE_FASTCHANNEL defines a per-domain rate_limit for performance requests: """ Rate Limit in microseconds, indicating the minimum time required between successive requests. A value of 0 indicates that this field is not applicable or supported on the platform. """" The field is first defined in SCMI v2.0. Add support to fetch this value and advertise it through a fast_switch_rate_limit() callback. Signed-off-by: Pierre Gondois Reviewed-by: Cristian Marussi Acked-by: Sudeep Holla Signed-off-by: Viresh Kumar --- drivers/firmware/arm_scmi/driver.c | 5 ++++- drivers/firmware/arm_scmi/perf.c | 32 +++++++++++++++++++++++---- drivers/firmware/arm_scmi/powercap.c | 12 ++++++---- drivers/firmware/arm_scmi/protocols.h | 4 +++- include/linux/scmi_protocol.h | 4 ++++ 5 files changed, 47 insertions(+), 10 deletions(-) diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 3ea64b22cf0d..1d38ecfafc59 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -1617,7 +1617,7 @@ static void scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, u8 describe_id, u32 message_id, u32 valid_size, u32 domain, void __iomem **p_addr, - struct scmi_fc_db_info **p_db) + struct scmi_fc_db_info **p_db, u32 *rate_limit) { int ret; u32 flags; @@ -1661,6 +1661,9 @@ scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, goto err_xfer; } + if (rate_limit) + *rate_limit = le32_to_cpu(resp->rate_limit) & GENMASK(19, 0); + phys_addr = le32_to_cpu(resp->chan_addr_low); phys_addr |= (u64)le32_to_cpu(resp->chan_addr_high) << 32; addr = devm_ioremap(ph->dev, phys_addr, size); diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 3269415ff77d..be1801fb848d 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -776,23 +776,27 @@ static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph, ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LEVEL_GET, 4, dom->id, - &fc[PERF_FC_LEVEL].get_addr, NULL); + &fc[PERF_FC_LEVEL].get_addr, NULL, + &fc[PERF_FC_LEVEL].rate_limit); ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LIMITS_GET, 8, dom->id, - &fc[PERF_FC_LIMIT].get_addr, NULL); + &fc[PERF_FC_LIMIT].get_addr, NULL, + &fc[PERF_FC_LIMIT].rate_limit); if (dom->info.set_perf) ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LEVEL_SET, 4, dom->id, &fc[PERF_FC_LEVEL].set_addr, - &fc[PERF_FC_LEVEL].set_db); + &fc[PERF_FC_LEVEL].set_db, + &fc[PERF_FC_LEVEL].rate_limit); if (dom->set_limits) ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LIMITS_SET, 8, dom->id, &fc[PERF_FC_LIMIT].set_addr, - &fc[PERF_FC_LIMIT].set_db); + &fc[PERF_FC_LIMIT].set_db, + &fc[PERF_FC_LIMIT].rate_limit); dom->fc_info = fc; } @@ -961,6 +965,25 @@ static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph, return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr; } +static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit) +{ + struct perf_dom_info *dom; + + if (!rate_limit) + return -EINVAL; + + dom = scmi_perf_domain_lookup(ph, domain); + if (IS_ERR(dom)) + return PTR_ERR(dom); + + if (!dom->fc_info) + return -EINVAL; + + *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit; + return 0; +} + static enum scmi_power_scale scmi_power_scale_get(const struct scmi_protocol_handle *ph) { @@ -983,6 +1006,7 @@ static const struct scmi_perf_proto_ops perf_proto_ops = { .freq_get = scmi_dvfs_freq_get, .est_power_get = scmi_dvfs_est_power_get, .fast_switch_possible = scmi_fast_switch_possible, + .fast_switch_rate_limit = scmi_fast_switch_rate_limit, .power_scale_get = scmi_power_scale_get, }; diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c index a4c6cd4716fe..604184c044ff 100644 --- a/drivers/firmware/arm_scmi/powercap.c +++ b/drivers/firmware/arm_scmi/powercap.c @@ -703,20 +703,24 @@ static void scmi_powercap_domain_init_fc(const struct scmi_protocol_handle *ph, ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_CAP_SET, 4, domain, &fc[POWERCAP_FC_CAP].set_addr, - &fc[POWERCAP_FC_CAP].set_db); + &fc[POWERCAP_FC_CAP].set_db, + &fc[POWERCAP_FC_CAP].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_CAP_GET, 4, domain, - &fc[POWERCAP_FC_CAP].get_addr, NULL); + &fc[POWERCAP_FC_CAP].get_addr, NULL, + &fc[POWERCAP_FC_CAP].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_PAI_SET, 4, domain, &fc[POWERCAP_FC_PAI].set_addr, - &fc[POWERCAP_FC_PAI].set_db); + &fc[POWERCAP_FC_PAI].set_db, + &fc[POWERCAP_FC_PAI].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_PAI_GET, 4, domain, - &fc[POWERCAP_FC_PAI].get_addr, NULL); + &fc[POWERCAP_FC_PAI].get_addr, NULL, + &fc[POWERCAP_PAI_GET].rate_limit); *p_fc = fc; } diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h index e683c26f24eb..8b5d9ce4a33a 100644 --- a/drivers/firmware/arm_scmi/protocols.h +++ b/drivers/firmware/arm_scmi/protocols.h @@ -234,6 +234,7 @@ struct scmi_fc_info { void __iomem *set_addr; void __iomem *get_addr; struct scmi_fc_db_info *set_db; + u32 rate_limit; }; /** @@ -268,7 +269,8 @@ struct scmi_proto_helpers_ops { u8 describe_id, u32 message_id, u32 valid_size, u32 domain, void __iomem **p_addr, - struct scmi_fc_db_info **p_db); + struct scmi_fc_db_info **p_db, + u32 *rate_limit); void (*fastchannel_db_ring)(struct scmi_fc_db_info *db); }; diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index acd956ffcb84..fafedb3b6604 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -139,6 +139,8 @@ struct scmi_perf_domain_info { * at a given frequency * @fast_switch_possible: indicates if fast DVFS switching is possible or not * for a given device + * @fast_switch_rate_limit: gets the minimum time (us) required between + * successive fast_switching requests * @power_scale_mw_get: indicates if the power values provided are in milliWatts * or in some other (abstract) scale */ @@ -168,6 +170,8 @@ struct scmi_perf_proto_ops { unsigned long *rate, unsigned long *power); bool (*fast_switch_possible)(const struct scmi_protocol_handle *ph, u32 domain); + int (*fast_switch_rate_limit)(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit); enum scmi_power_scale (*power_scale_get)(const struct scmi_protocol_handle *ph); }; From ad2a91086e288c9ab1d74eee57edabe08bd90471 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 22 Feb 2024 14:57:01 +0100 Subject: [PATCH 0999/1038] cpufreq: scmi: Set transition_delay_us Make use of the newly added callbacks: - rate_limit_get() - fast_switch_rate_limit() to populate policies's `transition_delay_us`, defined as the 'Preferred average time interval between consecutive invocations of the driver to set the frequency for this policy.' Signed-off-by: Pierre Gondois Signed-off-by: Viresh Kumar --- drivers/cpufreq/scmi-cpufreq.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 4ee23f4ebf4a..0b483bd0d3ca 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -144,6 +144,29 @@ scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power, return 0; } +static int +scmi_get_rate_limit(u32 domain, bool has_fast_switch) +{ + int ret, rate_limit; + + if (has_fast_switch) { + /* + * Fast channels are used whenever available, + * so use their rate_limit value if populated. + */ + ret = perf_ops->fast_switch_rate_limit(ph, domain, + &rate_limit); + if (!ret && rate_limit) + return rate_limit; + } + + ret = perf_ops->rate_limit_get(ph, domain, &rate_limit); + if (ret) + return 0; + + return rate_limit; +} + static int scmi_cpufreq_init(struct cpufreq_policy *policy) { int ret, nr_opp, domain; @@ -250,6 +273,9 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy) policy->fast_switch_possible = perf_ops->fast_switch_possible(ph, domain); + policy->transition_delay_us = + scmi_get_rate_limit(domain, policy->fast_switch_possible); + return 0; out_free_opp: From ccabbb6768fc72d6cb0223324925c93658d91e63 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 1 Mar 2024 19:23:45 +0100 Subject: [PATCH 1000/1038] pmdomain: renesas: rcar-gen4-sysc: Reduce atomic delays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delays used with the various atomic polling loops are already at the maximum value of ~10µs, as documented for read_poll_timeout_atomic(). Hence reduce the delays from 10 to 1 µs. Increase PDRESR_RETRIES accordingly, to retain the old (generous) timeout value. Measurements on R-Car V3U, S4, V4H, and V4M show that the first three polling loops rarely (never?) loop, so the actual delay does not matter. The fourth loop (for SYSCISCR in rcar_gen4_sysc_power()) typically ran for one or two cycles with the old delay. With the reduced delay, it typically runs for two to 17 cycles, and finishes earlier than before, which can reduce loop time up to a factor of three. While at it, rename the SYSCISR_{TIMEOUT,DELAY_US} definitions to SYSCISCR_{TIMEOUT,DELAY_US}, to match the register name they apply to. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/77f150522096d55c6da0ff983db61e0cf6309344.1709317289.git.geert+renesas@glider.be Signed-off-by: Ulf Hansson --- drivers/pmdomain/renesas/rcar-gen4-sysc.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pmdomain/renesas/rcar-gen4-sysc.c b/drivers/pmdomain/renesas/rcar-gen4-sysc.c index 728248659a97..66409cff2083 100644 --- a/drivers/pmdomain/renesas/rcar-gen4-sysc.c +++ b/drivers/pmdomain/renesas/rcar-gen4-sysc.c @@ -50,13 +50,13 @@ #define SYSCSR_BUSY GENMASK(1, 0) /* All bit sets is not busy */ #define SYSCSR_TIMEOUT 10000 -#define SYSCSR_DELAY_US 10 +#define SYSCSR_DELAY_US 1 -#define PDRESR_RETRIES 1000 -#define PDRESR_DELAY_US 10 +#define PDRESR_RETRIES 10000 +#define PDRESR_DELAY_US 1 -#define SYSCISR_TIMEOUT 10000 -#define SYSCISR_DELAY_US 10 +#define SYSCISCR_TIMEOUT 10000 +#define SYSCISCR_DELAY_US 1 #define RCAR_GEN4_PD_ALWAYS_ON 64 #define NUM_DOMAINS_EACH_REG BITS_PER_TYPE(u32) @@ -97,7 +97,7 @@ static int clear_irq_flags(unsigned int reg_idx, unsigned int isr_mask) ret = readl_poll_timeout_atomic(rcar_gen4_sysc_base + SYSCISCR(reg_idx), val, !(val & isr_mask), - SYSCISR_DELAY_US, SYSCISR_TIMEOUT); + SYSCISCR_DELAY_US, SYSCISCR_TIMEOUT); if (ret < 0) { pr_err("\n %s : Can not clear IRQ flags in SYSCISCR", __func__); return -EIO; @@ -157,7 +157,7 @@ static int rcar_gen4_sysc_power(u8 pdr, bool on) /* Wait until the power shutoff or resume request has completed * */ ret = readl_poll_timeout_atomic(rcar_gen4_sysc_base + SYSCISCR(reg_idx), val, (val & isr_mask), - SYSCISR_DELAY_US, SYSCISR_TIMEOUT); + SYSCISCR_DELAY_US, SYSCISCR_TIMEOUT); if (ret < 0) { ret = -EIO; goto out; From 59d894a078cbed0335bb280dca411c91f686a9fd Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 6 Mar 2024 08:31:12 +0300 Subject: [PATCH 1001/1038] thermal: core: remove unnecessary check in trip_point_hyst_store() This code was shuffled around a bit recently. We no longer need to check the value of "ret" because we know it's zero. Signed-off-by: Dan Carpenter Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 7c02d35384ce..5b533fa40437 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -181,7 +181,7 @@ trip_point_hyst_store(struct device *dev, struct device_attribute *attr, mutex_unlock(&tz->lock); - return ret ? ret : count; + return count; } static ssize_t From e6f0b08a036734552628ab788ecb528ca53814ab Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 6 Mar 2024 16:52:29 +0300 Subject: [PATCH 1002/1038] regulator: lp8788-buck: fix copy and paste bug in lp8788_dvs_gpio_request() "gpio2" as intended here, not "gpio1". Fixes: 95daa868f22b ("regulator: lp8788-buck: Fully convert to GPIO descriptors") Signed-off-by: Dan Carpenter Link: https://msgid.link/r/19f62cc2-bdcf-46f7-a5c5-971ef05e1ea7@moroto.mountain Signed-off-by: Mark Brown --- drivers/regulator/lp8788-buck.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/regulator/lp8788-buck.c b/drivers/regulator/lp8788-buck.c index 712eaa6ff8ab..2ade249ab6df 100644 --- a/drivers/regulator/lp8788-buck.c +++ b/drivers/regulator/lp8788-buck.c @@ -430,9 +430,9 @@ static int lp8788_dvs_gpio_request(struct platform_device *pdev, gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS1"); buck->gpio2 = devm_gpiod_get_index(dev, "dvs", 1, GPIOD_OUT_LOW); - if (IS_ERR(buck->gpio1)) - return PTR_ERR(buck->gpio1); - gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS2"); + if (IS_ERR(buck->gpio2)) + return PTR_ERR(buck->gpio2); + gpiod_set_consumer_name(buck->gpio2, "LP8788_B2_DVS2"); buck->dvs = pdata->buck2_dvs; break; From a114d9f1f2cf4896d838ab0a9c30a75411736829 Mon Sep 17 00:00:00 2001 From: Jan Kratochvil Date: Mon, 12 Feb 2024 21:32:56 +0800 Subject: [PATCH 1003/1038] Fix cpupower-frequency-info.1 man page typo cpupower-frequency-info.1 man page type is incorrect for related-cpus. Fix it. utils/cpufreq-info.c {"related-cpus", no_argument, NULL, 'r'}, {"affected-cpus", no_argument, NULL, 'a'}, Fixed changelog before applying: Shuah Khan Signed-off-by: Jan Kratochvil Signed-off-by: Shuah Khan --- tools/power/cpupower/man/cpupower-frequency-info.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/power/cpupower/man/cpupower-frequency-info.1 b/tools/power/cpupower/man/cpupower-frequency-info.1 index dd545b499480..47fdd7218748 100644 --- a/tools/power/cpupower/man/cpupower-frequency-info.1 +++ b/tools/power/cpupower/man/cpupower-frequency-info.1 @@ -32,7 +32,7 @@ Gets the currently used cpufreq policy. \fB\-g\fR \fB\-\-governors\fR Determines available cpufreq governors. .TP -\fB\-a\fR \fB\-\-related\-cpus\fR +\fB\-r\fR \fB\-\-related\-cpus\fR Determines which CPUs run at the same hardware frequency. .TP \fB\-a\fR \fB\-\-affected\-cpus\fR From 39714fd73c6b60a8d27bcc5b431afb0828bf4434 Mon Sep 17 00:00:00 2001 From: Ethan Zhao Date: Tue, 5 Mar 2024 20:21:14 +0800 Subject: [PATCH 1004/1038] PCI: Make pci_dev_is_disconnected() helper public for other drivers Make pci_dev_is_disconnected() public so that it can be called from Intel VT-d driver to quickly fix/workaround the surprise removal unplug hang issue for those ATS capable devices on PCIe switch downstream hotplug capable ports. Beside pci_device_is_present() function, this one has no config space space access, so is light enough to optimize the normal pure surprise removal and safe removal flow. Acked-by: Bjorn Helgaas Reviewed-by: Dan Carpenter Tested-by: Haorong Ye Signed-off-by: Ethan Zhao Link: https://lore.kernel.org/r/20240301080727.3529832-2-haifeng.zhao@linux.intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/pci/pci.h | 5 ----- include/linux/pci.h | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index e9750b1b19ba..bfc56f7bee1c 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -368,11 +368,6 @@ static inline int pci_dev_set_disconnected(struct pci_dev *dev, void *unused) return 0; } -static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) -{ - return dev->error_state == pci_channel_io_perm_failure; -} - /* pci_dev priv_flags */ #define PCI_DEV_ADDED 0 #define PCI_DPC_RECOVERED 1 diff --git a/include/linux/pci.h b/include/linux/pci.h index 7ab0d13672da..213109d3c601 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2517,6 +2517,11 @@ static inline struct pci_dev *pcie_find_root_port(struct pci_dev *dev) return NULL; } +static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) +{ + return dev->error_state == pci_channel_io_perm_failure; +} + void pci_request_acs(void); bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags); bool pci_acs_path_enabled(struct pci_dev *start, From 4fc82cd907ac075648789cc3a00877778aa1838b Mon Sep 17 00:00:00 2001 From: Ethan Zhao Date: Tue, 5 Mar 2024 20:21:15 +0800 Subject: [PATCH 1005/1038] iommu/vt-d: Don't issue ATS Invalidation request when device is disconnected For those endpoint devices connect to system via hotplug capable ports, users could request a hot reset to the device by flapping device's link through setting the slot's link control register, as pciehp_ist() DLLSC interrupt sequence response, pciehp will unload the device driver and then power it off. thus cause an IOMMU device-TLB invalidation (Intel VT-d spec, or ATS Invalidation in PCIe spec r6.1) request for non-existence target device to be sent and deadly loop to retry that request after ITE fault triggered in interrupt context. That would cause following continuous hard lockup warning and system hang [ 4211.433662] pcieport 0000:17:01.0: pciehp: Slot(108): Link Down [ 4211.433664] pcieport 0000:17:01.0: pciehp: Slot(108): Card not present [ 4223.822591] NMI watchdog: Watchdog detected hard LOCKUP on cpu 144 [ 4223.822622] CPU: 144 PID: 1422 Comm: irq/57-pciehp Kdump: loaded Tainted: G S OE kernel version xxxx [ 4223.822623] Hardware name: vendorname xxxx 666-106, BIOS 01.01.02.03.01 05/15/2023 [ 4223.822623] RIP: 0010:qi_submit_sync+0x2c0/0x490 [ 4223.822624] Code: 48 be 00 00 00 00 00 08 00 00 49 85 74 24 20 0f 95 c1 48 8b 57 10 83 c1 04 83 3c 1a 03 0f 84 a2 01 00 00 49 8b 04 24 8b 70 34 <40> f6 c6 1 0 74 17 49 8b 04 24 8b 80 80 00 00 00 89 c2 d3 fa 41 39 [ 4223.822624] RSP: 0018:ffffc4f074f0bbb8 EFLAGS: 00000093 [ 4223.822625] RAX: ffffc4f040059000 RBX: 0000000000000014 RCX: 0000000000000005 [ 4223.822625] RDX: ffff9f3841315800 RSI: 0000000000000000 RDI: ffff9f38401a8340 [ 4223.822625] RBP: ffff9f38401a8340 R08: ffffc4f074f0bc00 R09: 0000000000000000 [ 4223.822626] R10: 0000000000000010 R11: 0000000000000018 R12: ffff9f384005e200 [ 4223.822626] R13: 0000000000000004 R14: 0000000000000046 R15: 0000000000000004 [ 4223.822626] FS: 0000000000000000(0000) GS:ffffa237ae400000(0000) knlGS:0000000000000000 [ 4223.822627] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4223.822627] CR2: 00007ffe86515d80 CR3: 000002fd3000a001 CR4: 0000000000770ee0 [ 4223.822627] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4223.822628] DR3: 0000000000000000 DR6: 00000000fffe07f0 DR7: 0000000000000400 [ 4223.822628] PKRU: 55555554 [ 4223.822628] Call Trace: [ 4223.822628] qi_flush_dev_iotlb+0xb1/0xd0 [ 4223.822628] __dmar_remove_one_dev_info+0x224/0x250 [ 4223.822629] dmar_remove_one_dev_info+0x3e/0x50 [ 4223.822629] intel_iommu_release_device+0x1f/0x30 [ 4223.822629] iommu_release_device+0x33/0x60 [ 4223.822629] iommu_bus_notifier+0x7f/0x90 [ 4223.822630] blocking_notifier_call_chain+0x60/0x90 [ 4223.822630] device_del+0x2e5/0x420 [ 4223.822630] pci_remove_bus_device+0x70/0x110 [ 4223.822630] pciehp_unconfigure_device+0x7c/0x130 [ 4223.822631] pciehp_disable_slot+0x6b/0x100 [ 4223.822631] pciehp_handle_presence_or_link_change+0xd8/0x320 [ 4223.822631] pciehp_ist+0x176/0x180 [ 4223.822631] ? irq_finalize_oneshot.part.50+0x110/0x110 [ 4223.822632] irq_thread_fn+0x19/0x50 [ 4223.822632] irq_thread+0x104/0x190 [ 4223.822632] ? irq_forced_thread_fn+0x90/0x90 [ 4223.822632] ? irq_thread_check_affinity+0xe0/0xe0 [ 4223.822633] kthread+0x114/0x130 [ 4223.822633] ? __kthread_cancel_work+0x40/0x40 [ 4223.822633] ret_from_fork+0x1f/0x30 [ 4223.822633] Kernel panic - not syncing: Hard LOCKUP [ 4223.822634] CPU: 144 PID: 1422 Comm: irq/57-pciehp Kdump: loaded Tainted: G S OE kernel version xxxx [ 4223.822634] Hardware name: vendorname xxxx 666-106, BIOS 01.01.02.03.01 05/15/2023 [ 4223.822634] Call Trace: [ 4223.822634] [ 4223.822635] dump_stack+0x6d/0x88 [ 4223.822635] panic+0x101/0x2d0 [ 4223.822635] ? ret_from_fork+0x11/0x30 [ 4223.822635] nmi_panic.cold.14+0xc/0xc [ 4223.822636] watchdog_overflow_callback.cold.8+0x6d/0x81 [ 4223.822636] __perf_event_overflow+0x4f/0xf0 [ 4223.822636] handle_pmi_common+0x1ef/0x290 [ 4223.822636] ? __set_pte_vaddr+0x28/0x40 [ 4223.822637] ? flush_tlb_one_kernel+0xa/0x20 [ 4223.822637] ? __native_set_fixmap+0x24/0x30 [ 4223.822637] ? ghes_copy_tofrom_phys+0x70/0x100 [ 4223.822637] ? __ghes_peek_estatus.isra.16+0x49/0xa0 [ 4223.822637] intel_pmu_handle_irq+0xba/0x2b0 [ 4223.822638] perf_event_nmi_handler+0x24/0x40 [ 4223.822638] nmi_handle+0x4d/0xf0 [ 4223.822638] default_do_nmi+0x49/0x100 [ 4223.822638] exc_nmi+0x134/0x180 [ 4223.822639] end_repeat_nmi+0x16/0x67 [ 4223.822639] RIP: 0010:qi_submit_sync+0x2c0/0x490 [ 4223.822639] Code: 48 be 00 00 00 00 00 08 00 00 49 85 74 24 20 0f 95 c1 48 8b 57 10 83 c1 04 83 3c 1a 03 0f 84 a2 01 00 00 49 8b 04 24 8b 70 34 <40> f6 c6 10 74 17 49 8b 04 24 8b 80 80 00 00 00 89 c2 d3 fa 41 39 [ 4223.822640] RSP: 0018:ffffc4f074f0bbb8 EFLAGS: 00000093 [ 4223.822640] RAX: ffffc4f040059000 RBX: 0000000000000014 RCX: 0000000000000005 [ 4223.822640] RDX: ffff9f3841315800 RSI: 0000000000000000 RDI: ffff9f38401a8340 [ 4223.822641] RBP: ffff9f38401a8340 R08: ffffc4f074f0bc00 R09: 0000000000000000 [ 4223.822641] R10: 0000000000000010 R11: 0000000000000018 R12: ffff9f384005e200 [ 4223.822641] R13: 0000000000000004 R14: 0000000000000046 R15: 0000000000000004 [ 4223.822641] ? qi_submit_sync+0x2c0/0x490 [ 4223.822642] ? qi_submit_sync+0x2c0/0x490 [ 4223.822642] [ 4223.822642] qi_flush_dev_iotlb+0xb1/0xd0 [ 4223.822642] __dmar_remove_one_dev_info+0x224/0x250 [ 4223.822643] dmar_remove_one_dev_info+0x3e/0x50 [ 4223.822643] intel_iommu_release_device+0x1f/0x30 [ 4223.822643] iommu_release_device+0x33/0x60 [ 4223.822643] iommu_bus_notifier+0x7f/0x90 [ 4223.822644] blocking_notifier_call_chain+0x60/0x90 [ 4223.822644] device_del+0x2e5/0x420 [ 4223.822644] pci_remove_bus_device+0x70/0x110 [ 4223.822644] pciehp_unconfigure_device+0x7c/0x130 [ 4223.822644] pciehp_disable_slot+0x6b/0x100 [ 4223.822645] pciehp_handle_presence_or_link_change+0xd8/0x320 [ 4223.822645] pciehp_ist+0x176/0x180 [ 4223.822645] ? irq_finalize_oneshot.part.50+0x110/0x110 [ 4223.822645] irq_thread_fn+0x19/0x50 [ 4223.822646] irq_thread+0x104/0x190 [ 4223.822646] ? irq_forced_thread_fn+0x90/0x90 [ 4223.822646] ? irq_thread_check_affinity+0xe0/0xe0 [ 4223.822646] kthread+0x114/0x130 [ 4223.822647] ? __kthread_cancel_work+0x40/0x40 [ 4223.822647] ret_from_fork+0x1f/0x30 [ 4223.822647] Kernel Offset: 0x6400000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff) Such issue could be triggered by all kinds of regular surprise removal hotplug operation. like: 1. pull EP(endpoint device) out directly. 2. turn off EP's power. 3. bring the link down. etc. this patch aims to work for regular safe removal and surprise removal unplug. these hot unplug handling process could be optimized for fix the ATS Invalidation hang issue by calling pci_dev_is_disconnected() in function devtlb_invalidation_with_pasid() to check target device state to avoid sending meaningless ATS Invalidation request to iommu when device is gone. (see IMPLEMENTATION NOTE in PCIe spec r6.1 section 10.3.1) For safe removal, device wouldn't be removed until the whole software handling process is done, it wouldn't trigger the hard lock up issue caused by too long ATS Invalidation timeout wait. In safe removal path, device state isn't set to pci_channel_io_perm_failure in pciehp_unconfigure_device() by checking 'presence' parameter, calling pci_dev_is_disconnected() in devtlb_invalidation_with_pasid() will return false there, wouldn't break the function. For surprise removal, device state is set to pci_channel_io_perm_failure in pciehp_unconfigure_device(), means device is already gone (disconnected) call pci_dev_is_disconnected() in devtlb_invalidation_with_pasid() will return true to break the function not to send ATS Invalidation request to the disconnected device blindly, thus avoid to trigger further ITE fault, and ITE fault will block all invalidation request to be handled. furthermore retry the timeout request could trigger hard lockup. safe removal (present) & surprise removal (not present) pciehp_ist() pciehp_handle_presence_or_link_change() pciehp_disable_slot() remove_board() pciehp_unconfigure_device(presence) { if (!presence) pci_walk_bus(parent, pci_dev_set_disconnected, NULL); } this patch works for regular safe removal and surprise removal of ATS capable endpoint on PCIe switch downstream ports. Fixes: 6f7db75e1c46 ("iommu/vt-d: Add second level page table interface") Reviewed-by: Dan Carpenter Tested-by: Haorong Ye Signed-off-by: Ethan Zhao Link: https://lore.kernel.org/r/20240301080727.3529832-3-haifeng.zhao@linux.intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 3239cefa4c33..953592125e4a 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -214,6 +214,9 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, if (!info || !info->ats_enabled) return; + if (pci_dev_is_disconnected(to_pci_dev(dev))) + return; + sid = info->bus << 8 | info->devfn; qdep = info->ats_qdep; pfsid = info->pfsid; From 80a9b50c0b9e297669a8a400eb35468cd87a9aed Mon Sep 17 00:00:00 2001 From: Ethan Zhao Date: Tue, 5 Mar 2024 20:21:16 +0800 Subject: [PATCH 1006/1038] iommu/vt-d: Improve ITE fault handling if target device isn't present Because surprise removal could happen anytime, e.g. user could request safe removal to EP(endpoint device) via sysfs and brings its link down to do surprise removal cocurrently. such aggressive cases would cause ATS invalidation request issued to non-existence target device, then deadly loop to retry that request after ITE fault triggered in interrupt context. this patch aims to optimize the ITE handling by checking the target device presence state to avoid retrying the timeout request blindly, thus avoid hard lockup or system hang. Devices TLB should only be invalidated when devices are in the iommu->device_rbtree (probed, not released) and present. Fixes: 6ba6c3a4cacf ("VT-d: add device IOTLB invalidation support") Reviewed-by: Dan Carpenter Signed-off-by: Ethan Zhao Link: https://lore.kernel.org/r/20240301080727.3529832-4-haifeng.zhao@linux.intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index d14797aabb7a..36d7427b1202 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1273,6 +1273,8 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) { u32 fault; int head, tail; + struct device *dev; + u64 iqe_err, ite_sid; struct q_inval *qi = iommu->qi; int shift = qi_shift(iommu); @@ -1317,6 +1319,13 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) tail = readl(iommu->reg + DMAR_IQT_REG); tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; + /* + * SID field is valid only when the ITE field is Set in FSTS_REG + * see Intel VT-d spec r4.1, section 11.4.9.9 + */ + iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG); + ite_sid = DMAR_IQER_REG_ITESID(iqe_err); + writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG); pr_info("Invalidation Time-out Error (ITE) cleared\n"); @@ -1326,6 +1335,19 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) head = (head - 2 + QI_LENGTH) % QI_LENGTH; } while (head != tail); + /* + * If device was released or isn't present, no need to retry + * the ATS invalidate request anymore. + * + * 0 value of ite_sid means old VT-d device, no ite_sid value. + * see Intel VT-d spec r4.1, section 11.4.9.9 + */ + if (ite_sid) { + dev = device_rbtree_find(iommu, ite_sid); + if (!dev || !dev_is_pci(dev) || + !pci_device_is_present(to_pci_dev(dev))) + return -ETIMEDOUT; + } if (qi->desc_status[wait_index] == QI_ABORT) return -EAGAIN; } From 0061ffe289e19caabeea8103e69cb0f1896e34d8 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 5 Mar 2024 20:21:17 +0800 Subject: [PATCH 1007/1038] iommu: Add static iommu_ops->release_domain The current device_release callback for individual iommu drivers does the following: 1) Silent IOMMU DMA translation: It detaches any existing domain from the device and puts it into a blocking state (some drivers might use the identity state). 2) Resource release: It releases resources allocated during the device_probe callback and restores the device to its pre-probe state. Step 1 is challenging for individual iommu drivers because each must check if a domain is already attached to the device. Additionally, if a deferred attach never occurred, the device_release should avoid modifying hardware configuration regardless of the reason for its call. To simplify this process, introduce a static release_domain within the iommu_ops structure. It can be either a blocking or identity domain depending on the iommu hardware. The iommu core will decide whether to attach this domain before the device_release callback, eliminating the need for repetitive code in various drivers. Consequently, the device_release callback can focus solely on the opposite operations of device_probe, including releasing all resources allocated during that callback. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240305013305.204605-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 19 +++++++++++++++---- include/linux/iommu.h | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..cd1210026ac5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -463,13 +463,24 @@ static void iommu_deinit_device(struct device *dev) /* * release_device() must stop using any attached domain on the device. - * If there are still other devices in the group they are not effected + * If there are still other devices in the group, they are not affected * by this callback. * - * The IOMMU driver must set the device to either an identity or - * blocking translation and stop using any domain pointer, as it is - * going to be freed. + * If the iommu driver provides release_domain, the core code ensures + * that domain is attached prior to calling release_device. Drivers can + * use this to enforce a translation on the idle iommu. Typically, the + * global static blocked_domain is a good choice. + * + * Otherwise, the iommu driver must set the device to either an identity + * or a blocking translation in release_device() and stop using any + * domain pointer, as it is going to be freed. + * + * Regardless, if a delayed attach never occurred, then the release + * should still avoid touching any hardware configuration either. */ + if (!dev->iommu->attach_deferred && ops->release_domain) + ops->release_domain->ops->attach_dev(ops->release_domain, dev); + if (ops->release_device) ops->release_device(dev); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1ea2a820e1eb..a0a07e1680a2 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -487,6 +487,7 @@ struct iommu_ops { struct module *owner; struct iommu_domain *identity_domain; struct iommu_domain *blocked_domain; + struct iommu_domain *release_domain; struct iommu_domain *default_domain; }; From 81e921fd321614c2ad8ac333b041aae1da7a1c6d Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 5 Mar 2024 20:21:18 +0800 Subject: [PATCH 1008/1038] iommu/vt-d: Fix NULL domain on device release In the kdump kernel, the IOMMU operates in deferred_attach mode. In this mode, info->domain may not yet be assigned by the time the release_device function is called. It leads to the following crash in the crash kernel: BUG: kernel NULL pointer dereference, address: 000000000000003c ... RIP: 0010:do_raw_spin_lock+0xa/0xa0 ... _raw_spin_lock_irqsave+0x1b/0x30 intel_iommu_release_device+0x96/0x170 iommu_deinit_device+0x39/0xf0 __iommu_group_remove_device+0xa0/0xd0 iommu_bus_notifier+0x55/0xb0 notifier_call_chain+0x5a/0xd0 blocking_notifier_call_chain+0x41/0x60 bus_notify+0x34/0x50 device_del+0x269/0x3d0 pci_remove_bus_device+0x77/0x100 p2sb_bar+0xae/0x1d0 ... i801_probe+0x423/0x740 Use the release_domain mechanism to fix it. The scalable mode context entry which is not part of release domain should be cleared in release_device(). Fixes: 586081d3f6b1 ("iommu/vt-d: Remove DEFER_DEVICE_DOMAIN_INFO") Reported-by: Eric Badger Closes: https://lore.kernel.org/r/20240113181713.1817855-1-ebadger@purestorage.com Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240305013305.204605-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 31 ++++-------------- drivers/iommu/intel/pasid.c | 64 +++++++++++++++++++++++++++++++++++++ drivers/iommu/intel/pasid.h | 1 + 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 60aa2dce32ef..eff7abcc420b 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3818,30 +3818,6 @@ static void domain_context_clear(struct device_domain_info *info) &domain_context_clear_one_cb, info); } -static void dmar_remove_one_dev_info(struct device *dev) -{ - struct device_domain_info *info = dev_iommu_priv_get(dev); - struct dmar_domain *domain = info->domain; - struct intel_iommu *iommu = info->iommu; - unsigned long flags; - - if (!dev_is_real_dma_subdevice(info->dev)) { - if (dev_is_pci(info->dev) && sm_supported(iommu)) - intel_pasid_tear_down_entry(iommu, info->dev, - IOMMU_NO_PASID, false); - - iommu_disable_pci_caps(info); - domain_context_clear(info); - } - - spin_lock_irqsave(&domain->lock, flags); - list_del(&info->link); - spin_unlock_irqrestore(&domain->lock, flags); - - domain_detach_iommu(domain, iommu); - info->domain = NULL; -} - /* * Clear the page table pointer in context or pasid table entries so that * all DMA requests without PASID from the device are blocked. If the page @@ -4367,7 +4343,11 @@ static void intel_iommu_release_device(struct device *dev) mutex_lock(&iommu->iopf_lock); device_rbtree_remove(info); mutex_unlock(&iommu->iopf_lock); - dmar_remove_one_dev_info(dev); + + if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) && + !context_copied(iommu, info->bus, info->devfn)) + intel_pasid_teardown_sm_context(dev); + intel_pasid_free_table(dev); intel_iommu_debugfs_remove_dev(info); kfree(info); @@ -4826,6 +4806,7 @@ static const struct iommu_dirty_ops intel_dirty_ops = { const struct iommu_ops intel_iommu_ops = { .blocked_domain = &blocking_domain, + .release_domain = &blocking_domain, .capable = intel_iommu_capable, .hw_info = intel_iommu_hw_info, .domain_alloc = intel_iommu_domain_alloc, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 953592125e4a..135ed1651124 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -669,3 +669,67 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, return 0; } + +/* + * Interfaces to setup or teardown a pasid table to the scalable-mode + * context table entry: + */ + +static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct context_entry *context; + + spin_lock(&iommu->lock); + context = iommu_context_addr(iommu, bus, devfn, false); + if (!context) { + spin_unlock(&iommu->lock); + return; + } + + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + spin_unlock(&iommu->lock); + + /* + * Cache invalidation for changes to a scalable-mode context table + * entry. + * + * Section 6.5.3.3 of the VT-d spec: + * - Device-selective context-cache invalidation; + * - Domain-selective PASID-cache invalidation to affected domains + * (can be skipped if all PASID entries were not-present); + * - Domain-selective IOTLB invalidation to affected domains; + * - Global Device-TLB invalidation to affected functions. + * + * The iommu has been parked in the blocking state. All domains have + * been detached from the device or PASID. The PASID and IOTLB caches + * have been invalidated during the domain detach path. + */ + iommu->flush.flush_context(iommu, 0, PCI_DEVID(bus, devfn), + DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); + devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); +} + +static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data) +{ + struct device *dev = data; + + if (dev == &pdev->dev) + device_pasid_table_teardown(dev, PCI_BUS_NUM(alias), alias & 0xff); + + return 0; +} + +void intel_pasid_teardown_sm_context(struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + if (!dev_is_pci(dev)) { + device_pasid_table_teardown(dev, info->bus, info->devfn); + return; + } + + pci_for_each_dma_alias(to_pci_dev(dev), pci_pasid_table_teardown, dev); +} diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 8d40d4c66e31..c299e4c6e94b 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -319,4 +319,5 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, bool fault_ignore); void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu, struct device *dev, u32 pasid); +void intel_pasid_teardown_sm_context(struct device *dev); #endif /* __INTEL_PASID_H */ From 301f1a80487fd2f51012533792583d4425e8b8c0 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 5 Mar 2024 20:21:19 +0800 Subject: [PATCH 1009/1038] iommu/vt-d: Setup scalable mode context entry in probe path In contrast to legacy mode, the DMA translation table is configured in the PASID table entry instead of the context entry for scalable mode. For this reason, it is more appropriate to set up the scalable mode context entry in the device_probe callback and direct it to the appropriate PASID table. The iommu domain attach/detach operations only affect the PASID table entry. Therefore, there is no need to modify the context entry when configuring the translation type and page table. The only exception is the kdump case, where context entry setup is postponed until the device driver invokes the first DMA interface. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240305013305.204605-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 12 ++++ drivers/iommu/intel/pasid.c | 138 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intel/pasid.h | 1 + 3 files changed, 151 insertions(+) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index eff7abcc420b..659c49f98f7f 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4012,6 +4012,10 @@ int prepare_domain_attach_device(struct iommu_domain *domain, dmar_domain->agaw--; } + if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) && + context_copied(iommu, info->bus, info->devfn)) + return intel_pasid_setup_sm_context(dev); + return 0; } @@ -4322,11 +4326,19 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) dev_err(dev, "PASID table allocation failed\n"); goto clear_rbtree; } + + if (!context_copied(iommu, info->bus, info->devfn)) { + ret = intel_pasid_setup_sm_context(dev); + if (ret) + goto free_table; + } } intel_iommu_debugfs_create_dev(info); return &iommu->iommu; +free_table: + intel_pasid_free_table(dev); clear_rbtree: device_rbtree_remove(info); free: diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 135ed1651124..e8e3f4a50b9e 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -733,3 +733,141 @@ void intel_pasid_teardown_sm_context(struct device *dev) pci_for_each_dma_alias(to_pci_dev(dev), pci_pasid_table_teardown, dev); } + +/* + * Get the PASID directory size for scalable mode context entry. + * Value of X in the PDTS field of a scalable mode context entry + * indicates PASID directory with 2^(X + 7) entries. + */ +static unsigned long context_get_sm_pds(struct pasid_table *table) +{ + unsigned long pds, max_pde; + + max_pde = table->max_pasid >> PASID_PDE_SHIFT; + pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS); + if (pds < 7) + return 0; + + return pds - 7; +} + +static int context_entry_set_pasid_table(struct context_entry *context, + struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct pasid_table *table = info->pasid_table; + struct intel_iommu *iommu = info->iommu; + unsigned long pds; + + context_clear_entry(context); + + pds = context_get_sm_pds(table); + context->lo = (u64)virt_to_phys(table->table) | context_pdts(pds); + context_set_sm_rid2pasid(context, IOMMU_NO_PASID); + + if (info->ats_supported) + context_set_sm_dte(context); + if (info->pri_supported) + context_set_sm_pre(context); + if (info->pasid_supported) + context_set_pasid(context); + + context_set_fault_enable(context); + context_set_present(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + + return 0; +} + +static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct context_entry *context; + + spin_lock(&iommu->lock); + context = iommu_context_addr(iommu, bus, devfn, true); + if (!context) { + spin_unlock(&iommu->lock); + return -ENOMEM; + } + + if (context_present(context) && !context_copied(iommu, bus, devfn)) { + spin_unlock(&iommu->lock); + return 0; + } + + if (context_copied(iommu, bus, devfn)) { + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + + /* + * For kdump cases, old valid entries may be cached due to + * the in-flight DMA and copied pgtable, but there is no + * unmapping behaviour for them, thus we need explicit cache + * flushes for all affected domain IDs and PASIDs used in + * the copied PASID table. Given that we have no idea about + * which domain IDs and PASIDs were used in the copied tables, + * upgrade them to global PASID and IOTLB cache invalidation. + */ + iommu->flush.flush_context(iommu, 0, + PCI_DEVID(bus, devfn), + DMA_CCMD_MASK_NOBIT, + DMA_CCMD_DEVICE_INVL); + qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); + devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); + + /* + * At this point, the device is supposed to finish reset at + * its driver probe stage, so no in-flight DMA will exist, + * and we don't need to worry anymore hereafter. + */ + clear_context_copied(iommu, bus, devfn); + } + + context_entry_set_pasid_table(context, dev); + spin_unlock(&iommu->lock); + + /* + * It's a non-present to present mapping. If hardware doesn't cache + * non-present entry we don't need to flush the caches. If it does + * cache non-present entries, then it does so in the special + * domain #0, which we have to flush: + */ + if (cap_caching_mode(iommu->cap)) { + iommu->flush.flush_context(iommu, 0, + PCI_DEVID(bus, devfn), + DMA_CCMD_MASK_NOBIT, + DMA_CCMD_DEVICE_INVL); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH); + } + + return 0; +} + +static int pci_pasid_table_setup(struct pci_dev *pdev, u16 alias, void *data) +{ + struct device *dev = data; + + if (dev != &pdev->dev) + return 0; + + return device_pasid_table_setup(dev, PCI_BUS_NUM(alias), alias & 0xff); +} + +/* + * Set the device's PASID table to its context table entry. + * + * The PASID table is set to the context entries of both device itself + * and its alias requester ID for DMA. + */ +int intel_pasid_setup_sm_context(struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + if (!dev_is_pci(dev)) + return device_pasid_table_setup(dev, info->bus, info->devfn); + + return pci_for_each_dma_alias(to_pci_dev(dev), pci_pasid_table_setup, dev); +} diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index c299e4c6e94b..045b9ec5b8e8 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -319,5 +319,6 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, bool fault_ignore); void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu, struct device *dev, u32 pasid); +int intel_pasid_setup_sm_context(struct device *dev); void intel_pasid_teardown_sm_context(struct device *dev); #endif /* __INTEL_PASID_H */ From a016e53843ed8bb9cccb832768e2be9856b0a913 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 5 Mar 2024 20:21:20 +0800 Subject: [PATCH 1010/1038] iommu/vt-d: Remove scalable mode context entry setup from attach_dev The scalable mode context entry is now setup in the probe_device path, eliminating the need to configure it in the attach_dev path. Removes the redundant code from the attach_dev path to avoid dead code. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240305013305.204605-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 156 ++++++++++-------------------------- 1 file changed, 44 insertions(+), 112 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 659c49f98f7f..e752258d42b0 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1802,34 +1802,17 @@ static void domain_exit(struct dmar_domain *domain) kfree(domain); } -/* - * Get the PASID directory size for scalable mode context entry. - * Value of X in the PDTS field of a scalable mode context entry - * indicates PASID directory with 2^(X + 7) entries. - */ -static unsigned long context_get_sm_pds(struct pasid_table *table) -{ - unsigned long pds, max_pde; - - max_pde = table->max_pasid >> PASID_PDE_SHIFT; - pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS); - if (pds < 7) - return 0; - - return pds - 7; -} - static int domain_context_mapping_one(struct dmar_domain *domain, struct intel_iommu *iommu, - struct pasid_table *table, u8 bus, u8 devfn) { struct device_domain_info *info = domain_lookup_dev_info(domain, iommu, bus, devfn); u16 did = domain_id_iommu(domain, iommu); int translation = CONTEXT_TT_MULTI_LEVEL; + struct dma_pte *pgd = domain->pgd; struct context_entry *context; - int ret; + int agaw, ret; if (hw_pass_through && domain_type_is_si(domain)) translation = CONTEXT_TT_PASS_THROUGH; @@ -1872,65 +1855,37 @@ static int domain_context_mapping_one(struct dmar_domain *domain, } context_clear_entry(context); + context_set_domain_id(context, did); - if (sm_supported(iommu)) { - unsigned long pds; - - /* Setup the PASID DIR pointer: */ - pds = context_get_sm_pds(table); - context->lo = (u64)virt_to_phys(table->table) | - context_pdts(pds); - - /* Setup the RID_PASID field: */ - context_set_sm_rid2pasid(context, IOMMU_NO_PASID); - + if (translation != CONTEXT_TT_PASS_THROUGH) { /* - * Setup the Device-TLB enable bit and Page request - * Enable bit: + * Skip top levels of page tables for iommu which has + * less agaw than default. Unnecessary for PT mode. */ - if (info && info->ats_supported) - context_set_sm_dte(context); - if (info && info->pri_supported) - context_set_sm_pre(context); - if (info && info->pasid_supported) - context_set_pasid(context); - } else { - struct dma_pte *pgd = domain->pgd; - int agaw; - - context_set_domain_id(context, did); - - if (translation != CONTEXT_TT_PASS_THROUGH) { - /* - * Skip top levels of page tables for iommu which has - * less agaw than default. Unnecessary for PT mode. - */ - for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { - ret = -ENOMEM; - pgd = phys_to_virt(dma_pte_addr(pgd)); - if (!dma_pte_present(pgd)) - goto out_unlock; - } - - if (info && info->ats_supported) - translation = CONTEXT_TT_DEV_IOTLB; - else - translation = CONTEXT_TT_MULTI_LEVEL; - - context_set_address_root(context, virt_to_phys(pgd)); - context_set_address_width(context, agaw); - } else { - /* - * In pass through mode, AW must be programmed to - * indicate the largest AGAW value supported by - * hardware. And ASR is ignored by hardware. - */ - context_set_address_width(context, iommu->msagaw); + for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { + ret = -ENOMEM; + pgd = phys_to_virt(dma_pte_addr(pgd)); + if (!dma_pte_present(pgd)) + goto out_unlock; } - context_set_translation_type(context, translation); + if (info && info->ats_supported) + translation = CONTEXT_TT_DEV_IOTLB; + else + translation = CONTEXT_TT_MULTI_LEVEL; + + context_set_address_root(context, virt_to_phys(pgd)); + context_set_address_width(context, agaw); + } else { + /* + * In pass through mode, AW must be programmed to + * indicate the largest AGAW value supported by + * hardware. And ASR is ignored by hardware. + */ + context_set_address_width(context, iommu->msagaw); } + context_set_translation_type(context, translation); context_set_fault_enable(context); context_set_present(context); if (!ecap_coherent(iommu->ecap)) @@ -1960,43 +1915,29 @@ out_unlock: return ret; } -struct domain_context_mapping_data { - struct dmar_domain *domain; - struct intel_iommu *iommu; - struct pasid_table *table; -}; - static int domain_context_mapping_cb(struct pci_dev *pdev, u16 alias, void *opaque) { - struct domain_context_mapping_data *data = opaque; + struct device_domain_info *info = dev_iommu_priv_get(&pdev->dev); + struct intel_iommu *iommu = info->iommu; + struct dmar_domain *domain = opaque; - return domain_context_mapping_one(data->domain, data->iommu, - data->table, PCI_BUS_NUM(alias), - alias & 0xff); + return domain_context_mapping_one(domain, iommu, + PCI_BUS_NUM(alias), alias & 0xff); } static int domain_context_mapping(struct dmar_domain *domain, struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); - struct domain_context_mapping_data data; struct intel_iommu *iommu = info->iommu; u8 bus = info->bus, devfn = info->devfn; - struct pasid_table *table; - - table = intel_pasid_get_table(dev); if (!dev_is_pci(dev)) - return domain_context_mapping_one(domain, iommu, table, - bus, devfn); - - data.domain = domain; - data.iommu = iommu; - data.table = table; + return domain_context_mapping_one(domain, iommu, bus, devfn); return pci_for_each_dma_alias(to_pci_dev(dev), - &domain_context_mapping_cb, &data); + domain_context_mapping_cb, domain); } /* Returns a number of VTD pages, but aligned to MM page size */ @@ -2353,28 +2294,19 @@ static int dmar_domain_attach_device(struct dmar_domain *domain, list_add(&info->link, &domain->devices); spin_unlock_irqrestore(&domain->lock, flags); - /* PASID table is mandatory for a PCI device in scalable mode. */ - if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { - /* Setup the PASID entry for requests without PASID: */ - if (hw_pass_through && domain_type_is_si(domain)) - ret = intel_pasid_setup_pass_through(iommu, - dev, IOMMU_NO_PASID); - else if (domain->use_first_level) - ret = domain_setup_first_level(iommu, domain, dev, - IOMMU_NO_PASID); - else - ret = intel_pasid_setup_second_level(iommu, domain, - dev, IOMMU_NO_PASID); - if (ret) { - dev_err(dev, "Setup RID2PASID failed\n"); - device_block_translation(dev); - return ret; - } - } + if (dev_is_real_dma_subdevice(dev)) + return 0; + + if (!sm_supported(iommu)) + ret = domain_context_mapping(domain, dev); + else if (hw_pass_through && domain_type_is_si(domain)) + ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); + else if (domain->use_first_level) + ret = domain_setup_first_level(iommu, domain, dev, IOMMU_NO_PASID); + else + ret = intel_pasid_setup_second_level(iommu, domain, dev, IOMMU_NO_PASID); - ret = domain_context_mapping(domain, dev); if (ret) { - dev_err(dev, "Domain context map failed\n"); device_block_translation(dev); return ret; } From 80ca79f398bff2708ee7a19d5904804415680aad Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 5 Mar 2024 20:21:21 +0800 Subject: [PATCH 1011/1038] iommu/vt-d: Remove scalabe mode in domain_context_clear_one() domain_context_clear_one() only handles the context entry teardown in legacy mode. Remove the scalable mode check in it to avoid dead code. Remove an unnecessary check in the code as well. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240305013305.204605-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index e752258d42b0..6fb1eb7a78ab 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2124,9 +2124,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 struct context_entry *context; u16 did_old; - if (!iommu) - return; - spin_lock(&iommu->lock); context = iommu_context_addr(iommu, bus, devfn, 0); if (!context) { @@ -2134,14 +2131,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 return; } - if (sm_supported(iommu)) { - if (hw_pass_through && domain_type_is_si(info->domain)) - did_old = FLPT_DEFAULT_DID; - else - did_old = domain_id_iommu(info->domain, iommu); - } else { - did_old = context_domain_id(context); - } + did_old = context_domain_id(context); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); @@ -2152,9 +2142,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); - if (sm_supported(iommu)) - qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0); - iommu->flush.flush_iotlb(iommu, did_old, 0, From e2addba4930558a6f37a2e4b2cb6f726638a6dce Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 4 Mar 2024 12:05:42 +0000 Subject: [PATCH 1012/1038] iommu/dma: Document min_align_mask assumption iommu-dma does not explicitly reference min_align_mask since we already assume that will be less than or equal to any typical IOVA granule. We wouldn't realistically expect to see the case where it is larger, and that would be non-trivial to support, however for the sake of reasoning (particularly around the interaction with SWIOTLB), let's clearly enforce the assumption. Signed-off-by: Robin Murphy Reviewed-by: Michael Kelley Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/dbb4d2d8e5d1691ac9a6c67e9758904e6c447ba5.1709553942.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 50ccc4f1ef81..b58f5a3311c3 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -859,6 +859,11 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, iommu_deferred_attach(dev, domain)) return DMA_MAPPING_ERROR; + /* If anyone ever wants this we'd need support in the IOVA allocator */ + if (dev_WARN_ONCE(dev, dma_get_min_align_mask(dev) > iova_mask(iovad), + "Unsupported alignment constraint\n")) + return DMA_MAPPING_ERROR; + size = iova_align(iovad, size + iova_off); iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev); From f30e5ed1306be8a900b33317bc429dd3794d81a1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 6 Mar 2024 07:27:39 -0700 Subject: [PATCH 1013/1038] dm-integrity: set max_integrity_segments in dm_integrity_io_hints Set max_integrity_segments with the other queue limits instead of updating it later. This also uncovered that the driver is trying to set the limit to UINT_MAX while max_integrity_segments is an unsigned short, so fix it up to use USHRT_MAX instead. Signed-off-by: Christoph Hellwig Signed-off-by: Mike Snitzer --- drivers/md/dm-integrity.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index c3c666937112..bb3486bfab77 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -3487,6 +3487,7 @@ static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *lim blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT); limits->dma_alignment = limits->logical_block_size - 1; } + limits->max_integrity_segments = USHRT_MAX; } static void calculate_journal_section_size(struct dm_integrity_c *ic) @@ -3654,7 +3655,6 @@ static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic) bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT; blk_integrity_register(disk, &bi); - blk_queue_max_integrity_segments(disk->queue, UINT_MAX); } static void dm_integrity_free_page_list(struct page_list *pl) From 5ee91605ad9ad363766a7ed13dc7d47f5102982a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 6 Mar 2024 17:59:40 +0200 Subject: [PATCH 1014/1038] spi: Exctract spi_set_all_cs_unused() helper It seems a few functions implement the similar for-loop to mark all chip select pins unused. Let's deduplicate that code in order to have a single place of that for better maintenance. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240306160114.3471398-2-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 74 +++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index ba4d3fde2054..5e530fa790b0 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -745,6 +745,23 @@ int spi_add_device(struct spi_device *spi) } EXPORT_SYMBOL_GPL(spi_add_device); +static void spi_set_all_cs_unused(struct spi_device *spi) +{ + u8 idx; + + /* + * Zero(0) is a valid physical CS value and can be located at any + * logical CS in the spi->chip_select[]. If all the physical CS + * are initialized to 0 then It would be difficult to differentiate + * between a valid physical CS 0 & an unused logical CS whose physical + * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. + * Now all the unused logical CS will have 0xFF physical CS value & can be + * ignore while performing physical CS validity checks. + */ + for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) + spi_set_chipselect(spi, idx, 0xFF); +} + /** * spi_new_device - instantiate one new SPI device * @ctlr: Controller to which device is connected @@ -764,7 +781,6 @@ struct spi_device *spi_new_device(struct spi_controller *ctlr, { struct spi_device *proxy; int status; - u8 idx; /* * NOTE: caller did any chip->bus_num checks necessary. @@ -780,19 +796,10 @@ struct spi_device *spi_new_device(struct spi_controller *ctlr, WARN_ON(strlen(chip->modalias) >= sizeof(proxy->modalias)); - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(proxy, idx, 0xFF); - + /* Use provided chip-select for proxy device */ + spi_set_all_cs_unused(proxy); spi_set_chipselect(proxy, 0, chip->chip_select); + proxy->max_speed_hz = chip->max_speed_hz; proxy->mode = chip->mode; proxy->irq = chip->irq; @@ -2418,17 +2425,7 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi, return -EINVAL; } - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(spi, idx, 0xFF); + spi_set_all_cs_unused(spi); /* Device address */ rc = of_property_read_variable_u32_array(nc, "reg", &cs[0], 1, @@ -2565,7 +2562,6 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi, struct spi_controller *ctlr = spi->controller; struct spi_device *ancillary; int rc = 0; - u8 idx; /* Alloc an spi_device */ ancillary = spi_alloc_device(ctlr); @@ -2576,19 +2572,8 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi, strscpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias)); - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(ancillary, idx, 0xFF); - /* Use provided chip-select for ancillary device */ + spi_set_all_cs_unused(ancillary); spi_set_chipselect(ancillary, 0, chip_select); /* Take over SPI mode/speed from SPI main device */ @@ -2805,7 +2790,6 @@ struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr, struct acpi_spi_lookup lookup = {}; struct spi_device *spi; int ret; - u8 idx; if (!ctlr && index == -1) return ERR_PTR(-EINVAL); @@ -2841,24 +2825,14 @@ struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr, return ERR_PTR(-ENOMEM); } - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(spi, idx, 0xFF); + spi_set_all_cs_unused(spi); + spi_set_chipselect(spi, 0, lookup.chip_select); ACPI_COMPANION_SET(&spi->dev, adev); spi->max_speed_hz = lookup.max_speed_hz; spi->mode |= lookup.mode; spi->irq = lookup.irq; spi->bits_per_word = lookup.bits_per_word; - spi_set_chipselect(spi, 0, lookup.chip_select); /* * spi->chip_select[i] gives the corresponding physical CS for logical CS i * logical CS number is represented by setting the ith bit in spi->cs_index_mask From 9086d0f23b7c292f162a828967975e29e97c0680 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 6 Mar 2024 17:59:41 +0200 Subject: [PATCH 1015/1038] spi: Exctract spi_dev_check_cs() helper It seems a few functions implement the similar for-loop to validate chip select pins for uniqueness. Let's deduplicate that code in order to have a single place of that for better maintenance. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240306160114.3471398-3-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 5e530fa790b0..eb7e3ddf909b 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -608,23 +608,35 @@ static void spi_dev_set_name(struct spi_device *spi) spi_get_chipselect(spi, 0)); } +static inline int spi_dev_check_cs(struct device *dev, + struct spi_device *spi, u8 idx, + struct spi_device *new_spi, u8 new_idx) +{ + u8 cs, cs_new; + u8 idx_new; + + cs = spi_get_chipselect(spi, idx); + for (idx_new = new_idx; idx_new < SPI_CS_CNT_MAX; idx_new++) { + cs_new = spi_get_chipselect(new_spi, idx_new); + if (cs != 0xFF && cs_new != 0xFF && cs == cs_new) { + dev_err(dev, "chipselect %u already in use\n", cs_new); + return -EBUSY; + } + } + return 0; +} + static int spi_dev_check(struct device *dev, void *data) { struct spi_device *spi = to_spi_device(dev); struct spi_device *new_spi = data; - int idx, nw_idx; - u8 cs, cs_nw; + int status, idx; if (spi->controller == new_spi->controller) { for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - cs = spi_get_chipselect(spi, idx); - for (nw_idx = 0; nw_idx < SPI_CS_CNT_MAX; nw_idx++) { - cs_nw = spi_get_chipselect(new_spi, nw_idx); - if (cs != 0xFF && cs_nw != 0xFF && cs == cs_nw) { - dev_err(dev, "chipselect %d already in use\n", cs_nw); - return -EBUSY; - } - } + status = spi_dev_check_cs(dev, spi, idx, new_spi, 0); + if (status) + return status; } } return 0; @@ -640,8 +652,8 @@ static int __spi_add_device(struct spi_device *spi) { struct spi_controller *ctlr = spi->controller; struct device *dev = ctlr->dev.parent; - int status, idx, nw_idx; - u8 cs, nw_cs; + int status, idx; + u8 cs; for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { /* Chipselects are numbered 0..max; validate. */ @@ -658,14 +670,9 @@ static int __spi_add_device(struct spi_device *spi) * For example, spi->chip_select[0] != spi->chip_select[1] and so on. */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - cs = spi_get_chipselect(spi, idx); - for (nw_idx = idx + 1; nw_idx < SPI_CS_CNT_MAX; nw_idx++) { - nw_cs = spi_get_chipselect(spi, nw_idx); - if (cs != 0xFF && nw_cs != 0xFF && cs == nw_cs) { - dev_err(dev, "chipselect %d already in use\n", nw_cs); - return -EBUSY; - } - } + status = spi_dev_check_cs(dev, spi, idx, spi, idx + 1); + if (status) + return status; } /* Set the bus ID string */ From 4af59a8df5ea930038cd3355e822f5eedf4accc1 Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Wed, 6 Mar 2024 10:44:38 +0900 Subject: [PATCH 1016/1038] mmc: core: Fix switch on gp3 partition Commit e7794c14fd73 ("mmc: rpmb: fixes pause retune on all RPMB partitions.") added a mask check for 'part_type', but the mask used was wrong leading to the code intended for rpmb also being executed for GP3. On some MMCs (but not all) this would make gp3 partition inaccessible: armadillo:~# head -c 1 < /dev/mmcblk2gp3 head: standard input: I/O error armadillo:~# dmesg -c [ 422.976583] mmc2: running CQE recovery [ 423.058182] mmc2: running CQE recovery [ 423.137607] mmc2: running CQE recovery [ 423.137802] blk_update_request: I/O error, dev mmcblk2gp3, sector 0 op 0x0:(READ) flags 0x80700 phys_seg 4 prio class 0 [ 423.237125] mmc2: running CQE recovery [ 423.318206] mmc2: running CQE recovery [ 423.397680] mmc2: running CQE recovery [ 423.397837] blk_update_request: I/O error, dev mmcblk2gp3, sector 0 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 0 [ 423.408287] Buffer I/O error on dev mmcblk2gp3, logical block 0, async page read the part_type values of interest here are defined as follow: main 0 boot0 1 boot1 2 rpmb 3 gp0 4 gp1 5 gp2 6 gp3 7 so mask with EXT_CSD_PART_CONFIG_ACC_MASK (7) to correctly identify rpmb Fixes: e7794c14fd73 ("mmc: rpmb: fixes pause retune on all RPMB partitions.") Cc: stable@vger.kernel.org Cc: Jorge Ramirez-Ortiz Signed-off-by: Dominique Martinet Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20240306-mmc-partswitch-v1-1-bf116985d950@codewreck.org Signed-off-by: Ulf Hansson --- drivers/mmc/core/block.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 32d49100dff5..86efa6084696 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -874,10 +874,11 @@ static const struct block_device_operations mmc_bdops = { static int mmc_blk_part_switch_pre(struct mmc_card *card, unsigned int part_type) { - const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB; + const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK; + const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB; int ret = 0; - if ((part_type & mask) == mask) { + if ((part_type & mask) == rpmb) { if (card->ext_csd.cmdq_en) { ret = mmc_cmdq_disable(card); if (ret) @@ -892,10 +893,11 @@ static int mmc_blk_part_switch_pre(struct mmc_card *card, static int mmc_blk_part_switch_post(struct mmc_card *card, unsigned int part_type) { - const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB; + const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK; + const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB; int ret = 0; - if ((part_type & mask) == mask) { + if ((part_type & mask) == rpmb) { mmc_retune_unpause(card->host); if (card->reenable_cmdq && !card->ext_csd.cmdq_en) ret = mmc_cmdq_enable(card); From faf3b8014c357d71c7a9414302e217a1dd1679af Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Tue, 5 Mar 2024 16:49:30 -0300 Subject: [PATCH 1017/1038] mmc: core: make mmc_host_class constant Since commit 43a7206b0963 ("driver core: class: make class_register() take a const *"), the driver core allows for struct class to be in read-only memory, so move the mmc_host_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Link: https://lore.kernel.org/r/20240305-class_cleanup-mmc-v1-1-4a66e7122ff3@marliere.net Signed-off-by: Ulf Hansson --- drivers/mmc/core/host.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 3aa9dda6fd4f..8f8781d6c25e 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -88,7 +88,7 @@ static int mmc_host_classdev_shutdown(struct device *dev) return 0; } -static struct class mmc_host_class = { +static const struct class mmc_host_class = { .name = "mmc_host", .dev_release = mmc_host_classdev_release, .shutdown_pre = mmc_host_classdev_shutdown, From 1209c5566f9b244bd047478b7fc90318c9a310f0 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 7 Mar 2024 17:00:59 +0200 Subject: [PATCH 1018/1038] spi: Consistently use BIT for cs_index_mask Some of the parts related to the chip select are using BIT() macro the rest are using plain numbers. Unify all of them to use BIT(). While at it, make the (repetitive) comment clearer when assigning cs_index_mask during SPI target device enumeration. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240307150256.3789138-2-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index eb7e3ddf909b..4ab155f698c8 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1021,7 +1021,7 @@ static inline bool spi_is_last_cs(struct spi_device *spi) bool last = false; for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - if ((spi->cs_index_mask >> idx) & 0x01) { + if (spi->cs_index_mask & BIT(idx)) { if (spi->controller->last_cs[idx] == spi_get_chipselect(spi, idx)) last = true; } @@ -1072,8 +1072,7 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force) * into account. */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - if (((spi->cs_index_mask >> idx) & 0x01) && - spi_get_csgpiod(spi, idx)) { + if ((spi->cs_index_mask & BIT(idx)) && spi_get_csgpiod(spi, idx)) { if (has_acpi_companion(&spi->dev)) gpiod_set_value_cansleep(spi_get_csgpiod(spi, idx), !enable); @@ -2456,14 +2455,10 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi, spi_set_chipselect(spi, idx, cs[idx]); /* - * spi->chip_select[i] gives the corresponding physical CS for logical CS i - * logical CS number is represented by setting the ith bit in spi->cs_index_mask - * So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and - * spi->chip_select[0] will give the physical CS. - * By default spi->chip_select[0] will hold the physical CS number so, set - * spi->cs_index_mask as 0x01. + * By default spi->chip_select[0] will hold the physical CS number, + * so set bit 0 in spi->cs_index_mask. */ - spi->cs_index_mask = 0x01; + spi->cs_index_mask = BIT(0); /* Device speed */ if (!of_property_read_u32(nc, "spi-max-frequency", &value)) @@ -2587,14 +2582,10 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi, ancillary->max_speed_hz = spi->max_speed_hz; ancillary->mode = spi->mode; /* - * spi->chip_select[i] gives the corresponding physical CS for logical CS i - * logical CS number is represented by setting the ith bit in spi->cs_index_mask - * So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and - * spi->chip_select[0] will give the physical CS. - * By default spi->chip_select[0] will hold the physical CS number so, set - * spi->cs_index_mask as 0x01. + * By default spi->chip_select[0] will hold the physical CS number, + * so set bit 0 in spi->cs_index_mask. */ - ancillary->cs_index_mask = 0x01; + ancillary->cs_index_mask = BIT(0); WARN_ON(!mutex_is_locked(&ctlr->add_lock)); @@ -2841,14 +2832,10 @@ struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr, spi->irq = lookup.irq; spi->bits_per_word = lookup.bits_per_word; /* - * spi->chip_select[i] gives the corresponding physical CS for logical CS i - * logical CS number is represented by setting the ith bit in spi->cs_index_mask - * So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and - * spi->chip_select[0] will give the physical CS. - * By default spi->chip_select[0] will hold the physical CS number so, set - * spi->cs_index_mask as 0x01. + * By default spi->chip_select[0] will hold the physical CS number, + * so set bit 0 in spi->cs_index_mask. */ - spi->cs_index_mask = 0x01; + spi->cs_index_mask = BIT(0); return spi; } From 14fe5a98fb24192f73639590d9d3cdb5640d48db Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 7 Mar 2024 17:01:00 +0200 Subject: [PATCH 1019/1038] spi: Fix types of the last chip select storage variables First of all, last_cs_index_mask should be aligned with the original cs_index_mask, which is 16-bit (for now) wide. Use the same pattern for the last_cs_index_mask. Second, last_cs can be negative and since 'char' is equal to 'unsigned char' in the kernel, it's incorrect, strictly speaking, to assign signed number to it. Use s8 type as it's done for *_native_cs ones. With this change, regroup a bit the ordering to avoid too much memory space to be wasted due to paddings. Shuffle kernel documentation accordignly. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240307150256.3789138-3-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index ddfb66dd4caf..b05d5a87c313 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -450,9 +450,11 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * the @cur_msg_completion. This flag is used to signal the context that * is running spi_finalize_current_message() that it needs to complete() * @cur_msg_mapped: message has been mapped for DMA + * @fallback: fallback to PIO if DMA transfer return failure with + * SPI_TRANS_FAIL_NO_START. + * @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs. * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip * selected - * @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs. * @xfer_completion: used by core transfer_one_message() * @busy: message pump is busy * @running: message pump is running @@ -529,8 +531,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * If the driver does not set this, the SPI core takes the snapshot as * close to the driver hand-over as possible. * @irq_flags: Interrupt enable state during PTP system timestamping - * @fallback: fallback to PIO if DMA transfer return failure with - * SPI_TRANS_FAIL_NO_START. * @queue_empty: signal green light for opportunistically skipping the queue * for spi_sync transfers. * @must_async: disable all fast paths in the core @@ -710,10 +710,10 @@ struct spi_controller { bool rt; bool auto_runtime_pm; bool cur_msg_mapped; - char last_cs[SPI_CS_CNT_MAX]; - char last_cs_index_mask; - bool last_cs_mode_high; bool fallback; + bool last_cs_mode_high; + s8 last_cs[SPI_CS_CNT_MAX]; + u32 last_cs_index_mask : SPI_CS_CNT_MAX; struct completion xfer_completion; size_t max_dma_len; From be84be4a35fa99cca7e81e6dd21516a324cca413 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 7 Mar 2024 17:01:01 +0200 Subject: [PATCH 1020/1038] spi: Introduce SPI_INVALID_CS and is_valid_cs() The SPI core inconsistently uses the marker value for unused chip select pin. Define a constant (with appropriate type) and introduce is_valid_cs() helper function to avoid spreading this inconsistency in the future. Signed-off-by: Andy Shevchenko Link: https://msgid.link/r/20240307150256.3789138-4-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 4ab155f698c8..f18738ae95f8 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -608,6 +608,22 @@ static void spi_dev_set_name(struct spi_device *spi) spi_get_chipselect(spi, 0)); } +/* + * Zero(0) is a valid physical CS value and can be located at any + * logical CS in the spi->chip_select[]. If all the physical CS + * are initialized to 0 then It would be difficult to differentiate + * between a valid physical CS 0 & an unused logical CS whose physical + * CS can be 0. As a solution to this issue initialize all the CS to -1. + * Now all the unused logical CS will have -1 physical CS value & can be + * ignored while performing physical CS validity checks. + */ +#define SPI_INVALID_CS ((s8)-1) + +static inline bool is_valid_cs(s8 chip_select) +{ + return chip_select != SPI_INVALID_CS; +} + static inline int spi_dev_check_cs(struct device *dev, struct spi_device *spi, u8 idx, struct spi_device *new_spi, u8 new_idx) @@ -618,7 +634,7 @@ static inline int spi_dev_check_cs(struct device *dev, cs = spi_get_chipselect(spi, idx); for (idx_new = new_idx; idx_new < SPI_CS_CNT_MAX; idx_new++) { cs_new = spi_get_chipselect(new_spi, idx_new); - if (cs != 0xFF && cs_new != 0xFF && cs == cs_new) { + if (is_valid_cs(cs) && is_valid_cs(cs_new) && cs == cs_new) { dev_err(dev, "chipselect %u already in use\n", cs_new); return -EBUSY; } @@ -658,7 +674,7 @@ static int __spi_add_device(struct spi_device *spi) for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { /* Chipselects are numbered 0..max; validate. */ cs = spi_get_chipselect(spi, idx); - if (cs != 0xFF && cs >= ctlr->num_chipselect) { + if (is_valid_cs(cs) && cs >= ctlr->num_chipselect) { dev_err(dev, "cs%d >= max %d\n", spi_get_chipselect(spi, idx), ctlr->num_chipselect); return -EINVAL; @@ -698,7 +714,7 @@ static int __spi_add_device(struct spi_device *spi) for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { cs = spi_get_chipselect(spi, idx); - if (cs != 0xFF) + if (is_valid_cs(cs)) spi_set_csgpiod(spi, idx, ctlr->cs_gpiods[cs]); } } @@ -756,17 +772,8 @@ static void spi_set_all_cs_unused(struct spi_device *spi) { u8 idx; - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(spi, idx, 0xFF); + spi_set_chipselect(spi, idx, SPI_INVALID_CS); } /** @@ -1050,7 +1057,7 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force) spi->controller->last_cs_index_mask = spi->cs_index_mask; for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi->controller->last_cs[idx] = enable ? spi_get_chipselect(spi, 0) : -1; + spi->controller->last_cs[idx] = enable ? spi_get_chipselect(spi, 0) : SPI_INVALID_CS; spi->controller->last_cs_mode_high = spi->mode & SPI_CS_HIGH; if (spi->mode & SPI_CS_HIGH) @@ -3333,9 +3340,9 @@ int spi_register_controller(struct spi_controller *ctlr) goto free_bus_id; } - /* Setting last_cs to -1 means no chip selected */ + /* Setting last_cs to SPI_INVALID_CS means no chip selected */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - ctlr->last_cs[idx] = -1; + ctlr->last_cs[idx] = SPI_INVALID_CS; status = device_add(&ctlr->dev); if (status < 0) From 3f003fda98a7a8d5f399057d92e6ed56b468657c Mon Sep 17 00:00:00 2001 From: Josua Mayer Date: Thu, 7 Mar 2024 12:06:58 +0100 Subject: [PATCH 1021/1038] hwmon: (amc6821) add of_match table Add of_match table for "ti,amc6821" compatible string. This fixes automatic driver loading by userspace when using device-tree, and if built as a module like major linux distributions do. While devices probe just fine with i2c_device_id table, userspace can't match the "ti,amc6821" compatible string from dt with the plain "amc6821" device id. As a result, the kernel module can not be loaded. Cc: stable@vger.kernel.org Signed-off-by: Josua Mayer Link: https://lore.kernel.org/r/20240307-amc6821-of-match-v1-1-5f40464a3110@solid-run.com [groeck: Cleaned up patch description] Signed-off-by: Guenter Roeck --- drivers/hwmon/amc6821.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c index 2a7a4b6b0094..9b02b304c2f5 100644 --- a/drivers/hwmon/amc6821.c +++ b/drivers/hwmon/amc6821.c @@ -934,10 +934,21 @@ static const struct i2c_device_id amc6821_id[] = { MODULE_DEVICE_TABLE(i2c, amc6821_id); +static const struct of_device_id __maybe_unused amc6821_of_match[] = { + { + .compatible = "ti,amc6821", + .data = (void *)amc6821, + }, + { } +}; + +MODULE_DEVICE_TABLE(of, amc6821_of_match); + static struct i2c_driver amc6821_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "amc6821", + .of_match_table = of_match_ptr(amc6821_of_match), }, .probe = amc6821_probe, .id_table = amc6821_id, From c71d2502edf0a22684928ed3fdae8e1617609985 Mon Sep 17 00:00:00 2001 From: Naresh Solanki Date: Thu, 7 Mar 2024 17:03:24 +0530 Subject: [PATCH 1022/1038] dt-bindings: hwmon: tda38640: Add interrupt & regulator properties tda38640 has a single regulator output along with CAT_FAULT# pin to report internal events. Hence add properties for regulator & interrupt. Signed-off-by: Naresh Solanki Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240307113325.3800181-1-naresh.solanki@9elements.com Signed-off-by: Guenter Roeck --- .../hwmon/pmbus/infineon,tda38640.yaml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml index ded1c115764b..5c4e52b472ad 100644 --- a/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml +++ b/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml @@ -30,6 +30,23 @@ properties: unconnected(has internal pull-down). type: boolean + interrupts: + maxItems: 1 + + regulators: + type: object + description: + list of regulators provided by this controller. + + properties: + vout: + $ref: /schemas/regulator/regulator.yaml# + type: object + + unevaluatedProperties: false + + additionalProperties: false + required: - compatible - reg @@ -38,6 +55,7 @@ additionalProperties: false examples: - | + #include i2c { #address-cells = <1>; #size-cells = <0>; @@ -45,5 +63,15 @@ examples: tda38640@40 { compatible = "infineon,tda38640"; reg = <0x40>; + + interrupt-parent = <&smb_pex_cpu0_event>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; + + regulators { + pvnn_main_cpu0: vout { + regulator-name = "pvnn_main_cpu0"; + regulator-enable-ramp-delay = <200>; + }; + }; }; }; From 3b0ac1f90c44178b295432e1bf7ef05c2ca20014 Mon Sep 17 00:00:00 2001 From: Naresh Solanki Date: Wed, 21 Feb 2024 18:40:23 +0800 Subject: [PATCH 1023/1038] dt-bindings: hwmon: fan: Add fan binding to schema Add common fan properties bindings to a schema. Bindings for fan controllers can reference the common schema for the fan child nodes: patternProperties: "^fan@[0-2]": type: object $ref: fan-common.yaml# unevaluatedProperties: false Reviewed-by: Rob Herring Signed-off-by: Naresh Solanki Signed-off-by: Billy Tsai Link: https://lore.kernel.org/r/20240221104025.1306227-2-billy_tsai@aspeedtech.com Signed-off-by: Guenter Roeck --- .../devicetree/bindings/hwmon/fan-common.yaml | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 Documentation/devicetree/bindings/hwmon/fan-common.yaml diff --git a/Documentation/devicetree/bindings/hwmon/fan-common.yaml b/Documentation/devicetree/bindings/hwmon/fan-common.yaml new file mode 100644 index 000000000000..0fb738081699 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/fan-common.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/fan-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Common Fan Properties + +maintainers: + - Naresh Solanki + - Billy Tsai + +properties: + max-rpm: + description: + Max RPM supported by fan. + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 100000 + + min-rpm: + description: + Min RPM supported by fan. + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 1000 + + pulses-per-revolution: + description: + The number of pulse from fan sensor per revolution. + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 4 + + tach-div: + description: + Divisor for the tach sampling clock, which determines the sensitivity of the tach pin. + $ref: /schemas/types.yaml#/definitions/uint32 + + target-rpm: + description: + The default desired fan speed in RPM. + $ref: /schemas/types.yaml#/definitions/uint32 + + fan-driving-mode: + description: + Select the driving mode of the fan.(DC, PWM and so on) + $ref: /schemas/types.yaml#/definitions/string + enum: [ dc, pwm ] + + pwms: + description: + PWM provider. + maxItems: 1 + + "#cooling-cells": + const: 2 + + cooling-levels: + description: + The control value which correspond to thermal cooling states. + $ref: /schemas/types.yaml#/definitions/uint32-array + + tach-ch: + description: + The tach channel used for the fan. + $ref: /schemas/types.yaml#/definitions/uint8-array + + label: + description: + Optional fan label + + fan-supply: + description: + Power supply for fan. + + reg: + maxItems: 1 + +additionalProperties: true + +... From df9d235c300d143d319ff8373988ef213ec19024 Mon Sep 17 00:00:00 2001 From: Billy Tsai Date: Wed, 21 Feb 2024 18:40:24 +0800 Subject: [PATCH 1024/1038] dt-bindings: hwmon: Support Aspeed g6 PWM TACH Control Document the compatible for aspeed,ast2600-pwm-tach device, which can support up to 16 PWM outputs and 16 fan tach input. Signed-off-by: Billy Tsai Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240221104025.1306227-3-billy_tsai@aspeedtech.com Signed-off-by: Guenter Roeck --- .../bindings/hwmon/aspeed,g6-pwm-tach.yaml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml diff --git a/Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml b/Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml new file mode 100644 index 000000000000..9e5ed901ae54 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) 2023 Aspeed, Inc. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/aspeed,g6-pwm-tach.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ASPEED G6 PWM and Fan Tach controller + +maintainers: + - Billy Tsai + +description: | + The ASPEED PWM controller can support up to 16 PWM outputs. + The ASPEED Fan Tacho controller can support up to 16 fan tach input. + They are independent hardware blocks, which are different from the + previous version of the ASPEED chip. + +properties: + compatible: + enum: + - aspeed,ast2600-pwm-tach + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + "#pwm-cells": + const: 3 + +patternProperties: + "^fan-[0-9]+$": + $ref: fan-common.yaml# + unevaluatedProperties: false + required: + - tach-ch + +required: + - reg + - clocks + - resets + - "#pwm-cells" + - compatible + +additionalProperties: false + +examples: + - | + #include + pwm_tach: pwm-tach-controller@1e610000 { + compatible = "aspeed,ast2600-pwm-tach"; + reg = <0x1e610000 0x100>; + clocks = <&syscon ASPEED_CLK_AHB>; + resets = <&syscon ASPEED_RESET_PWM>; + #pwm-cells = <3>; + + fan-0 { + tach-ch = /bits/ 8 <0x0>; + pwms = <&pwm_tach 0 40000 0>; + }; + + fan-1 { + tach-ch = /bits/ 8 <0x1 0x2>; + pwms = <&pwm_tach 1 40000 0>; + }; + }; From 7e1449cd15d1096157d1a9923b82e37602fb7eb0 Mon Sep 17 00:00:00 2001 From: Billy Tsai Date: Wed, 21 Feb 2024 18:40:25 +0800 Subject: [PATCH 1025/1038] hwmon: (aspeed-g6-pwm-tacho): Support for ASPEED g6 PWM/Fan tach The driver support two functions: PWM and Tachometer. The PWM feature can handle up to 16 output ports, while the Tachometer can monitor to up to 16 input ports as well. This driver implements them by exposing two kernel subsystems: PWM and HWMON. The PWM subsystem can be utilized alongside existing drivers for controlling elements such as fans (pwm-fan.c), beepers (pwm-beeper.c) and so on. Through the HWMON subsystem, the driver provides sysfs interfaces for fan. Signed-off-by: Billy Tsai Link: https://lore.kernel.org/r/20240221104025.1306227-4-billy_tsai@aspeedtech.com Signed-off-by: Guenter Roeck --- Documentation/hwmon/aspeed-g6-pwm-tach.rst | 26 + Documentation/hwmon/index.rst | 1 + drivers/hwmon/Kconfig | 11 + drivers/hwmon/Makefile | 1 + drivers/hwmon/aspeed-g6-pwm-tach.c | 549 +++++++++++++++++++++ 5 files changed, 588 insertions(+) create mode 100644 Documentation/hwmon/aspeed-g6-pwm-tach.rst create mode 100644 drivers/hwmon/aspeed-g6-pwm-tach.c diff --git a/Documentation/hwmon/aspeed-g6-pwm-tach.rst b/Documentation/hwmon/aspeed-g6-pwm-tach.rst new file mode 100644 index 000000000000..17398fe397fe --- /dev/null +++ b/Documentation/hwmon/aspeed-g6-pwm-tach.rst @@ -0,0 +1,26 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver aspeed-g6-pwm-tach +================================= + +Supported chips: + ASPEED AST2600 + +Authors: + + +Description: +------------ +This driver implements support for ASPEED AST2600 Fan Tacho controller. +The controller supports up to 16 tachometer inputs. + +The driver provides the following sensor accesses in sysfs: + +=============== ======= ====================================================== +fanX_input ro provide current fan rotation value in RPM as reported + by the fan to the device. +fanX_div rw Fan divisor: Supported value are power of 4 (1, 4, 16 + 64, ... 4194304) + The larger divisor, the less rpm accuracy and the less + affected by fan signal glitch. +=============== ======= ====================================================== diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index c4af2a894c42..1ca7a4fe1f8f 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -44,6 +44,7 @@ Hardware Monitoring Kernel Drivers aquacomputer_d5next asb100 asc7621 + aspeed-g6-pwm-tach aspeed-pwm-tacho asus_ec_sensors asus_rog_ryujin diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 56260821d658..83945397b6eb 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -422,6 +422,17 @@ config SENSORS_ASPEED This driver can also be built as a module. If so, the module will be called aspeed_pwm_tacho. +config SENSORS_ASPEED_G6 + tristate "ASPEED g6 PWM and Fan tach driver" + depends on ARCH_ASPEED || COMPILE_TEST + depends on PWM + help + This driver provides support for ASPEED G6 PWM and Fan Tach + controllers. + + This driver can also be built as a module. If so, the module + will be called aspeed_pwm_tacho. + config SENSORS_ATXP1 tristate "Attansic ATXP1 VID controller" depends on I2C diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index f45c31aff009..5c31808f6378 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -55,6 +55,7 @@ obj-$(CONFIG_SENSORS_ARM_SCPI) += scpi-hwmon.o obj-$(CONFIG_SENSORS_AS370) += as370-hwmon.o obj-$(CONFIG_SENSORS_ASC7621) += asc7621.o obj-$(CONFIG_SENSORS_ASPEED) += aspeed-pwm-tacho.o +obj-$(CONFIG_SENSORS_ASPEED_G6) += aspeed-g6-pwm-tach.o obj-$(CONFIG_SENSORS_ASUS_ROG_RYUJIN) += asus_rog_ryujin.o obj-$(CONFIG_SENSORS_ATXP1) += atxp1.o obj-$(CONFIG_SENSORS_AXI_FAN_CONTROL) += axi-fan-control.o diff --git a/drivers/hwmon/aspeed-g6-pwm-tach.c b/drivers/hwmon/aspeed-g6-pwm-tach.c new file mode 100644 index 000000000000..597b3b019d49 --- /dev/null +++ b/drivers/hwmon/aspeed-g6-pwm-tach.c @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2021 Aspeed Technology Inc. + * + * PWM/TACH controller driver for Aspeed ast2600 SoCs. + * This drivers doesn't support earlier version of the IP. + * + * The hardware operates in time quantities of length + * Q := (DIV_L + 1) << DIV_H / input-clk + * The length of a PWM period is (DUTY_CYCLE_PERIOD + 1) * Q. + * The maximal value for DUTY_CYCLE_PERIOD is used here to provide + * a fine grained selection for the duty cycle. + * + * This driver uses DUTY_CYCLE_RISING_POINT = 0, so from the start of a + * period the output is active until DUTY_CYCLE_FALLING_POINT * Q. Note + * that if DUTY_CYCLE_RISING_POINT = DUTY_CYCLE_FALLING_POINT the output is + * always active. + * + * Register usage: + * PIN_ENABLE: When it is unset the pwm controller will emit inactive level to the external. + * Use to determine whether the PWM channel is enabled or disabled + * CLK_ENABLE: When it is unset the pwm controller will assert the duty counter reset and + * emit inactive level to the PIN_ENABLE mux after that the driver can still change the pwm period + * and duty and the value will apply when CLK_ENABLE be set again. + * Use to determine whether duty_cycle bigger than 0. + * PWM_ASPEED_CTRL_INVERSE: When it is toggled the output value will inverse immediately. + * PWM_ASPEED_DUTY_CYCLE_FALLING_POINT/PWM_ASPEED_DUTY_CYCLE_RISING_POINT: When these two + * values are equal it means the duty cycle = 100%. + * + * The glitch may generate at: + * - Enabled changing when the duty_cycle bigger than 0% and less than 100%. + * - Polarity changing when the duty_cycle bigger than 0% and less than 100%. + * + * Limitations: + * - When changing both duty cycle and period, we cannot prevent in + * software that the output might produce a period with mixed + * settings. + * - Disabling the PWM doesn't complete the current period. + * + * Improvements: + * - When only changing one of duty cycle or period, our pwm controller will not + * generate the glitch, the configure will change at next cycle of pwm. + * This improvement can disable/enable through PWM_ASPEED_CTRL_DUTY_SYNC_DISABLE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The channel number of Aspeed pwm controller */ +#define PWM_ASPEED_NR_PWMS 16 +/* PWM Control Register */ +#define PWM_ASPEED_CTRL(ch) ((ch) * 0x10 + 0x00) +#define PWM_ASPEED_CTRL_LOAD_SEL_RISING_AS_WDT BIT(19) +#define PWM_ASPEED_CTRL_DUTY_LOAD_AS_WDT_ENABLE BIT(18) +#define PWM_ASPEED_CTRL_DUTY_SYNC_DISABLE BIT(17) +#define PWM_ASPEED_CTRL_CLK_ENABLE BIT(16) +#define PWM_ASPEED_CTRL_LEVEL_OUTPUT BIT(15) +#define PWM_ASPEED_CTRL_INVERSE BIT(14) +#define PWM_ASPEED_CTRL_OPEN_DRAIN_ENABLE BIT(13) +#define PWM_ASPEED_CTRL_PIN_ENABLE BIT(12) +#define PWM_ASPEED_CTRL_CLK_DIV_H GENMASK(11, 8) +#define PWM_ASPEED_CTRL_CLK_DIV_L GENMASK(7, 0) + +/* PWM Duty Cycle Register */ +#define PWM_ASPEED_DUTY_CYCLE(ch) ((ch) * 0x10 + 0x04) +#define PWM_ASPEED_DUTY_CYCLE_PERIOD GENMASK(31, 24) +#define PWM_ASPEED_DUTY_CYCLE_POINT_AS_WDT GENMASK(23, 16) +#define PWM_ASPEED_DUTY_CYCLE_FALLING_POINT GENMASK(15, 8) +#define PWM_ASPEED_DUTY_CYCLE_RISING_POINT GENMASK(7, 0) + +/* PWM fixed value */ +#define PWM_ASPEED_FIXED_PERIOD FIELD_MAX(PWM_ASPEED_DUTY_CYCLE_PERIOD) + +/* The channel number of Aspeed tach controller */ +#define TACH_ASPEED_NR_TACHS 16 +/* TACH Control Register */ +#define TACH_ASPEED_CTRL(ch) (((ch) * 0x10) + 0x08) +#define TACH_ASPEED_IER BIT(31) +#define TACH_ASPEED_INVERS_LIMIT BIT(30) +#define TACH_ASPEED_LOOPBACK BIT(29) +#define TACH_ASPEED_ENABLE BIT(28) +#define TACH_ASPEED_DEBOUNCE_MASK GENMASK(27, 26) +#define TACH_ASPEED_DEBOUNCE_BIT 26 +#define TACH_ASPEED_IO_EDGE_MASK GENMASK(25, 24) +#define TACH_ASPEED_IO_EDGE_BIT 24 +#define TACH_ASPEED_CLK_DIV_T_MASK GENMASK(23, 20) +#define TACH_ASPEED_CLK_DIV_BIT 20 +#define TACH_ASPEED_THRESHOLD_MASK GENMASK(19, 0) +/* [27:26] */ +#define DEBOUNCE_3_CLK 0x00 +#define DEBOUNCE_2_CLK 0x01 +#define DEBOUNCE_1_CLK 0x02 +#define DEBOUNCE_0_CLK 0x03 +/* [25:24] */ +#define F2F_EDGES 0x00 +#define R2R_EDGES 0x01 +#define BOTH_EDGES 0x02 +/* [23:20] */ +/* divisor = 4 to the nth power, n = register value */ +#define DEFAULT_TACH_DIV 1024 +#define DIV_TO_REG(divisor) (ilog2(divisor) >> 1) + +/* TACH Status Register */ +#define TACH_ASPEED_STS(ch) (((ch) * 0x10) + 0x0C) + +/*PWM_TACH_STS */ +#define TACH_ASPEED_ISR BIT(31) +#define TACH_ASPEED_PWM_OUT BIT(25) +#define TACH_ASPEED_PWM_OEN BIT(24) +#define TACH_ASPEED_DEB_INPUT BIT(23) +#define TACH_ASPEED_RAW_INPUT BIT(22) +#define TACH_ASPEED_VALUE_UPDATE BIT(21) +#define TACH_ASPEED_FULL_MEASUREMENT BIT(20) +#define TACH_ASPEED_VALUE_MASK GENMASK(19, 0) +/********************************************************** + * Software setting + *********************************************************/ +#define DEFAULT_FAN_PULSE_PR 2 + +struct aspeed_pwm_tach_data { + struct device *dev; + void __iomem *base; + struct clk *clk; + struct reset_control *reset; + unsigned long clk_rate; + struct pwm_chip chip; + bool tach_present[TACH_ASPEED_NR_TACHS]; + u32 tach_divisor; +}; + +static inline struct aspeed_pwm_tach_data * +aspeed_pwm_chip_to_data(struct pwm_chip *chip) +{ + return container_of(chip, struct aspeed_pwm_tach_data, chip); +} + +static int aspeed_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, + struct pwm_state *state) +{ + struct aspeed_pwm_tach_data *priv = aspeed_pwm_chip_to_data(chip); + u32 hwpwm = pwm->hwpwm; + bool polarity, pin_en, clk_en; + u32 duty_pt, val; + u64 div_h, div_l, duty_cycle_period, dividend; + + val = readl(priv->base + PWM_ASPEED_CTRL(hwpwm)); + polarity = FIELD_GET(PWM_ASPEED_CTRL_INVERSE, val); + pin_en = FIELD_GET(PWM_ASPEED_CTRL_PIN_ENABLE, val); + clk_en = FIELD_GET(PWM_ASPEED_CTRL_CLK_ENABLE, val); + div_h = FIELD_GET(PWM_ASPEED_CTRL_CLK_DIV_H, val); + div_l = FIELD_GET(PWM_ASPEED_CTRL_CLK_DIV_L, val); + val = readl(priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + duty_pt = FIELD_GET(PWM_ASPEED_DUTY_CYCLE_FALLING_POINT, val); + duty_cycle_period = FIELD_GET(PWM_ASPEED_DUTY_CYCLE_PERIOD, val); + /* + * This multiplication doesn't overflow, the upper bound is + * 1000000000 * 256 * 256 << 15 = 0x1dcd650000000000 + */ + dividend = (u64)NSEC_PER_SEC * (div_l + 1) * (duty_cycle_period + 1) + << div_h; + state->period = DIV_ROUND_UP_ULL(dividend, priv->clk_rate); + + if (clk_en && duty_pt) { + dividend = (u64)NSEC_PER_SEC * (div_l + 1) * duty_pt + << div_h; + state->duty_cycle = DIV_ROUND_UP_ULL(dividend, priv->clk_rate); + } else { + state->duty_cycle = clk_en ? state->period : 0; + } + state->polarity = polarity ? PWM_POLARITY_INVERSED : PWM_POLARITY_NORMAL; + state->enabled = pin_en; + return 0; +} + +static int aspeed_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, + const struct pwm_state *state) +{ + struct aspeed_pwm_tach_data *priv = aspeed_pwm_chip_to_data(chip); + u32 hwpwm = pwm->hwpwm, duty_pt, val; + u64 div_h, div_l, divisor, expect_period; + bool clk_en; + + expect_period = div64_u64(ULLONG_MAX, (u64)priv->clk_rate); + expect_period = min(expect_period, state->period); + dev_dbg(chip->dev, "expect period: %lldns, duty_cycle: %lldns", + expect_period, state->duty_cycle); + /* + * Pick the smallest value for div_h so that div_l can be the biggest + * which results in a finer resolution near the target period value. + */ + divisor = (u64)NSEC_PER_SEC * (PWM_ASPEED_FIXED_PERIOD + 1) * + (FIELD_MAX(PWM_ASPEED_CTRL_CLK_DIV_L) + 1); + div_h = order_base_2(DIV64_U64_ROUND_UP(priv->clk_rate * expect_period, divisor)); + if (div_h > 0xf) + div_h = 0xf; + + divisor = ((u64)NSEC_PER_SEC * (PWM_ASPEED_FIXED_PERIOD + 1)) << div_h; + div_l = div64_u64(priv->clk_rate * expect_period, divisor); + + if (div_l == 0) + return -ERANGE; + + div_l -= 1; + + if (div_l > 255) + div_l = 255; + + dev_dbg(chip->dev, "clk source: %ld div_h %lld, div_l : %lld\n", + priv->clk_rate, div_h, div_l); + /* duty_pt = duty_cycle * (PERIOD + 1) / period */ + duty_pt = div64_u64(state->duty_cycle * priv->clk_rate, + (u64)NSEC_PER_SEC * (div_l + 1) << div_h); + dev_dbg(chip->dev, "duty_cycle = %lld, duty_pt = %d\n", + state->duty_cycle, duty_pt); + + /* + * Fixed DUTY_CYCLE_PERIOD to its max value to get a + * fine-grained resolution for duty_cycle at the expense of a + * coarser period resolution. + */ + val = readl(priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + val &= ~PWM_ASPEED_DUTY_CYCLE_PERIOD; + val |= FIELD_PREP(PWM_ASPEED_DUTY_CYCLE_PERIOD, + PWM_ASPEED_FIXED_PERIOD); + writel(val, priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + + if (duty_pt == 0) { + /* emit inactive level and assert the duty counter reset */ + clk_en = 0; + } else { + clk_en = 1; + if (duty_pt >= (PWM_ASPEED_FIXED_PERIOD + 1)) + duty_pt = 0; + val = readl(priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + val &= ~(PWM_ASPEED_DUTY_CYCLE_RISING_POINT | + PWM_ASPEED_DUTY_CYCLE_FALLING_POINT); + val |= FIELD_PREP(PWM_ASPEED_DUTY_CYCLE_FALLING_POINT, duty_pt); + writel(val, priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + } + + val = readl(priv->base + PWM_ASPEED_CTRL(hwpwm)); + val &= ~(PWM_ASPEED_CTRL_CLK_DIV_H | PWM_ASPEED_CTRL_CLK_DIV_L | + PWM_ASPEED_CTRL_PIN_ENABLE | PWM_ASPEED_CTRL_CLK_ENABLE | + PWM_ASPEED_CTRL_INVERSE); + val |= FIELD_PREP(PWM_ASPEED_CTRL_CLK_DIV_H, div_h) | + FIELD_PREP(PWM_ASPEED_CTRL_CLK_DIV_L, div_l) | + FIELD_PREP(PWM_ASPEED_CTRL_PIN_ENABLE, state->enabled) | + FIELD_PREP(PWM_ASPEED_CTRL_CLK_ENABLE, clk_en) | + FIELD_PREP(PWM_ASPEED_CTRL_INVERSE, state->polarity); + writel(val, priv->base + PWM_ASPEED_CTRL(hwpwm)); + + return 0; +} + +static const struct pwm_ops aspeed_pwm_ops = { + .apply = aspeed_pwm_apply, + .get_state = aspeed_pwm_get_state, +}; + +static void aspeed_tach_ch_enable(struct aspeed_pwm_tach_data *priv, u8 tach_ch, + bool enable) +{ + if (enable) + writel(readl(priv->base + TACH_ASPEED_CTRL(tach_ch)) | + TACH_ASPEED_ENABLE, + priv->base + TACH_ASPEED_CTRL(tach_ch)); + else + writel(readl(priv->base + TACH_ASPEED_CTRL(tach_ch)) & + ~TACH_ASPEED_ENABLE, + priv->base + TACH_ASPEED_CTRL(tach_ch)); +} + +static int aspeed_tach_val_to_rpm(struct aspeed_pwm_tach_data *priv, u32 tach_val) +{ + u64 rpm; + u32 tach_div; + + tach_div = tach_val * priv->tach_divisor * DEFAULT_FAN_PULSE_PR; + + dev_dbg(priv->dev, "clk %ld, tach_val %d , tach_div %d\n", + priv->clk_rate, tach_val, tach_div); + + rpm = (u64)priv->clk_rate * 60; + do_div(rpm, tach_div); + + return (int)rpm; +} + +static int aspeed_get_fan_tach_ch_rpm(struct aspeed_pwm_tach_data *priv, + u8 fan_tach_ch) +{ + u32 val; + + val = readl(priv->base + TACH_ASPEED_STS(fan_tach_ch)); + + if (!(val & TACH_ASPEED_FULL_MEASUREMENT)) + return 0; + val = FIELD_GET(TACH_ASPEED_VALUE_MASK, val); + return aspeed_tach_val_to_rpm(priv, val); +} + +static int aspeed_tach_hwmon_read(struct device *dev, + enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct aspeed_pwm_tach_data *priv = dev_get_drvdata(dev); + u32 reg_val; + + switch (attr) { + case hwmon_fan_input: + *val = aspeed_get_fan_tach_ch_rpm(priv, channel); + break; + case hwmon_fan_div: + reg_val = readl(priv->base + TACH_ASPEED_CTRL(channel)); + reg_val = FIELD_GET(TACH_ASPEED_CLK_DIV_T_MASK, reg_val); + *val = BIT(reg_val << 1); + break; + default: + return -EOPNOTSUPP; + } + return 0; +} + +static int aspeed_tach_hwmon_write(struct device *dev, + enum hwmon_sensor_types type, u32 attr, + int channel, long val) +{ + struct aspeed_pwm_tach_data *priv = dev_get_drvdata(dev); + u32 reg_val; + + switch (attr) { + case hwmon_fan_div: + if (!is_power_of_2(val) || (ilog2(val) % 2) || + DIV_TO_REG(val) > 0xb) + return -EINVAL; + priv->tach_divisor = val; + reg_val = readl(priv->base + TACH_ASPEED_CTRL(channel)); + reg_val &= ~TACH_ASPEED_CLK_DIV_T_MASK; + reg_val |= FIELD_PREP(TACH_ASPEED_CLK_DIV_T_MASK, + DIV_TO_REG(priv->tach_divisor)); + writel(reg_val, priv->base + TACH_ASPEED_CTRL(channel)); + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static umode_t aspeed_tach_dev_is_visible(const void *drvdata, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct aspeed_pwm_tach_data *priv = drvdata; + + if (!priv->tach_present[channel]) + return 0; + switch (attr) { + case hwmon_fan_input: + return 0444; + case hwmon_fan_div: + return 0644; + } + return 0; +} + +static const struct hwmon_ops aspeed_tach_ops = { + .is_visible = aspeed_tach_dev_is_visible, + .read = aspeed_tach_hwmon_read, + .write = aspeed_tach_hwmon_write, +}; + +static const struct hwmon_channel_info *aspeed_tach_info[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV), + NULL +}; + +static const struct hwmon_chip_info aspeed_tach_chip_info = { + .ops = &aspeed_tach_ops, + .info = aspeed_tach_info, +}; + +static void aspeed_present_fan_tach(struct aspeed_pwm_tach_data *priv, u8 *tach_ch, int count) +{ + u8 ch, index; + u32 val; + + for (index = 0; index < count; index++) { + ch = tach_ch[index]; + priv->tach_present[ch] = true; + priv->tach_divisor = DEFAULT_TACH_DIV; + + val = readl(priv->base + TACH_ASPEED_CTRL(ch)); + val &= ~(TACH_ASPEED_INVERS_LIMIT | TACH_ASPEED_DEBOUNCE_MASK | + TACH_ASPEED_IO_EDGE_MASK | TACH_ASPEED_CLK_DIV_T_MASK | + TACH_ASPEED_THRESHOLD_MASK); + val |= (DEBOUNCE_3_CLK << TACH_ASPEED_DEBOUNCE_BIT) | + F2F_EDGES | + FIELD_PREP(TACH_ASPEED_CLK_DIV_T_MASK, + DIV_TO_REG(priv->tach_divisor)); + writel(val, priv->base + TACH_ASPEED_CTRL(ch)); + + aspeed_tach_ch_enable(priv, ch, true); + } +} + +static int aspeed_create_fan_monitor(struct device *dev, + struct device_node *child, + struct aspeed_pwm_tach_data *priv) +{ + int ret, count; + u8 *tach_ch; + + count = of_property_count_u8_elems(child, "tach-ch"); + if (count < 1) + return -EINVAL; + tach_ch = devm_kcalloc(dev, count, sizeof(*tach_ch), GFP_KERNEL); + if (!tach_ch) + return -ENOMEM; + ret = of_property_read_u8_array(child, "tach-ch", tach_ch, count); + if (ret) + return ret; + + aspeed_present_fan_tach(priv, tach_ch, count); + + return 0; +} + +static void aspeed_pwm_tach_reset_assert(void *data) +{ + struct reset_control *rst = data; + + reset_control_assert(rst); +} + +static int aspeed_pwm_tach_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev, *hwmon; + int ret; + struct device_node *child; + struct aspeed_pwm_tach_data *priv; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + priv->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(priv->clk)) + return dev_err_probe(dev, PTR_ERR(priv->clk), + "Couldn't get clock\n"); + priv->clk_rate = clk_get_rate(priv->clk); + priv->reset = devm_reset_control_get_exclusive(dev, NULL); + if (IS_ERR(priv->reset)) + return dev_err_probe(dev, PTR_ERR(priv->reset), + "Couldn't get reset control\n"); + + ret = reset_control_deassert(priv->reset); + if (ret) + return dev_err_probe(dev, ret, + "Couldn't deassert reset control\n"); + ret = devm_add_action_or_reset(dev, aspeed_pwm_tach_reset_assert, + priv->reset); + if (ret) + return ret; + + priv->chip.dev = dev; + priv->chip.ops = &aspeed_pwm_ops; + priv->chip.npwm = PWM_ASPEED_NR_PWMS; + + ret = devm_pwmchip_add(dev, &priv->chip); + if (ret) + return dev_err_probe(dev, ret, "Failed to add PWM chip\n"); + + for_each_child_of_node(dev->of_node, child) { + ret = aspeed_create_fan_monitor(dev, child, priv); + if (ret) { + of_node_put(child); + dev_warn(dev, "Failed to create fan %d", ret); + return 0; + } + } + + hwmon = devm_hwmon_device_register_with_info(dev, "aspeed_tach", priv, + &aspeed_tach_chip_info, NULL); + ret = PTR_ERR_OR_ZERO(hwmon); + if (ret) + return dev_err_probe(dev, ret, + "Failed to register hwmon device\n"); + + of_platform_populate(dev->of_node, NULL, NULL, dev); + + return 0; +} + +static int aspeed_pwm_tach_remove(struct platform_device *pdev) +{ + struct aspeed_pwm_tach_data *priv = platform_get_drvdata(pdev); + + reset_control_assert(priv->reset); + + return 0; +} + +static const struct of_device_id aspeed_pwm_tach_match[] = { + { + .compatible = "aspeed,ast2600-pwm-tach", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, aspeed_pwm_tach_match); + +static struct platform_driver aspeed_pwm_tach_driver = { + .probe = aspeed_pwm_tach_probe, + .remove = aspeed_pwm_tach_remove, + .driver = { + .name = "aspeed-g6-pwm-tach", + .of_match_table = aspeed_pwm_tach_match, + }, +}; + +module_platform_driver(aspeed_pwm_tach_driver); + +MODULE_AUTHOR("Billy Tsai "); +MODULE_DESCRIPTION("Aspeed ast2600 PWM and Fan Tach device driver"); +MODULE_LICENSE("GPL"); From cb824724dccb3195d22cad96e7b65fe13621d0a6 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Thu, 7 Mar 2024 16:02:35 -0500 Subject: [PATCH 1026/1038] dm vdo: document minimum metadata size requirements Signed-off-by: Matthew Sakai Signed-off-by: Mike Snitzer --- Documentation/admin-guide/device-mapper/vdo.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/admin-guide/device-mapper/vdo.rst b/Documentation/admin-guide/device-mapper/vdo.rst index a3572582ead8..7e1ecafdf91e 100644 --- a/Documentation/admin-guide/device-mapper/vdo.rst +++ b/Documentation/admin-guide/device-mapper/vdo.rst @@ -34,6 +34,17 @@ The repo also contains additional userspace tools which can be used to inspect a vdo target's on-disk metadata. Fortunately, these tools are rarely needed except by dm-vdo developers. +Metadata requirements +===================== + +Each vdo volume reserves 3GB of space for metadata, or more depending on +its configuration. It is helpful to check that the space saved by +deduplication and compression is not cancelled out by the metadata +requirements. An estimation of the space saved for a specific dataset can +be computed with the vdo estimator tool, which is available at: + +https://github.com/dm-vdo/vdoestimator/ + Target interface ================ From a0c8bf0a474eea8ee2590332a1ebdec6048ce40c Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 7 Mar 2024 05:27:38 +0000 Subject: [PATCH 1027/1038] iommu/amd: Fix sleeping in atomic context Commit cf70873e3d01 ("iommu/amd: Refactor GCR3 table helper functions") changed GFP flag we use for GCR3 table. Original plan was to move GCR3 table allocation outside spinlock. But this requires complete rework of attach device path. Hence we didn't do it as part of SVA series. For now revert the GFP flag to ATOMIC (same as original code). Fixes: cf70873e3d01 ("iommu/amd: Refactor GCR3 table helper functions") Reported-by: Dan Carpenter Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20240307052738.116035-1-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f688443c2764..d35c1b8c8e65 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1793,7 +1793,7 @@ static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, /* Allocate per device domain ID */ gcr3_info->domid = domain_id_alloc(); - gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_KERNEL); + gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_ATOMIC); if (gcr3_info->gcr3_tbl == NULL) { domain_id_free(gcr3_info->domid); return -ENOMEM; From 70bad345e622c23bb530016925c936ab04a646ac Mon Sep 17 00:00:00 2001 From: Bert Karwatzki Date: Thu, 7 Mar 2024 20:44:19 +0100 Subject: [PATCH 1028/1038] iommu: Fix compilation without CONFIG_IOMMU_INTEL When the kernel is comiled with CONFIG_IRQ_REMAP=y but without CONFIG_IOMMU_INTEL compilation fails since commit def054b01a8678 with an undefined reference to device_rbtree_find(). This patch makes sure that intel specific code is only compiled with CONFIG_IOMMU_INTEL=y. Signed-off-by: Bert Karwatzki Fixes: 80a9b50c0b9e ("iommu/vt-d: Improve ITE fault handling if target device isn't present") Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20240307194419.15801-1-spasswolf@web.de Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 2 +- drivers/iommu/intel/Makefile | 2 ++ drivers/iommu/irq_remapping.c | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 9a29d742617e..9dbb55e745bd 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -196,7 +196,7 @@ source "drivers/iommu/iommufd/Kconfig" config IRQ_REMAP bool "Support for Interrupt Remapping" depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI - select DMAR_TABLE + select DMAR_TABLE if INTEL_IOMMU help Supports Interrupt remapping for IO-APIC and MSI devices. To use x2apic mode in the CPU's which support x2APIC enhancements or diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile index 5dabf081a779..5402b699a122 100644 --- a/drivers/iommu/intel/Makefile +++ b/drivers/iommu/intel/Makefile @@ -5,5 +5,7 @@ obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o obj-$(CONFIG_DMAR_PERF) += perf.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o +ifdef CONFIG_INTEL_IOMMU obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o +endif obj-$(CONFIG_INTEL_IOMMU_PERF_EVENTS) += perfmon.o diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 83314b9d8f38..ee59647c2050 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -99,7 +99,8 @@ int __init irq_remapping_prepare(void) if (disable_irq_remap) return -ENOSYS; - if (intel_irq_remap_ops.prepare() == 0) + if (IS_ENABLED(CONFIG_INTEL_IOMMU) && + intel_irq_remap_ops.prepare() == 0) remap_ops = &intel_irq_remap_ops; else if (IS_ENABLED(CONFIG_AMD_IOMMU) && amd_iommu_irq_ops.prepare() == 0) From 8636f19c2d1f8199b27b4559d9caa115b3011f06 Mon Sep 17 00:00:00 2001 From: Alexander Sverdlin Date: Thu, 7 Mar 2024 22:43:16 +0100 Subject: [PATCH 1029/1038] gpio: sysfs: repair export returning -EPERM on 1st attempt It would make sense to return -EPERM if the bit was already set (already used), not if it was cleared. Before this fix pins can only be exported on the 2nd attempt: $ echo 522 > /sys/class/gpio/export sh: write error: Operation not permitted $ echo 522 > /sys/class/gpio/export Fixes: 35b545332b80 ("gpio: remove gpio_lock") Signed-off-by: Alexander Sverdlin Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 67fc09a57f26..6853ecd98bcb 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -593,7 +593,7 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) if (!guard.gc) return -ENODEV; - if (!test_and_set_bit(FLAG_EXPORT, &desc->flags)) + if (test_and_set_bit(FLAG_EXPORT, &desc->flags)) return -EPERM; gdev = desc->gdev; From 8debe3c1295ef36958dae77487eed9cf6584c008 Mon Sep 17 00:00:00 2001 From: Armin Wolf Date: Sat, 9 Mar 2024 22:20:25 +0100 Subject: [PATCH 1030/1038] hwmon: (dell-smm) Add XPS 9315 to fan control whitelist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user reported that on this machine, disabling BIOS fan control is necessary in order to change the fan speed. Signed-off-by: Armin Wolf Acked-by: Pali Rohár Link: https://lore.kernel.org/r/20240309212025.13758-1-W_Armin@gmx.de Signed-off-by: Guenter Roeck --- drivers/hwmon/dell-smm-hwmon.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 6d8c0f328b7b..4fa837e65a61 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1450,10 +1450,15 @@ struct i8k_fan_control_data { }; enum i8k_fan_controls { + I8K_FAN_30A3_31A3, I8K_FAN_34A3_35A3, }; static const struct i8k_fan_control_data i8k_fan_control_data[] __initconst = { + [I8K_FAN_30A3_31A3] = { + .manual_fan = 0x30a3, + .auto_fan = 0x31a3, + }, [I8K_FAN_34A3_35A3] = { .manual_fan = 0x34a3, .auto_fan = 0x35a3, @@ -1517,6 +1522,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = { }, .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_34A3_35A3], }, + { + .ident = "Dell XPS 9315", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 9315"), + }, + .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3], + }, { } }; From abb3f9717a67a2666b2bc2f19543a657e3d4ad63 Mon Sep 17 00:00:00 2001 From: Sibi Sankar Date: Tue, 27 Feb 2024 23:04:32 +0530 Subject: [PATCH 1031/1038] OPP: Extend dev_pm_opp_data with turbo support Let's extend the dev_pm_opp_data with a turbo variable, to allow users to specify if it's a boost frequency for a dynamically added OPP. Signed-off-by: Sibi Sankar Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 1 + include/linux/pm_opp.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index c4e0432ae42a..e233734b7220 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2065,6 +2065,7 @@ int _opp_add_v1(struct opp_table *opp_table, struct device *dev, /* populate the opp table */ new_opp->rates[0] = data->freq; new_opp->level = data->level; + new_opp->turbo = data->turbo; tol = u_volt * opp_table->voltage_tolerance_v1 / 100; new_opp->supplies[0].u_volt = u_volt; new_opp->supplies[0].u_volt_min = u_volt - tol; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 76dcb7f37bcd..fa9b63c6bf0b 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -87,12 +87,14 @@ struct dev_pm_opp_config { /** * struct dev_pm_opp_data - The data to use to initialize an OPP. + * @turbo: Flag to indicate whether the OPP is to be marked turbo or not. * @level: The performance level for the OPP. Set level to OPP_LEVEL_UNSET if * level field isn't used. * @freq: The clock rate in Hz for the OPP. * @u_volt: The voltage in uV for the OPP. */ struct dev_pm_opp_data { + bool turbo; unsigned int level; unsigned long freq; unsigned long u_volt; From 838a4772bfc390a14b31c25dc4c9eb66de5f5b1a Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 18 Jan 2024 16:19:13 +0530 Subject: [PATCH 1032/1038] cpufreq: Move dev_pm_opp_{init|free}_cpufreq_table() to pm_opp.h Move the declaration of functions defined in the OPP core to pm_opp.h. These were added to cpufreq.h as it was the only user of the APIs, but that was a mistake perhaps. Fix it. Signed-off-by: Viresh Kumar --- include/linux/cpufreq.h | 20 -------------------- include/linux/pm_opp.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index afda5f24d3dd..8ff3e79727d8 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -694,26 +694,6 @@ struct cpufreq_frequency_table { * order */ }; -#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP) -int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table); -void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table); -#else -static inline int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table - **table) -{ - return -EINVAL; -} - -static inline void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table - **table) -{ -} -#endif - /* * cpufreq_for_each_entry - iterate over a cpufreq_frequency_table * @pos: the cpufreq_frequency_table * to use as a loop cursor. diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index fa9b63c6bf0b..065a47382302 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -16,6 +16,7 @@ #include struct clk; +struct cpufreq_frequency_table; struct regulator; struct dev_pm_opp; struct device; @@ -446,6 +447,21 @@ static inline int dev_pm_opp_sync_regulators(struct device *dev) #endif /* CONFIG_PM_OPP */ +#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP) +int dev_pm_opp_init_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table); +void dev_pm_opp_free_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table); +#else +static inline int dev_pm_opp_init_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table) +{ + return -EINVAL; +} + +static inline void dev_pm_opp_free_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table) +{ +} +#endif + + #if defined(CONFIG_PM_OPP) && defined(CONFIG_OF) int dev_pm_opp_of_add_table(struct device *dev); int dev_pm_opp_of_add_table_indexed(struct device *dev, int index); From 992e88335997a2f171c7da221a14c40771cd517d Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 27 Sep 2023 16:28:16 +0530 Subject: [PATCH 1033/1038] OPP: debugfs: Fix warning with W=1 builds We currently get the following warning: debugfs.c:105:54: error: '%d' directive output may be truncated writing between 1 and 11 bytes into a region of size 8 [-Werror=format-truncation=] snprintf(name, sizeof(name), "supply-%d", i); ^~ debugfs.c:105:46: note: directive argument in the range [-2147483644, 2147483646] snprintf(name, sizeof(name), "supply-%d", i); ^~~~~~~~~~~ debugfs.c:105:17: note: 'snprintf' output between 9 and 19 bytes into a destination of size 15 snprintf(name, sizeof(name), "supply-%d", i); Fix this and other potential issues it by allocating larger arrays. Use the exact string format to allocate the arrays without getting into these issues again. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402141313.81ltVF5g-lkp@intel.com/ Signed-off-by: Viresh Kumar Reviewed-by: Dhruva Gole --- drivers/opp/debugfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c index ec030b19164a..22cfde403625 100644 --- a/drivers/opp/debugfs.c +++ b/drivers/opp/debugfs.c @@ -56,11 +56,11 @@ static void opp_debug_create_bw(struct dev_pm_opp *opp, struct dentry *pdentry) { struct dentry *d; - char name[20]; + char name[] = "icc-path-XXXXXXXXXXX"; /* Integers can take 11 chars max */ int i; for (i = 0; i < opp_table->path_count; i++) { - snprintf(name, sizeof(name), "icc-path-%.1d", i); + snprintf(name, sizeof(name), "icc-path-%d", i); /* Create per-path directory */ d = debugfs_create_dir(name, pdentry); @@ -78,7 +78,7 @@ static void opp_debug_create_clks(struct dev_pm_opp *opp, struct opp_table *opp_table, struct dentry *pdentry) { - char name[12]; + char name[] = "rate_hz_XXXXXXXXXXX"; /* Integers can take 11 chars max */ int i; if (opp_table->clk_count == 1) { @@ -100,7 +100,7 @@ static void opp_debug_create_supplies(struct dev_pm_opp *opp, int i; for (i = 0; i < opp_table->regulator_count; i++) { - char name[15]; + char name[] = "supply-XXXXXXXXXXX"; /* Integers can take 11 chars max */ snprintf(name, sizeof(name), "supply-%d", i); From 28330ceb953e39880ea77da4895bb902a1244860 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 4 Mar 2024 16:48:28 +0530 Subject: [PATCH 1034/1038] OPP: debugfs: Fix warning around icc_get_name() If the kernel isn't built with interconnect support, icc_get_name() returns NULL and we get following warning: drivers/opp/debugfs.c: In function 'bw_name_read': drivers/opp/debugfs.c:43:42: error: '%.62s' directive argument is null [-Werror=format-overflow=] i = scnprintf(buf, sizeof(buf), "%.62s\n", icc_get_name(path)); Fix it. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402141313.81ltVF5g-lkp@intel.com/ Fixes: 0430b1d5704b0 ("opp: Expose bandwidth information via debugfs") Signed-off-by: Viresh Kumar Reviewed-by: Dhruva Gole --- drivers/opp/debugfs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c index 22cfde403625..105de7c3274a 100644 --- a/drivers/opp/debugfs.c +++ b/drivers/opp/debugfs.c @@ -37,10 +37,12 @@ static ssize_t bw_name_read(struct file *fp, char __user *userbuf, size_t count, loff_t *ppos) { struct icc_path *path = fp->private_data; + const char *name = icc_get_name(path); char buf[64]; - int i; + int i = 0; - i = scnprintf(buf, sizeof(buf), "%.62s\n", icc_get_name(path)); + if (name) + i = scnprintf(buf, sizeof(buf), "%.62s\n", name); return simple_read_from_buffer(userbuf, count, ppos, buf, i); } From 13c8cf339e1a7a3d3e48fdebbb882b3a5a90f708 Mon Sep 17 00:00:00 2001 From: David Heidelberg Date: Tue, 5 Mar 2024 00:43:06 +0100 Subject: [PATCH 1035/1038] dt-bindings: opp: drop maxItems from inner items With recent changes within matrix dimensions calculation, dropping maxItems: 1 provides a warning-free run. Fixes warning such as: arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dtb: opp-table: opp-200000000:opp-hz:0: [200000000, 0, 0, 150000000, 0, 0, 0, 0, 300000000] is too long Fixes: 3cb16ad69bef ("dt-bindings: opp: accept array of frequencies") Suggested-by: Rob Herring Signed-off-by: David Heidelberg Reviewed-by: Dhruva Gole Acked-by: Rob Herring Signed-off-by: Viresh Kumar --- Documentation/devicetree/bindings/opp/opp-v2-base.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Documentation/devicetree/bindings/opp/opp-v2-base.yaml b/Documentation/devicetree/bindings/opp/opp-v2-base.yaml index e2f8f7af3cf4..b1bb87c865ed 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-base.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-base.yaml @@ -57,8 +57,6 @@ patternProperties: specific binding. minItems: 1 maxItems: 32 - items: - maxItems: 1 opp-microvolt: description: | From b25b8f4b8ecef0f48c05f0c3572daeabefe16526 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Mon, 11 Mar 2024 13:42:55 -0400 Subject: [PATCH 1036/1038] dm raid: fix false positive for requeue needed during reshape An empty flush doesn't have a payload, so it should never be looked at when considering to possibly requeue a bio for the case when a reshape is in progress. Fixes: 9dbd1aa3a81c ("dm raid: add reshaping support to the target") Reported-by: Patrick Plenefisch Signed-off-by: Ming Lei Signed-off-by: Mike Snitzer --- drivers/md/dm-raid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 6bb1765be1e5..b3c8920fe723 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -3329,14 +3329,14 @@ static int raid_map(struct dm_target *ti, struct bio *bio) struct mddev *mddev = &rs->md; /* - * If we're reshaping to add disk(s)), ti->len and + * If we're reshaping to add disk(s), ti->len and * mddev->array_sectors will differ during the process * (ti->len > mddev->array_sectors), so we have to requeue * bios with addresses > mddev->array_sectors here or * there will occur accesses past EOD of the component * data images thus erroring the raid set. */ - if (unlikely(bio_end_sector(bio) > mddev->array_sectors)) + if (unlikely(bio_has_data(bio) && bio_end_sector(bio) > mddev->array_sectors)) return DM_MAPIO_REQUEUE; md_handle_request(mddev, bio); From 65e8fbde64520001abf1c8d0e573561b4746ef38 Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Mon, 11 Mar 2024 15:06:39 +0100 Subject: [PATCH 1037/1038] dm: call the resume method on internal suspend There is this reported crash when experimenting with the lvm2 testsuite. The list corruption is caused by the fact that the postsuspend and resume methods were not paired correctly; there were two consecutive calls to the origin_postsuspend function. The second call attempts to remove the "hash_list" entry from a list, while it was already removed by the first call. Fix __dm_internal_resume so that it calls the preresume and resume methods of the table's targets. If a preresume method of some target fails, we are in a tricky situation. We can't return an error because dm_internal_resume isn't supposed to return errors. We can't return success, because then the "resume" and "postsuspend" methods would not be paired correctly. So, we set the DMF_SUSPENDED flag and we fake normal suspend - it may confuse userspace tools, but it won't cause a kernel crash. ------------[ cut here ]------------ kernel BUG at lib/list_debug.c:56! invalid opcode: 0000 [#1] PREEMPT SMP CPU: 1 PID: 8343 Comm: dmsetup Not tainted 6.8.0-rc6 #4 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014 RIP: 0010:__list_del_entry_valid_or_report+0x77/0xc0 RSP: 0018:ffff8881b831bcc0 EFLAGS: 00010282 RAX: 000000000000004e RBX: ffff888143b6eb80 RCX: 0000000000000000 RDX: 0000000000000001 RSI: ffffffff819053d0 RDI: 00000000ffffffff RBP: ffff8881b83a3400 R08: 00000000fffeffff R09: 0000000000000058 R10: 0000000000000000 R11: ffffffff81a24080 R12: 0000000000000001 R13: ffff88814538e000 R14: ffff888143bc6dc0 R15: ffffffffa02e4bb0 FS: 00000000f7c0f780(0000) GS:ffff8893f0a40000(0000) knlGS:0000000000000000 CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033 CR2: 0000000057fb5000 CR3: 0000000143474000 CR4: 00000000000006b0 Call Trace: ? die+0x2d/0x80 ? do_trap+0xeb/0xf0 ? __list_del_entry_valid_or_report+0x77/0xc0 ? do_error_trap+0x60/0x80 ? __list_del_entry_valid_or_report+0x77/0xc0 ? exc_invalid_op+0x49/0x60 ? __list_del_entry_valid_or_report+0x77/0xc0 ? asm_exc_invalid_op+0x16/0x20 ? table_deps+0x1b0/0x1b0 [dm_mod] ? __list_del_entry_valid_or_report+0x77/0xc0 origin_postsuspend+0x1a/0x50 [dm_snapshot] dm_table_postsuspend_targets+0x34/0x50 [dm_mod] dm_suspend+0xd8/0xf0 [dm_mod] dev_suspend+0x1f2/0x2f0 [dm_mod] ? table_deps+0x1b0/0x1b0 [dm_mod] ctl_ioctl+0x300/0x5f0 [dm_mod] dm_compat_ctl_ioctl+0x7/0x10 [dm_mod] __x64_compat_sys_ioctl+0x104/0x170 do_syscall_64+0x184/0x1b0 entry_SYSCALL_64_after_hwframe+0x46/0x4e RIP: 0033:0xf7e6aead ---[ end trace 0000000000000000 ]--- Fixes: ffcc39364160 ("dm: enhance internal suspend and resume interface") Signed-off-by: Mikulas Patocka Signed-off-by: Mike Snitzer --- drivers/md/dm.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 1bed115a7e8e..f1fe8828e1de 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2945,6 +2945,9 @@ static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend static void __dm_internal_resume(struct mapped_device *md) { + int r; + struct dm_table *map; + BUG_ON(!md->internal_suspend_count); if (--md->internal_suspend_count) @@ -2953,12 +2956,23 @@ static void __dm_internal_resume(struct mapped_device *md) if (dm_suspended_md(md)) goto done; /* resume from nested suspend */ - /* - * NOTE: existing callers don't need to call dm_table_resume_targets - * (which may fail -- so best to avoid it for now by passing NULL map) - */ - (void) __dm_resume(md, NULL); - + map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); + r = __dm_resume(md, map); + if (r) { + /* + * If a preresume method of some target failed, we are in a + * tricky situation. We can't return an error to the caller. We + * can't fake success because then the "resume" and + * "postsuspend" methods would not be paired correctly, and it + * would break various targets, for example it would cause list + * corruption in the "origin" target. + * + * So, we fake normal suspend here, to make sure that the + * "resume" and "postsuspend" methods will be paired correctly. + */ + DMERR("Preresume method failed: %d", r); + set_bit(DMF_SUSPENDED, &md->flags); + } done: clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); smp_mb__after_atomic(); From f88c3fb81c4badb46c2fef7d168ff138043e86bb Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Tue, 12 Mar 2024 20:32:19 -0700 Subject: [PATCH 1038/1038] mm, slab: remove last vestiges of SLAB_MEM_SPREAD Yes, yes, I know the slab people were planning on going slow and letting every subsystem fight this thing on their own. But let's just rip off the band-aid and get it over and done with. I don't want to see a number of unnecessary pull requests just to get rid of a flag that no longer has any meaning. This was mainly done with a couple of 'sed' scripts and then some manual cleanup of the end result. Link: https://lore.kernel.org/all/CAHk-=wji0u+OOtmAOD-5JV3SXcRJF___k_+8XNKmak0yd5vW1Q@mail.gmail.com/ Signed-off-by: Linus Torvalds --- drivers/dax/super.c | 3 +-- drivers/usb/isp1760/isp1760-hcd.c | 8 +++----- fs/9p/v9fs.c | 2 +- fs/adfs/super.c | 2 +- fs/befs/linuxvfs.c | 3 +-- fs/bfs/inode.c | 2 +- fs/ceph/super.c | 18 +++++++++--------- fs/coda/inode.c | 4 ++-- fs/erofs/super.c | 2 +- fs/exfat/cache.c | 2 +- fs/exfat/super.c | 2 +- fs/ext2/super.c | 3 +-- fs/ext4/super.c | 3 +-- fs/fat/cache.c | 2 +- fs/fat/inode.c | 2 +- fs/freevxfs/vxfs_super.c | 2 +- fs/gfs2/main.c | 1 - fs/hpfs/super.c | 2 +- fs/isofs/inode.c | 2 +- fs/jffs2/super.c | 2 +- fs/nfs/direct.c | 3 +-- fs/nfs/inode.c | 2 +- fs/nfs/nfs42xattr.c | 2 +- fs/ntfs3/super.c | 2 +- fs/ocfs2/dlmfs/dlmfs.c | 2 +- fs/ocfs2/super.c | 7 +++---- fs/overlayfs/super.c | 2 +- fs/qnx4/inode.c | 2 +- fs/quota/dquot.c | 2 +- fs/smb/client/cifsfs.c | 2 +- fs/tracefs/inode.c | 1 - fs/ubifs/super.c | 4 ++-- fs/udf/super.c | 1 - fs/ufs/super.c | 3 +-- fs/vboxsf/super.c | 3 +-- fs/xfs/xfs_super.c | 7 +++---- fs/zonefs/super.c | 2 +- include/linux/slab.h | 3 --- net/sunrpc/rpc_pipe.c | 2 +- 39 files changed, 51 insertions(+), 68 deletions(-) diff --git a/drivers/dax/super.c b/drivers/dax/super.c index f4b635526345..a0244f6bb44b 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -532,8 +532,7 @@ static int dax_fs_init(void) int rc; dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0, - (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, init_once); if (!dax_cache) return -ENOMEM; diff --git a/drivers/usb/isp1760/isp1760-hcd.c b/drivers/usb/isp1760/isp1760-hcd.c index 76862ba40f35..0e5e4cb74c87 100644 --- a/drivers/usb/isp1760/isp1760-hcd.c +++ b/drivers/usb/isp1760/isp1760-hcd.c @@ -2521,21 +2521,19 @@ static const struct hc_driver isp1760_hc_driver = { int __init isp1760_init_kmem_once(void) { urb_listitem_cachep = kmem_cache_create("isp1760_urb_listitem", - sizeof(struct urb_listitem), 0, SLAB_TEMPORARY | - SLAB_MEM_SPREAD, NULL); + sizeof(struct urb_listitem), 0, SLAB_TEMPORARY, NULL); if (!urb_listitem_cachep) return -ENOMEM; qtd_cachep = kmem_cache_create("isp1760_qtd", - sizeof(struct isp1760_qtd), 0, SLAB_TEMPORARY | - SLAB_MEM_SPREAD, NULL); + sizeof(struct isp1760_qtd), 0, SLAB_TEMPORARY, NULL); if (!qtd_cachep) goto destroy_urb_listitem; qh_cachep = kmem_cache_create("isp1760_qh", sizeof(struct isp1760_qh), - 0, SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); + 0, SLAB_TEMPORARY, NULL); if (!qh_cachep) goto destroy_qtd; diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index 61dbe52bb3a3..281a1ed03a04 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -637,7 +637,7 @@ static int v9fs_init_inode_cache(void) v9fs_inode_cache = kmem_cache_create("v9fs_inode_cache", sizeof(struct v9fs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), v9fs_inode_init_once); if (!v9fs_inode_cache) return -ENOMEM; diff --git a/fs/adfs/super.c b/fs/adfs/super.c index e8bfc38239cd..9354b14bbfe3 100644 --- a/fs/adfs/super.c +++ b/fs/adfs/super.c @@ -249,7 +249,7 @@ static int __init init_inodecache(void) adfs_inode_cachep = kmem_cache_create("adfs_inode_cache", sizeof(struct adfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (adfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c index 2b4dda047450..d76f406d3b2e 100644 --- a/fs/befs/linuxvfs.c +++ b/fs/befs/linuxvfs.c @@ -435,8 +435,7 @@ befs_init_inodecache(void) { befs_inode_cachep = kmem_cache_create_usercopy("befs_inode_cache", sizeof(struct befs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct befs_inode_info, i_data.symlink), sizeof_field(struct befs_inode_info, diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c index 355957dbce39..db81570c9637 100644 --- a/fs/bfs/inode.c +++ b/fs/bfs/inode.c @@ -259,7 +259,7 @@ static int __init init_inodecache(void) bfs_inode_cachep = kmem_cache_create("bfs_inode_cache", sizeof(struct bfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (bfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/ceph/super.c b/fs/ceph/super.c index 5ec102f6b1ac..885cb5d4e771 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -928,36 +928,36 @@ static int __init init_caches(void) ceph_inode_cachep = kmem_cache_create("ceph_inode_info", sizeof(struct ceph_inode_info), __alignof__(struct ceph_inode_info), - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT, ceph_inode_init_once); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + ceph_inode_init_once); if (!ceph_inode_cachep) return -ENOMEM; - ceph_cap_cachep = KMEM_CACHE(ceph_cap, SLAB_MEM_SPREAD); + ceph_cap_cachep = KMEM_CACHE(ceph_cap, 0); if (!ceph_cap_cachep) goto bad_cap; - ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, SLAB_MEM_SPREAD); + ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, 0); if (!ceph_cap_snap_cachep) goto bad_cap_snap; ceph_cap_flush_cachep = KMEM_CACHE(ceph_cap_flush, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD); + SLAB_RECLAIM_ACCOUNT); if (!ceph_cap_flush_cachep) goto bad_cap_flush; ceph_dentry_cachep = KMEM_CACHE(ceph_dentry_info, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD); + SLAB_RECLAIM_ACCOUNT); if (!ceph_dentry_cachep) goto bad_dentry; - ceph_file_cachep = KMEM_CACHE(ceph_file_info, SLAB_MEM_SPREAD); + ceph_file_cachep = KMEM_CACHE(ceph_file_info, 0); if (!ceph_file_cachep) goto bad_file; - ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, SLAB_MEM_SPREAD); + ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, 0); if (!ceph_dir_file_cachep) goto bad_dir_file; - ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, SLAB_MEM_SPREAD); + ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, 0); if (!ceph_mds_request_cachep) goto bad_mds_req; diff --git a/fs/coda/inode.c b/fs/coda/inode.c index a50356c541f6..6898dc621011 100644 --- a/fs/coda/inode.c +++ b/fs/coda/inode.c @@ -72,8 +72,8 @@ int __init coda_init_inodecache(void) { coda_inode_cachep = kmem_cache_create("coda_inode_cache", sizeof(struct coda_inode_info), 0, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT, init_once); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); if (coda_inode_cachep == NULL) return -ENOMEM; return 0; diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 6fbb1fba2d31..69308fd73e4a 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -857,7 +857,7 @@ static int __init erofs_module_init(void) erofs_inode_cachep = kmem_cache_create("erofs_inode", sizeof(struct erofs_inode), 0, - SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, erofs_inode_init_once); if (!erofs_inode_cachep) return -ENOMEM; diff --git a/fs/exfat/cache.c b/fs/exfat/cache.c index 5a2f119b7e8c..7cc200d89821 100644 --- a/fs/exfat/cache.c +++ b/fs/exfat/cache.c @@ -46,7 +46,7 @@ int exfat_cache_init(void) { exfat_cachep = kmem_cache_create("exfat_cache", sizeof(struct exfat_cache), - 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, exfat_cache_init_once); if (!exfat_cachep) return -ENOMEM; diff --git a/fs/exfat/super.c b/fs/exfat/super.c index fcb658267765..3d5ea2cfad66 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -813,7 +813,7 @@ static int __init init_exfat_fs(void) exfat_inode_cachep = kmem_cache_create("exfat_inode_cache", sizeof(struct exfat_inode_info), - 0, SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, exfat_inode_init_once); if (!exfat_inode_cachep) { err = -ENOMEM; diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 01f9addc8b1f..cabea887314d 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -213,8 +213,7 @@ static int __init init_inodecache(void) { ext2_inode_cachep = kmem_cache_create_usercopy("ext2_inode_cache", sizeof(struct ext2_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct ext2_inode_info, i_data), sizeof_field(struct ext2_inode_info, i_data), init_once); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index a8ba84eabab2..59c72b6dd153 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1500,8 +1500,7 @@ static int __init init_inodecache(void) { ext4_inode_cachep = kmem_cache_create_usercopy("ext4_inode_cache", sizeof(struct ext4_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct ext4_inode_info, i_data), sizeof_field(struct ext4_inode_info, i_data), init_once); diff --git a/fs/fat/cache.c b/fs/fat/cache.c index 738e427e2d21..2af424e200b3 100644 --- a/fs/fat/cache.c +++ b/fs/fat/cache.c @@ -47,7 +47,7 @@ int __init fat_cache_init(void) { fat_cache_cachep = kmem_cache_create("fat_cache", sizeof(struct fat_cache), - 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, init_once); if (fat_cache_cachep == NULL) return -ENOMEM; diff --git a/fs/fat/inode.c b/fs/fat/inode.c index 5c813696d1ff..d9e6fbb6f246 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -787,7 +787,7 @@ static int __init fat_init_inodecache(void) fat_inode_cachep = kmem_cache_create("fat_inode_cache", sizeof(struct msdos_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (fat_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/freevxfs/vxfs_super.c b/fs/freevxfs/vxfs_super.c index e6e2a2185e7c..42e03b6b1cc7 100644 --- a/fs/freevxfs/vxfs_super.c +++ b/fs/freevxfs/vxfs_super.c @@ -307,7 +307,7 @@ vxfs_init(void) vxfs_inode_cachep = kmem_cache_create_usercopy("vxfs_inode", sizeof(struct vxfs_inode_info), 0, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, offsetof(struct vxfs_inode_info, vii_immed.vi_immed), sizeof_field(struct vxfs_inode_info, vii_immed.vi_immed), diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c index 79be0cdc730c..04cadc02e5a6 100644 --- a/fs/gfs2/main.c +++ b/fs/gfs2/main.c @@ -111,7 +111,6 @@ static int __init init_gfs2_fs(void) gfs2_inode_cachep = kmem_cache_create("gfs2_inode", sizeof(struct gfs2_inode), 0, SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD| SLAB_ACCOUNT, gfs2_init_inode_once); if (!gfs2_inode_cachep) diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c index 6b0ba3c1efba..314834a078e9 100644 --- a/fs/hpfs/super.c +++ b/fs/hpfs/super.c @@ -255,7 +255,7 @@ static int init_inodecache(void) hpfs_inode_cachep = kmem_cache_create("hpfs_inode_cache", sizeof(struct hpfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (hpfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index 3e4d53e26f94..25fca44149dd 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -93,7 +93,7 @@ static int __init init_inodecache(void) isofs_inode_cachep = kmem_cache_create("isofs_inode_cache", sizeof(struct iso_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (!isofs_inode_cachep) return -ENOMEM; diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c index f99591a634b4..aede1be4dc0c 100644 --- a/fs/jffs2/super.c +++ b/fs/jffs2/super.c @@ -387,7 +387,7 @@ static int __init init_jffs2_fs(void) jffs2_inode_cachep = kmem_cache_create("jffs2_i", sizeof(struct jffs2_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), jffs2_i_init_once); if (!jffs2_inode_cachep) { pr_err("error: Failed to initialise inode cache\n"); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index c03926a1cc73..7af5d270de28 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -1037,8 +1037,7 @@ int __init nfs_init_directcache(void) { nfs_direct_cachep = kmem_cache_create("nfs_direct_cache", sizeof(struct nfs_direct_req), - 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD), + 0, SLAB_RECLAIM_ACCOUNT, NULL); if (nfs_direct_cachep == NULL) return -ENOMEM; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index ebb8d60e1152..93ea49a7eb61 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -2372,7 +2372,7 @@ static int __init nfs_init_inodecache(void) nfs_inode_cachep = kmem_cache_create("nfs_inode_cache", sizeof(struct nfs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (nfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/nfs/nfs42xattr.c b/fs/nfs/nfs42xattr.c index 49aaf28a6950..b6e3d8f77b91 100644 --- a/fs/nfs/nfs42xattr.c +++ b/fs/nfs/nfs42xattr.c @@ -1017,7 +1017,7 @@ int __init nfs4_xattr_cache_init(void) nfs4_xattr_cache_cachep = kmem_cache_create("nfs4_xattr_cache_cache", sizeof(struct nfs4_xattr_cache), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD), + (SLAB_RECLAIM_ACCOUNT), nfs4_xattr_cache_init_once); if (nfs4_xattr_cache_cachep == NULL) return -ENOMEM; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index cef5467fd928..9df7c20d066f 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -1825,7 +1825,7 @@ static int __init init_ntfs_fs(void) ntfs_inode_cachep = kmem_cache_create( "ntfs_inode_cache", sizeof(struct ntfs_inode), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), init_once); if (!ntfs_inode_cachep) { err = -ENOMEM; diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index 85215162c9dd..7fc0e920eda7 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -578,7 +578,7 @@ static int __init init_dlmfs_fs(void) dlmfs_inode_cache = kmem_cache_create("dlmfs_inode_cache", sizeof(struct dlmfs_inode_private), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), dlmfs_init_once); if (!dlmfs_inode_cache) { status = -ENOMEM; diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index a70aff17d455..b3f860888e93 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1706,18 +1706,17 @@ static int ocfs2_initialize_mem_caches(void) sizeof(struct ocfs2_inode_info), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), ocfs2_inode_init_once); ocfs2_dquot_cachep = kmem_cache_create("ocfs2_dquot_cache", sizeof(struct ocfs2_dquot), 0, - (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD), + (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT), NULL); ocfs2_qf_chunk_cachep = kmem_cache_create("ocfs2_qf_chunk_cache", sizeof(struct ocfs2_quota_chunk), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD), + (SLAB_RECLAIM_ACCOUNT), NULL); if (!ocfs2_inode_cachep || !ocfs2_dquot_cachep || !ocfs2_qf_chunk_cachep) { diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 36d4b8b1f784..a40fc7e05525 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1503,7 +1503,7 @@ static int __init ovl_init(void) ovl_inode_cachep = kmem_cache_create("ovl_inode", sizeof(struct ovl_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), ovl_inode_init_once); if (ovl_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c index 7b5711f76709..d79841e94428 100644 --- a/fs/qnx4/inode.c +++ b/fs/qnx4/inode.c @@ -378,7 +378,7 @@ static int init_inodecache(void) qnx4_inode_cachep = kmem_cache_create("qnx4_inode_cache", sizeof(struct qnx4_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (qnx4_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 1f0c754416b6..eb6e9d95dea1 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -2984,7 +2984,7 @@ static int __init dquot_init(void) dquot_cachep = kmem_cache_create("dquot", sizeof(struct dquot), sizeof(unsigned long) * 4, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_PANIC), + SLAB_PANIC), NULL); order = 0; diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index fb368b191eef..e0d8c79cdde1 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -1664,7 +1664,7 @@ cifs_init_inodecache(void) cifs_inode_cachep = kmem_cache_create("cifs_inode_cache", sizeof(struct cifsInodeInfo), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), cifs_init_once); if (cifs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index d65ffad4c327..5545e6bf7d26 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -731,7 +731,6 @@ static int __init tracefs_init(void) tracefs_inode_cachep = kmem_cache_create("tracefs_inode_cache", sizeof(struct tracefs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD| SLAB_ACCOUNT), init_once); if (!tracefs_inode_cachep) diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index d2881041b393..7f4031a15f4d 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -2434,8 +2434,8 @@ static int __init ubifs_init(void) ubifs_inode_slab = kmem_cache_create("ubifs_inode_slab", sizeof(struct ubifs_inode), 0, - SLAB_MEM_SPREAD | SLAB_RECLAIM_ACCOUNT | - SLAB_ACCOUNT, &inode_slab_ctor); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + &inode_slab_ctor); if (!ubifs_inode_slab) return -ENOMEM; diff --git a/fs/udf/super.c b/fs/udf/super.c index 928a04d9d9e0..6f420f4ca005 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -177,7 +177,6 @@ static int __init init_inodecache(void) udf_inode_cachep = kmem_cache_create("udf_inode_cache", sizeof(struct udf_inode_info), 0, (SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD | SLAB_ACCOUNT), init_once); if (!udf_inode_cachep) diff --git a/fs/ufs/super.c b/fs/ufs/super.c index a480810cd4e3..44666afc6209 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1470,8 +1470,7 @@ static int __init init_inodecache(void) { ufs_inode_cachep = kmem_cache_create_usercopy("ufs_inode_cache", sizeof(struct ufs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), offsetof(struct ufs_inode_info, i_u1.i_symlink), sizeof_field(struct ufs_inode_info, i_u1.i_symlink), diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c index 1fb8f4df60cb..cabe8ac4fefc 100644 --- a/fs/vboxsf/super.c +++ b/fs/vboxsf/super.c @@ -339,8 +339,7 @@ static int vboxsf_setup(void) vboxsf_inode_cachep = kmem_cache_create("vboxsf_inode_cache", sizeof(struct vboxsf_inode), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, vboxsf_inode_init_once); if (!vboxsf_inode_cachep) { err = -ENOMEM; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 00fbd5b6e582..59c8c0541bdd 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -2043,8 +2043,7 @@ xfs_init_caches(void) xfs_buf_cache = kmem_cache_create("xfs_buf", sizeof(struct xfs_buf), 0, SLAB_HWCACHE_ALIGN | - SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, NULL); if (!xfs_buf_cache) goto out; @@ -2109,14 +2108,14 @@ xfs_init_caches(void) sizeof(struct xfs_inode), 0, (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD | SLAB_ACCOUNT), + SLAB_ACCOUNT), xfs_fs_inode_init_once); if (!xfs_inode_cache) goto out_destroy_efi_cache; xfs_ili_cache = kmem_cache_create("xfs_ili", sizeof(struct xfs_inode_log_item), 0, - SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, NULL); if (!xfs_ili_cache) goto out_destroy_inode_cache; diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 236a6d88306f..c6a124e8d565 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1422,7 +1422,7 @@ static int __init zonefs_init_inodecache(void) { zonefs_inode_cachep = kmem_cache_create("zonefs_inode_cache", sizeof(struct zonefs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, NULL); if (zonefs_inode_cachep == NULL) return -ENOMEM; diff --git a/include/linux/slab.h b/include/linux/slab.h index f6323763cd61..e53cbfa18325 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -202,9 +202,6 @@ enum _slab_flag_bits { #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ -/* Obsolete unused flag, to be removed */ -#define SLAB_MEM_SPREAD __SLAB_FLAG_UNUSED - /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c index dcc2b4f49e77..910a5d850d04 100644 --- a/net/sunrpc/rpc_pipe.c +++ b/net/sunrpc/rpc_pipe.c @@ -1490,7 +1490,7 @@ int register_rpc_pipefs(void) rpc_inode_cachep = kmem_cache_create("rpc_inode_cache", sizeof(struct rpc_inode), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (!rpc_inode_cachep) return -ENOMEM;